diff --git a/.air.toml b/.air.toml index fa8d9aa1..b7dbf81e 100644 --- a/.air.toml +++ b/.air.toml @@ -7,21 +7,21 @@ tmp_dir = "tmp" bin = "./tmp/main" cmd = "go build -o ./tmp/main ./cmd" delay = 1000 - exclude_dir = ["assets", "tmp", "vendor", "testdata"] - exclude_file = [] + exclude_dir = ["assets", "tmp", "vendor", "testdata",] + exclude_file = ["proxychain/ruleset/rule_resmod_types.gen.go", "proxychain/ruleset/rule_reqmod_types.gen.go", "handlers/api_modifiers_structdef.gen.go"] exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false - full_bin = "RULESET=./ruleset.yaml ./tmp/main" + full_bin = "./tmp/main --ruleset ./rulesets_v2" include_dir = [] - include_ext = ["go", "tpl", "tmpl", "yaml", "html"] + include_ext = ["go", "tpl", "tmpl", "yaml", "html", "js"] include_file = [] kill_delay = "0s" log = "build-errors.log" poll = false poll_interval = 0 post_cmd = [] - pre_cmd = ["echo 'dev' > handlers/VERSION"] + pre_cmd = ["git submodule update --init --recursive; git rev-parse --short HEAD > handlers/VERSION; git rev-parse --short HEAD > cmd/VERSION; cd proxychain/codegen && go run codegen.go && cd ../../handlers/api_modifiers_codegen && go run api_modifiers_codegen.go"] rerun = false rerun_delay = 500 send_interrupt = false diff --git a/.github/workflows/build-css.yaml b/.github/workflows/build-css.yaml index 47a33659..5ad392dd 100644 --- a/.github/workflows/build-css.yaml +++ b/.github/workflows/build-css.yaml @@ -3,7 +3,10 @@ name: Build Tailwind CSS on: push: paths: + - "handlers/error_page.html" - "handlers/form.html" + - "handlers/playground.html" + - "proxychain/responsemodifiers/vendor/generate_readable_outline.html" workflow_dispatch: jobs: @@ -27,16 +30,17 @@ jobs: name: Build Tailwind CSS run: pnpm build - - name: Commit generated stylesheet + name: Commit generated stylesheet for handlers/styles.css run: | - if git diff --quiet cmd/styles.css; then + if git diff --quiet handlers/styles.css; then echo "No changes to commit." exit 0 else echo "Changes detected, committing..." git config --global user.name "Github action" git config --global user.email "username@users.noreply.github.com" - git add cmd + git add handlers + git add proxychain/responsemodifiers/vendor/ git commit -m "Generated stylesheet" git push - fi \ No newline at end of file + fi diff --git a/.gitignore b/.gitignore index 212e5021..f943b087 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # dev binary ladder +tmp/main +tmp VERSION -output.css \ No newline at end of file +output.css +.aider* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..1c1ef8c7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "proxychain/responsemodifiers/vendor/ddg-tracker-surrogates"] + path = proxychain/responsemodifiers/vendor/ddg-tracker-surrogates + url = https://github.com/duckduckgo/tracker-surrogates +[submodule "proxychain/requestmodifiers/vendor/ua-parser-js"] + path = proxychain/requestmodifiers/vendor/ua-parser-js + url = https://github.com/faisalman/ua-parser-js.git diff --git a/Dockerfile b/Dockerfile index 07005717..c8f9ad4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY . . RUN go mod download -RUN CGO_ENABLED=0 GOOS=linux go build -o ladder cmd/main.go +RUN make build FROM debian:12-slim as release @@ -18,8 +18,4 @@ RUN chmod +x /app/ladder RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/* -#EXPOSE 8080 - -#ENTRYPOINT ["/usr/bin/dumb-init", "--"] - CMD ["sh", "-c", "/app/ladder"] \ No newline at end of file diff --git a/Makefile b/Makefile index 98f30974..b2c7a3d0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,11 @@ +build: + cd proxychain/codegen && go run codegen.go + cd handlers/api_modifiers_codegen && go run api_modifiers_codegen.go + git submodule update --init --recursive + git rev-parse --short HEAD > handlers/VERSION + git rev-parse --short HEAD > cmd/VERSION + go build -o ladder -ldflags="-s -w" cmd/main.go + lint: gofumpt -l -w . golangci-lint run -c .golangci-lint.yaml --fix @@ -7,4 +15,7 @@ lint: install-linters: go install mvdan.cc/gofumpt@latest - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 \ No newline at end of file + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 + +run: + go run ./cmd/. diff --git a/README.md b/README.md index be1b96fd..d07fe0a9 100644 --- a/README.md +++ b/README.md @@ -91,18 +91,28 @@ Or create a bookmark with the following URL: ```javascript javascript:window.location.href="http://localhost:8080/"+location.href ``` +### Outline +```bash +curl -X GET "http://localhost:8080/outline/https://www.example.com" +``` ### API ```bash -curl -X GET "http://localhost:8080/api/https://www.example.com" +curl -X GET "http://localhost:8080/api/content/https://www.example.com" ``` ### RAW -http://localhost:8080/raw/https://www.example.com +http://localhost:8080/api/raw/https://www.example.com ### Running Ruleset -http://localhost:8080/ruleset +http://localhost:8080/api/ruleset + +### Running Rule +http://localhost:8080/api/ruleset/https://example.com + +### List available modifiers +http://localhost:8080/api/modifiers ## Configuration @@ -189,7 +199,10 @@ There is a basic ruleset available in a separate repository [ruleset.yaml](https To run a development server at http://localhost:8080: ```bash -echo "dev" > handlers/VERSION +git clone git@github.com-ladddder:everywall/ladder.git +git submodule update --init --recursive +echo "dev " > handlers/VERSION +echo "dev " > cmd/VERSION RULESET="./ruleset.yaml" go run cmd/main.go ``` diff --git a/cmd/main.go b/cmd/main.go index d28787d1..08aa78ae 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,26 +1,23 @@ package main import ( - "embed" + _ "embed" "fmt" "log" "os" - "strings" - "ladder/handlers" - "ladder/handlers/cli" + "github.com/everywall/ladder/handlers" + "github.com/everywall/ladder/internal/cli" + + "github.com/everywall/ladder/proxychain/requestmodifiers/bot" + ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" - "github.com/gofiber/fiber/v2/middleware/basicauth" - "github.com/gofiber/fiber/v2/middleware/favicon" ) -//go:embed favicon.ico -var faviconData string - -//go:embed styles.css -var cssData embed.FS +//go:embed VERSION +var version string func main() { parser := argparse.NewParser("ladder", "Every Wall needs a Ladder") @@ -41,6 +38,23 @@ func main() { Help: "This will spawn multiple processes listening", }) + verbose := parser.Flag("v", "verbose", &argparse.Options{ + Required: false, + Help: "Adds verbose logging", + }) + + randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request", + }) + + randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request", + }) + + // TODO: add version flag that reads from handers/VERSION + ruleset := parser.String("r", "ruleset", &argparse.Options{ Required: false, Help: "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.", @@ -51,14 +65,9 @@ func main() { Help: "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.", }) - mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{ - Required: false, - Help: "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.", - }) - mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{ Required: false, - Help: "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.", + Help: "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.", }) err := parser.Parse(os.Args) @@ -66,20 +75,36 @@ func main() { fmt.Print(parser.Usage(err)) } + if *randomGoogleBot { + err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + if err != nil { + fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Googlebot identity") + } + } + + if *randomBingBot { + err := bot.BingBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json") + if err != nil { + fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Bingbot identity") + } + } + // utility cli flag to compile ruleset directory into single ruleset.yaml - if *mergeRulesets || *mergeRulesetsGzip { + if *mergeRulesets { output := os.Stdout if *mergeRulesetsOutput != "" { output, err = os.Create(*mergeRulesetsOutput) - + if err != nil { fmt.Println(err) os.Exit(1) } } - err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output) + err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output) if err != nil { fmt.Println(err) os.Exit(1) @@ -91,28 +116,30 @@ func main() { *prefork = true } + var rs ruleset_v2.IRuleset + + switch { + case *ruleset != "": + rs, err = ruleset_v2.NewRuleset(*ruleset) + if err != nil { + fmt.Printf("ERROR: failed to load ruleset from %s\n", *ruleset) + } + case os.Getenv("RULESET") != "": + rs = ruleset_v2.NewRulesetFromEnv() + } + app := fiber.New( fiber.Config{ - Prefork: *prefork, - GETOnly: true, + Prefork: *prefork, + GETOnly: false, + ReadBufferSize: 4096 * 4, // increase max header size + DisableStartupMessage: true, }, ) - userpass := os.Getenv("USERPASS") - if userpass != "" { - userpass := strings.Split(userpass, ":") - - app.Use(basicauth.New(basicauth.Config{ - Users: map[string]string{ - userpass[0]: userpass[1], - }, - })) - } - - app.Use(favicon.New(favicon.Config{ - Data: []byte(faviconData), - URL: "/favicon.ico", - })) + app.Use(handlers.Auth()) + app.Use(handlers.Favicon()) + app.Use(handlers.RenderErrorPage()) if os.Getenv("NOLOGS") != "true" { app.Use(func(c *fiber.Ctx) error { @@ -122,23 +149,28 @@ func main() { }) } + proxyOpts := &handlers.ProxyOptions{ + Verbose: *verbose, + Ruleset: rs, + } + app.Get("/", handlers.Form) - app.Get("/styles.css", func(c *fiber.Ctx) error { - cssData, err := cssData.ReadFile("styles.css") - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") - } + app.Get("styles.css", handlers.Styles) + app.Get("script.js", handlers.Script) + app.Get("playground-script.js", handlers.Script) - c.Set("Content-Type", "text/css") + app.All("api/raw/*", handlers.NewRawProxySiteHandler(proxyOpts)) - return c.Send(cssData) - }) + app.Get("api/modifiers", handlers.NewAPIModifersListHandler(proxyOpts)) + app.Get("api/ruleset/*", handlers.NewRulesetSiteHandler(proxyOpts)) + app.Get("api/content/*", handlers.NewAPIContentHandler("api/outline/*", proxyOpts)) + + app.Get("outline/*", handlers.NewOutlineHandler("outline/*", proxyOpts)) + app.All("playground/*", handlers.PlaygroundHandler("playground/*", proxyOpts)) - app.Get("ruleset", handlers.Ruleset) - app.Get("raw/*", handlers.Raw) - app.Get("api/*", handlers.Api) - app.Get("/*", handlers.ProxySite(*ruleset)) + app.All("/*", handlers.NewProxySiteHandler(proxyOpts)) + fmt.Println(cli.StartupMessage(version, *port, *ruleset)) log.Fatal(app.Listen(":" + *port)) } diff --git a/cmd/styles.css b/cmd/styles.css deleted file mode 100644 index 33803d81..00000000 --- a/cmd/styles.css +++ /dev/null @@ -1 +0,0 @@ -*,::after,::before{box-sizing:border-box;border-width:0;border-style:solid;border-color:#e5e7eb}::after,::before{--tw-content:''}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-feature-settings:inherit;font-variation-settings:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0;padding:0}legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::after,::before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.inset-y-0{top:0;bottom:0}.right-0{right:0}.mx-4{margin-left:1rem;margin-right:1rem}.mx-auto{margin-left:auto;margin-right:auto}.mt-10{margin-top:2.5rem}.block{display:block}.grid{display:grid}.hidden{display:none}.w-full{width:100%}.max-w-3xl{max-width:48rem}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.items-center{align-items:center}.gap-4{gap:1rem}.rounded-md{border-radius:.375rem}.bg-white{--tw-bg-opacity:1;background-color:rgb(255 255 255 / var(--tw-bg-opacity))}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-1\.5{padding-top:.375rem;padding-bottom:.375rem}.pl-2{padding-left:.5rem}.pr-2{padding-right:.5rem}.pr-3{padding-right:.75rem}.pt-10{padding-top:2.5rem}.text-center{text-align:center}.text-3xl{font-size:1.875rem;line-height:2.25rem}.text-sm{font-size:.875rem;line-height:1.25rem}.font-extrabold{font-weight:800}.leading-6{line-height:1.5rem}.tracking-tight{letter-spacing:-.025em}.text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.text-slate-500{--tw-text-opacity:1;color:rgb(100 116 139 / var(--tw-text-opacity))}.text-slate-600{--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}.text-slate-900{--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}.underline-offset-2{text-underline-offset:2px}.antialiased{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.shadow-sm{--tw-shadow:0 1px 2px 0 rgb(0 0 0 / 0.05);--tw-shadow-colored:0 1px 2px 0 var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.ring-1{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-slate-900\/10{--tw-ring-color:rgb(15 23 42 / 0.1)}.transition-colors{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}.duration-300{transition-duration:.3s}.hover\:text-blue-500:hover{--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}.hover\:text-slate-400:hover{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.hover\:underline:hover{text-decoration-line:underline}.hover\:ring-slate-300:hover{--tw-ring-opacity:1;--tw-ring-color:rgb(203 213 225 / var(--tw-ring-opacity))}@media (prefers-color-scheme:dark){.dark\:bg-slate-800{--tw-bg-opacity:1;background-color:rgb(30 41 59 / var(--tw-bg-opacity))}.dark\:bg-slate-900{--tw-bg-opacity:1;background-color:rgb(15 23 42 / var(--tw-bg-opacity))}.dark\:text-slate-200{--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}.dark\:text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.dark\:hover\:bg-slate-700:hover{--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}.hover\:dark\:text-slate-300:hover{--tw-text-opacity:1;color:rgb(203 213 225 / var(--tw-text-opacity))}}@media (min-width:640px){.sm\:text-4xl{font-size:2.25rem;line-height:2.5rem}} diff --git a/docker-compose.yaml b/docker-compose.yaml index 6325d095..f45e99ae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ services: ladder: image: ghcr.io/everywall/ladder:latest container_name: ladder - #build: . + build: . #restart: always #command: sh -c ./ladder environment: diff --git a/go.mod b/go.mod index 5d6ca5dd..15fb7f35 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,59 @@ -module ladder +module github.com/everywall/ladder go 1.21.1 require ( - github.com/PuerkitoBio/goquery v1.8.1 github.com/akamensky/argparse v1.4.0 - github.com/gofiber/fiber/v2 v2.50.0 + github.com/bogdanfinn/fhttp v0.5.24 + github.com/bogdanfinn/tls-client v1.6.1 + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c + github.com/gofiber/fiber/v2 v2.51.0 + github.com/markusmobius/go-trafilatura v1.5.1 github.com/stretchr/testify v1.8.4 + golang.org/x/net v0.19.0 + golang.org/x/term v0.15.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/abadojack/whatlanggo v1.0.1 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/bogdanfinn/utls v1.5.16 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elliotchance/pie/v2 v2.8.0 // indirect + github.com/forPelevin/gomoji v1.1.8 // indirect + github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 // indirect + github.com/gofiber/template v1.8.2 // indirect + github.com/gofiber/template/html/v2 v2.0.5 + github.com/gofiber/utils v1.1.0 // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/google/uuid v1.4.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/hablullah/go-hijri v1.0.2 // indirect + github.com/hablullah/go-juliandays v1.0.0 // indirect + github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/magefile/mage v1.15.0 // indirect + github.com/markusmobius/go-dateparser v1.2.1 // indirect + github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 // indirect + github.com/markusmobius/go-htmldate v1.2.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/rs/zerolog v1.31.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect + github.com/tetratelabs/wazero v1.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.50.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.14.0 + github.com/wasilibs/go-re2 v1.4.1 // indirect + github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + ) diff --git a/go.sum b/go.sum index f171d696..df0254a9 100644 --- a/go.sum +++ b/go.sum @@ -1,87 +1,158 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= +github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/bogdanfinn/fhttp v0.5.24 h1:OlyBKjvJp6a3TotN3wuj4mQHHRbfK7QUMrzCPOZGhRc= +github.com/bogdanfinn/fhttp v0.5.24/go.mod h1:brqi5woc5eSCVHdKYBV8aZLbO7HGqpwyDLeXW+fT18I= +github.com/bogdanfinn/tls-client v1.6.1 h1:GTIqQssFoIvLaDf4btoYRzDhUzudLqYD4axvfUCXl3I= +github.com/bogdanfinn/tls-client v1.6.1/go.mod h1:FtwQ3DndVZ0xAOO704v4iNAgbHOcEc5kPk9tjICTNQ0= +github.com/bogdanfinn/utls v1.5.16 h1:NhhWkegEcYETBMj9nvgO4lwvc6NcLH+znrXzO3gnw4M= +github.com/bogdanfinn/utls v1.5.16/go.mod h1:mHeRCi69cUiEyVBkKONB1cAbLjRcZnlJbGzttmiuK4o= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= -github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/elliotchance/pie/v2 v2.8.0 h1://QS43W8sEha8XV/fjngO5iMudN3XARJV5cpBayAcVY= +github.com/elliotchance/pie/v2 v2.8.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= +github.com/forPelevin/gomoji v1.1.8 h1:JElzDdt0TyiUlecy6PfITDL6eGvIaxqYH1V52zrd0qQ= +github.com/forPelevin/gomoji v1.1.8/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 h1:G6wSuUyCoLB9jrUokipsmFuRi8aJozt3phw/g9Sl4Xs= +github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ= +github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U= +github.com/gofiber/template v1.8.2 h1:PIv9s/7Uq6m+Fm2MDNd20pAFFKt5wWs7ZBd8iV9pWwk= +github.com/gofiber/template v1.8.2/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8= +github.com/gofiber/template/html/v2 v2.0.5 h1:BKLJ6Qr940NjntbGmpO3zVa4nFNGDCi/IfUiDB9OC20= +github.com/gofiber/template/html/v2 v2.0.5/go.mod h1:RCF14eLeQDCSUPp0IGc2wbSSDv6yt+V54XB/+Unz+LM= +github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= +github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k= +github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ= +github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0= +github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/markusmobius/go-dateparser v1.2.1 h1:mYRRdu3TzpAeE6fSl2Gn3arfxEtoTRvFOKlumlVsUtg= +github.com/markusmobius/go-dateparser v1.2.1/go.mod h1:5xYsZ1h7iB3sE1BSu8bkjYpbFST7EU1/AFxcyO3mgYg= +github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 h1:D83RvMz1lQ0ilKlJt6DWc65+Q77CXGRFmfihR0bfQvc= +github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3/go.mod h1:n1AYw0wiJDT3YXnIsElJPiDR63YGXT2yv3uq0CboGmU= +github.com/markusmobius/go-htmldate v1.2.2 h1:tp1IxhefCYpEoL9CM1LiU6l+2YayTpuTjkkdnik6hXE= +github.com/markusmobius/go-htmldate v1.2.2/go.mod h1:26VRz16sCosuiv42MNRW9iPBGnGLo+q/Z6TWitt8uzs= +github.com/markusmobius/go-trafilatura v1.5.1 h1:EXhZY2AVRyepUlLZHeuZUme3v7Ms9G8lDOLl4u+Jp5M= +github.com/markusmobius/go-trafilatura v1.5.1/go.mod h1:FhuBBPZ9ph4ufpGBKAkuq5oQwEhg0KKnIOUlv5h7EHg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= +github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0= +github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= -github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/wasilibs/go-re2 v1.4.1 h1:E5+9O1M8UoGeqLB2A9omeoaWImqpuYDs9cKwvTJq/Oo= +github.com/wasilibs/go-re2 v1.4.1/go.mod h1:ynB8eCwd9JsqUnsk8WlPDk6cEeme8BguZmnqOSURE4Y= +github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= +github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/api.go b/handlers/api.go index 190c8fe8..96ff82a3 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -2,7 +2,6 @@ package handlers import ( _ "embed" - "log" "github.com/gofiber/fiber/v2" ) @@ -12,48 +11,5 @@ import ( var version string func Api(c *fiber.Ctx) error { - // Get the url from the URL - urlQuery := c.Params("*") - - queries := c.Queries() - body, req, resp, err := fetchSite(urlQuery, queries) - if err != nil { - log.Println("ERROR:", err) - c.SendStatus(500) - return c.SendString(err.Error()) - } - - response := Response{ - Version: version, - Body: body, - } - - response.Request.Headers = make([]any, 0, len(req.Header)) - for k, v := range req.Header { - response.Request.Headers = append(response.Request.Headers, map[string]string{ - "key": k, - "value": v[0], - }) - } - - response.Response.Headers = make([]any, 0, len(resp.Header)) - for k, v := range resp.Header { - response.Response.Headers = append(response.Response.Headers, map[string]string{ - "key": k, - "value": v[0], - }) - } - - return c.JSON(response) -} - -type Response struct { - Version string `json:"version"` - Body string `json:"body"` - Request struct { - Headers []interface{} `json:"headers"` - } `json:"request"` - Response struct { - Headers []interface{} `json:"headers"` - } `json:"response"` + return nil } diff --git a/handlers/api.test.go b/handlers/api.test.go deleted file mode 100644 index 710a0b29..00000000 --- a/handlers/api.test.go +++ /dev/null @@ -1,44 +0,0 @@ -// BEGIN: 7d5e1f7c7d5e -package handlers - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/gofiber/fiber/v2" - "github.com/stretchr/testify/assert" -) - -func TestApi(t *testing.T) { - app := fiber.New() - app.Get("/api/*", Api) - - tests := []struct { - name string - url string - expectedStatus int - }{ - { - name: "valid url", - url: "https://www.google.com", - expectedStatus: http.StatusOK, - }, - { - name: "invalid url", - url: "invalid-url", - expectedStatus: http.StatusBadRequest, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/api/"+tt.url, nil) - resp, err := app.Test(req) - assert.NoError(t, err) - assert.Equal(t, tt.expectedStatus, resp.StatusCode) - }) - } -} - -// END: 7d5e1f7c7d5e diff --git a/handlers/api_content.go b/handlers/api_content.go new file mode 100644 index 00000000..306cbade --- /dev/null +++ b/handlers/api_content.go @@ -0,0 +1,45 @@ +package handlers + +import ( + rx "github.com/everywall/ladder/proxychain/requestmodifiers" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" + + "github.com/gofiber/fiber/v2" +) + +func NewAPIContentHandler(path string, opts *ProxyOptions) fiber.Handler { + // TODO: implement ruleset logic + /* + var rs ruleset.RuleSet + if opts.RulesetPath != "" { + r, err := ruleset.NewRuleset(opts.RulesetPath) + if err != nil { + panic(err) + } + rs = r + } + */ + + return func(c *fiber.Ctx) error { + proxychain := proxychain. + NewProxyChain(). + WithAPIPath(path). + SetDebugLogging(opts.Verbose). + SetRequestModifications( + rx.MasqueradeAsGoogleBot(), + rx.ForwardRequestHeaders(), + rx.SpoofReferrerFromGoogleSearch(), + ). + AddResponseModifications( + tx.DeleteIncomingCookies(), + tx.RewriteHTMLResourceURLs(), + tx.APIContent(), + ). + SetFiberCtx(c). + Execute() + + return proxychain + } +} diff --git a/handlers/api_modifiers.go b/handlers/api_modifiers.go new file mode 100644 index 00000000..cb30df50 --- /dev/null +++ b/handlers/api_modifiers.go @@ -0,0 +1,29 @@ +package handlers + +import ( + "encoding/json" + + "github.com/everywall/ladder/proxychain/responsemodifiers/api" + "github.com/gofiber/fiber/v2" +) + +func NewAPIModifersListHandler(opts *ProxyOptions) fiber.Handler { + payload := ModifiersAPIResponse{ + Success: true, + Result: AllMods, + } + body, err := json.MarshalIndent(payload, "", " ") + if err != nil { + panic(err) + } + + return func(c *fiber.Ctx) error { + c.Set("content-type", "application/json") + if err != nil { + c.SendStatus(500) + return c.SendStream(api.CreateAPIErrReader(err)) + } + + return c.Send(body) + } +} diff --git a/handlers/api_modifiers_codegen/api_modifiers_codegen.go b/handlers/api_modifiers_codegen/api_modifiers_codegen.go new file mode 100644 index 00000000..c3c3a443 --- /dev/null +++ b/handlers/api_modifiers_codegen/api_modifiers_codegen.go @@ -0,0 +1,196 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + "os/exec" + + //"io/fs" + "os" + "path/filepath" + "strings" + //"strings" +) + +func genModStruct(fn *ast.FuncDecl, githubEditLink string, filename string) string { + params := []string{} + for _, fd := range fn.Type.Params.List { + p := fmt.Sprintf(` {Name: "%s", Type: "%+v"},`, fd.Names[0], fd.Type) + params = append(params, p) + } + + block := fmt.Sprintf(`{ + Name: "%s", + Description: "%s", + CodeEditLink: "%s%s", + Params: []Param{ +%s + }, +},`, + fn.Name.String(), + strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(fn.Doc.Text()), "\n", " "), `"`, `\"`), + githubEditLink, filename, + strings.Join(params, "\n"), + ) + + return block +} + +func modCodeGen(dir string, githubEditLink string) (code string, err error) { + fset := token.NewFileSet() + + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + modStructs := []string{} + for _, file := range files { + if !shouldGenCodeFor(file) { + continue + } + + // Parse each Go file + node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments) + if err != nil { + return "", err + } + + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && fn.Recv == nil && fn.Name.IsExported() { + modStructs = append(modStructs, genModStruct(fn, githubEditLink, file.Name())) + } + return true + }) + + } + + code = strings.Join(modStructs, "\n") + return code, nil +} + +func shouldGenCodeFor(file fs.DirEntry) bool { + if file.IsDir() { + return false + } + if filepath.Ext(file.Name()) != ".go" { + return false + } + if strings.HasSuffix(file.Name(), "_test.go") { + return false + } + return true +} + +func getGitRemoteURL(remoteName string) (string, error) { + cmd := exec.Command("git", "remote", "get-url", remoteName) + output, err := cmd.Output() + if err != nil { + return "", err + } + url := strings.TrimSpace(string(output)) + + // Convert SSH format to HTTPS format + if strings.HasPrefix(url, "git@") { + url = strings.Replace(url, ":", "/", 1) + url = strings.Replace(url, "git@", "https://", 1) + url = strings.TrimSuffix(url, ".git") + } + + return url, nil +} + +func getCurrentGitBranch() (string, error) { + cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD") + output, err := cmd.Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)), nil +} + +func main() { + gitURL, err := getGitRemoteURL("origin") + if err != nil { + fmt.Println("Error getting Git remote URL:", err) + return + } + + branchName, err := getCurrentGitBranch() + if err != nil { + fmt.Println("Error getting current Git branch:", err) + return + } + + githubEditLink := fmt.Sprintf("%s/edit/%s/proxychain/requestmodifiers/", gitURL, branchName) + rqmCode, err := modCodeGen("../../proxychain/requestmodifiers/", githubEditLink) + if err != nil { + panic(err) + } + + githubEditLink = fmt.Sprintf("%s/edit/%s/proxychain/responsemodifiers/", gitURL, branchName) + rsmCode, err := modCodeGen("../../proxychain/responsemodifiers/", githubEditLink) + if err != nil { + panic(err) + } + + code := fmt.Sprintf(` +package handlers +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go +// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions. +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain/responsemodifiers/api" +) + +type ModifiersAPIResponse struct { + Success bool ||json:"success"|| + Error api.ErrorDetails ||json:"error"|| + Result Modifiers ||json:"result"|| +} + +type Modifiers struct { + RequestModifiers []Modifier ||json:"requestmodifiers"|| + ResponseModifiers []Modifier ||json:"responsemodifiers"|| +} + +type Modifier struct { + Name string ||json:"name"|| + Description string ||json:"description"|| + CodeEditLink string ||json:"code_edit_link"|| + Params []Param ||json:"params"|| +} + +type Param struct { + Name string ||json:"name"|| + Type string ||json:"type"|| +} + +var AllMods Modifiers = Modifiers{ + RequestModifiers: []Modifier{ +%s + }, + ResponseModifiers: []Modifier{ +%s + }, +} +`, rqmCode, rsmCode) + code = strings.ReplaceAll(code, "||", "`") + + //fmt.Println(code) + + fq, err := os.Create("../api_modifiers_structdef.gen.go") + if err != nil { + panic(err) + } + _, err = io.WriteString(fq, code) + if err != nil { + panic(err) + } +} diff --git a/handlers/api_modifiers_structdef.gen.go b/handlers/api_modifiers_structdef.gen.go new file mode 100644 index 00000000..bc4caffb --- /dev/null +++ b/handlers/api_modifiers_structdef.gen.go @@ -0,0 +1,567 @@ + +package handlers +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go +// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions. +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain/responsemodifiers/api" +) + +type ModifiersAPIResponse struct { + Success bool `json:"success"` + Error api.ErrorDetails `json:"error"` + Result Modifiers `json:"result"` +} + +type Modifiers struct { + RequestModifiers []Modifier `json:"requestmodifiers"` + ResponseModifiers []Modifier `json:"responsemodifiers"` +} + +type Modifier struct { + Name string `json:"name"` + Description string `json:"description"` + CodeEditLink string `json:"code_edit_link"` + Params []Param `json:"params"` +} + +type Param struct { + Name string `json:"name"` + Type string `json:"type"` +} + +var AllMods Modifiers = Modifiers{ + RequestModifiers: []Modifier{ +{ + Name: "AddCacheBusterQuery", + Description: "AddCacheBusterQuery modifies query params to add a random parameter key In order to get the upstream network stack to serve a fresh copy of the page.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/add_cache_buster_query.go", + Params: []Param{ + + }, +}, +{ + Name: "ForwardRequestHeaders", + Description: "ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/forward_request_headers.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsGoogleBot", + Description: "MasqueradeAsGoogleBot modifies user agent and x-forwarded for to appear to be a Google Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsBingBot", + Description: "MasqueradeAsBingBot modifies user agent and x-forwarded for to appear to be a Bing Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsWaybackMachineBot", + Description: "MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for to appear to be a archive.org (wayback machine) Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsFacebookBot", + Description: "MasqueradeAsFacebookBot modifies user agent and x-forwarded for to appear to be a Facebook Bot (link previews?)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsYandexBot", + Description: "MasqueradeAsYandexBot modifies user agent and x-forwarded for to appear to be a Yandex Spider Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsBaiduBot", + Description: "MasqueradeAsBaiduBot modifies user agent and x-forwarded for to appear to be a Baidu Spider Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsDuckDuckBot", + Description: "MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for to appear to be a DuckDuckGo Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsYahooBot", + Description: "MasqueradeAsYahooBot modifies user agent and x-forwarded for to appear to be a Yahoo Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "ModifyDomainWithRegex", + Description: "", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_domain_with_regex.go", + Params: []Param{ + {Name: "matchRegex", Type: "string"}, + {Name: "replacement", Type: "string"}, + }, +}, +{ + Name: "SetOutgoingCookie", + Description: "SetOutgoingCookie modifes a specific cookie name by modifying the request cookie headers going to the upstream server. If the cookie name does not already exist, it is created.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "SetOutgoingCookies", + Description: "SetOutgoingCookies modifies a client request's cookie header to a raw Cookie string, overwriting existing cookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "cookies", Type: "string"}, + }, +}, +{ + Name: "DeleteOutgoingCookie", + Description: "DeleteOutgoingCookie modifies the http request's cookies header to delete a specific request cookie going to the upstream server. If the cookie does not exist, it does not do anything.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + }, +}, +{ + Name: "DeleteOutgoingCookies", + Description: "DeleteOutgoingCookies removes the cookie header entirely, preventing any cookies from reaching the upstream server.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + + }, +}, +{ + Name: "DeleteOutgoingCookiesExcept", + Description: "DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client to the upstream proxy server. Cookies whose names are in the whitelist are not removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "whitelist", Type: "&{Ellipsis:12476 Elt:string}"}, + }, +}, +{ + Name: "ModifyPathWithRegex", + Description: "", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_path_with_regex.go", + Params: []Param{ + {Name: "matchRegex", Type: "string"}, + {Name: "replacement", Type: "string"}, + }, +}, +{ + Name: "ModifyQueryParams", + Description: "ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. If the query param key doesn't exist, it is created.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_query_params.go", + Params: []Param{ + {Name: "key", Type: "string"}, + {Name: "value", Type: "string"}, + }, +}, +{ + Name: "SetRequestHeader", + Description: "SetRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "DeleteRequestHeader", + Description: "DeleteRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go", + Params: []Param{ + {Name: "name", Type: "string"}, + }, +}, +{ + Name: "RequestArchiveIs", + Description: "RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_archive_is.go", + Params: []Param{ + + }, +}, +{ + Name: "RequestGoogleCache", + Description: "RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_google_cache.go", + Params: []Param{ + + }, +}, +{ + Name: "RequestWaybackMachine", + Description: "RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_wayback_machine.go", + Params: []Param{ + + }, +}, +{ + Name: "ResolveWithGoogleDoH", + Description: "ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL using Google's DNS over HTTPs service", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/resolve_with_google_doh.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofOrigin", + Description: "SpoofOrigin modifies the origin header if the upstream server returns a Vary header it means you might get a different response if you change this", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go", + Params: []Param{ + {Name: "url", Type: "string"}, + }, +}, +{ + Name: "HideOrigin", + Description: "HideOrigin modifies the origin header so that it is the original origin, not the proxy", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrer", + Description: "SpoofReferrer modifies the referrer header. It is useful if the page can be accessed from a search engine or social media site, but not by browsing the website itself. if url is \"\", then the referrer header is removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go", + Params: []Param{ + {Name: "url", Type: "string"}, + }, +}, +{ + Name: "HideReferrer", + Description: "HideReferrer modifies the referrer header so that it is the original referrer, not the proxy", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromBaiduSearch", + Description: "SpoofReferrerFromBaiduSearch modifies the referrer header pretending to be from a BaiduSearch", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromBingSearch", + Description: "SpoofReferrerFromBingSearch modifies the referrer header pretending to be from a bing search site", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromGoogleSearch", + Description: "SpoofReferrerFromGoogleSearch modifies the referrer header pretending to be from a google search site", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_google_search.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromLinkedInPost", + Description: "SpoofReferrerFromLinkedInPost modifies the referrer header pretending to be from a linkedin post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromNaverSearch", + Description: "SpoofReferrerFromNaverSearch modifies the referrer header pretending to be from a Naver search (popular in South Korea)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromPinterestPost", + Description: "SpoofReferrerFromPinterestPost modifies the referrer header pretending to be from a pinterest post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromQQPost", + Description: "SpoofReferrerFromQQPost modifies the referrer header pretending to be from a QQ post (popular social media in China)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromRedditPost", + Description: "SpoofReferrerFromRedditPost modifies the referrer header pretending to be from a reddit post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromTumblrPost", + Description: "SpoofReferrerFromTumblrPost modifies the referrer header pretending to be from a tumblr post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromTwitterPost", + Description: "SpoofReferrerFromTwitterPost modifies the referrer header pretending to be from a twitter post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromVkontaktePost", + Description: "SpoofReferrerFromVkontaktePost modifies the referrer header pretending to be from a vkontakte post (popular in Russia)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromWeiboPost", + Description: "SpoofReferrerFromWeiboPost modifies the referrer header pretending to be from a Weibo post (popular in China)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofUserAgent", + Description: "SpoofUserAgent modifies the user agent", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_user_agent.go", + Params: []Param{ + {Name: "ua", Type: "string"}, + }, +}, +{ + Name: "SpoofXForwardedFor", + Description: "SpoofXForwardedFor modifies the X-Forwarded-For header in some cases, a forward proxy may interpret this as the source IP", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_x_forwarded_for.go", + Params: []Param{ + {Name: "ip", Type: "string"}, + }, +}, + }, + ResponseModifiers: []Modifier{ +{ + Name: "APIContent", + Description: "APIContent creates an JSON representation of the article and returns it as an API response.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/api_content.go", + Params: []Param{ + + }, +}, +{ + Name: "BlockElementRemoval", + Description: "BlockElementRemoval prevents paywall javascript from removing a particular element by detecting the removal, then immediately reinserting it. This is useful when a page will return a \"fake\" 404, after flashing the content briefly. If the /outline/ API works, but the regular API doesn't, try this modifier.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/block_element_removal.go", + Params: []Param{ + {Name: "cssSelector", Type: "string"}, + }, +}, +{ + Name: "BlockThirdPartyScripts", + Description: "BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/block_third_party_scripts.go", + Params: []Param{ + + }, +}, +{ + Name: "BypassCORS", + Description: "BypassCORS modifies response headers to prevent the browser from enforcing any CORS restrictions. This should run at the end of the chain.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_cors.go", + Params: []Param{ + + }, +}, +{ + Name: "BypassContentSecurityPolicy", + Description: "BypassContentSecurityPolicy modifies response headers to prevent the browser from enforcing any CSP restrictions. This should run at the end of the chain.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go", + Params: []Param{ + + }, +}, +{ + Name: "SetContentSecurityPolicy", + Description: "SetContentSecurityPolicy modifies response headers to a specific CSP", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go", + Params: []Param{ + {Name: "csp", Type: "string"}, + }, +}, +{ + Name: "DeleteLocalStorageData", + Description: "DeleteLocalStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteSessionStorageData and DeleteIncomingCookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/delete_localstorage_data.go", + Params: []Param{ + + }, +}, +{ + Name: "DeleteSessionStorageData", + Description: "DeleteSessionStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteLocalStorageData and DeleteIncomingCookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/delete_sessionstorage_data.go", + Params: []Param{ + + }, +}, +{ + Name: "ForwardResponseHeaders", + Description: "ForwardResponseHeaders forwards the response headers from the upstream server to the client", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/forward_response_headers.go", + Params: []Param{ + + }, +}, +{ + Name: "GenerateReadableOutline", + Description: "GenerateReadableOutline creates an reader-friendly distilled representation of the article. This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/generate_readable_outline.go", + Params: []Param{ + + }, +}, +{ + Name: "InjectScriptBeforeDOMContentLoaded", + Description: "InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "InjectScriptAfterDOMContentLoaded", + Description: "InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "InjectScriptAfterDOMIdle", + Description: "InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "DeleteIncomingCookies", + Description: "DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server back down to the client.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "_", Type: "&{Ellipsis:18780 Elt:string}"}, + }, +}, +{ + Name: "DeleteIncomingCookiesExcept", + Description: "DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server to the client. Cookies whose names are in the whitelist are not removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "whitelist", Type: "&{Ellipsis:19325 Elt:string}"}, + }, +}, +{ + Name: "SetIncomingCookies", + Description: "SetIncomingCookies adds a raw cookie string being sent from the proxy server down to the client", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "cookies", Type: "string"}, + }, +}, +{ + Name: "SetIncomingCookie", + Description: "SetIncomingCookie modifies a specific cookie in the response from the proxy server to the client.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "ModifyIncomingScriptsWithRegex", + Description: "ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline + + ladder | error + + + +
+
+ + +
+ +
+
+ +
+

Error

+
+
+          
{{.Status}}
+
{{.Message}}
+
Type: {{.Type}}
+
Cause: {{.Cause}}
+
+
+ + +
+ + diff --git a/handlers/favicon.go b/handlers/favicon.go new file mode 100644 index 00000000..1c93d687 --- /dev/null +++ b/handlers/favicon.go @@ -0,0 +1,18 @@ +package handlers + +import ( + _ "embed" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/favicon" +) + +//go:embed favicon.ico +var faviconData string + +func Favicon() fiber.Handler { + return favicon.New(favicon.Config{ + Data: []byte(faviconData), + URL: "/favicon.ico", + }) +} diff --git a/cmd/favicon.ico b/handlers/favicon.ico similarity index 100% rename from cmd/favicon.ico rename to handlers/favicon.ico diff --git a/handlers/form.html b/handlers/form.html index 2174b731..41e7ed9c 100644 --- a/handlers/form.html +++ b/handlers/form.html @@ -1,79 +1,326 @@ - - - - + + + + ladder - - + + + - -
- - - - -
-

ladddddddder

-
-
-
- - + +
+
+ +
+ +
+ + + + +
+ +
+

+ ladddddddder +

+
+ +
+
+ + +
+ +
+ + +
+
+ + + +
- - - - - \ No newline at end of file + + diff --git a/handlers/outline.go b/handlers/outline.go new file mode 100644 index 00000000..6fec17b0 --- /dev/null +++ b/handlers/outline.go @@ -0,0 +1,32 @@ +package handlers + +import ( + rx "github.com/everywall/ladder/proxychain/requestmodifiers" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" + + "github.com/gofiber/fiber/v2" +) + +func NewOutlineHandler(path string, opts *ProxyOptions) fiber.Handler { + return func(c *fiber.Ctx) error { + return proxychain. + NewProxyChain(). + WithAPIPath(path). + SetDebugLogging(opts.Verbose). + SetRequestModifications( + rx.MasqueradeAsGoogleBot(), + rx.ForwardRequestHeaders(), + rx.SpoofReferrerFromGoogleSearch(), + ). + AddResponseModifications( + tx.SetResponseHeader("content-type", "text/html"), + tx.DeleteIncomingCookies(), + tx.RewriteHTMLResourceURLs(), + tx.GenerateReadableOutline(), // <-- this response modification does the outline rendering + ). + SetFiberCtx(c). + Execute() + } +} diff --git a/handlers/playground-script.js b/handlers/playground-script.js new file mode 100644 index 00000000..be4c4bbc --- /dev/null +++ b/handlers/playground-script.js @@ -0,0 +1,491 @@ +const modifierContainer = document.getElementById("modifierContainer"); +const modalContainer = document.getElementById("modalContainer"); +const modalBody = document.getElementById("modal-body"); +const modalContent = document.getElementById("modal-content"); +const modalSubmitButton = document.getElementById("modal-submit"); +const modalClose = document.getElementById("modal-close"); + +let hasFetched = false; +let payload = { + requestmodifications: [], + responsemodifications: [], +}; +let ninjaData = []; + +initialize(); + +// Rerun handleThemeChange() so style is applied to Ninja Keys +handleThemeChange(); + +// Add event listener to the iframe so it closes dropdown when clicked +closeDropdownOnClickWithinIframe(); + +async function initialize() { + if (!hasFetched) { + try { + await fetchPayload(); + hasFetched = true; + } catch (error) { + console.error("Fetch error:", error); + } + } +} + +function closeDropdownOnClickWithinIframe() { + const iframe = document.getElementById("resultIframe"); + iframe.contentWindow.document.addEventListener( + "click", + () => { + if ( + !document.getElementById("dropdown_panel").classList.contains("hidden") + ) { + toggleDropdown(); + } + }, + true + ); +} + +async function fetchPayload() { + try { + const response = await fetch("/api/modifiers"); + const data = await response.json(); + + Object.entries(data.result.requestmodifiers ?? []).forEach(([_, value]) => { + addModifierToNinjaData( + value.name, + value.description, + value.params, + "requestmodifications" + ); + }); + + Object.entries(data.result.responsemodifiers ?? []).forEach( + ([_, value]) => { + addModifierToNinjaData( + value.name, + value.description, + value.params, + "responsemodifications" + ); + } + ); + + return data; + } catch (error) { + console.error("Fetch error:", error); + throw error; + } +} + +async function submitForm() { + if (!document.getElementById("inputForm").checkValidity()) { + return; + } + + try { + const response = await fetch("/playground/" + inputField.value, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error("Request failed"); + } + + const result = await response.text(); + updateResultIframe(result); + } catch (error) { + console.error(error); + } +} + +function updateResultIframe(result) { + const resultIframe = parent.document.getElementById("resultIframe"); + resultIframe.contentDocument.open(); + resultIframe.contentDocument.write(result); + closeDropdownOnClickWithinIframe(); + resultIframe.contentDocument.close(); +} + +document.getElementById("inputForm").addEventListener("submit", function (e) { + e.preventDefault(); + submitForm(); +}); + +if (navigator.userAgent.includes("Mac")) { + document.getElementById("ninjaKey").textContent = "⌘"; +} else { + document.getElementById("ninjaKey").textContent = "Ctrl"; +} + +function downloadYaml() { + function jsonToYaml(payload) { + const jsonObject = { + rules: [ + { + domains: [hostname], + responsemodifications: [], + requestmodifications: [], + ...payload, + }, + ], + }; + return jsyaml.dump(jsonObject); + } + + if (!document.getElementById("inputForm").checkValidity()) { + alert("Please enter a valid URL."); + return; + } + const hostname = new URL(inputField.value).hostname; + const ruleHostname = hostname.replace(/^www\./, "").replace(/\./g, "-"); + const yamlString = jsonToYaml(payload); + const blob = new Blob([yamlString], { type: "text/yaml;charset=utf-8" }); + const href = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = href; + link.download = `${ruleHostname}.yaml`; + link.click(); + URL.revokeObjectURL(href); +} + +function getValues(type, id, description, params) { + const focusTrap = trap(modalBody); + let values = []; + let existingValues = []; + const inputs = []; + const inputEventListeners = []; + + function closeModal() { + focusTrap.destroy(); + modalBody.removeEventListener("keydown", handleKeyboardEvents); + modalContainer.removeEventListener("click", handleClickOutside); + modalSubmitButton.removeEventListener("click", closeModal); + modalClose.removeEventListener("click", closeModal); + inputEventListeners.forEach((listener, index) => { + if (listener !== undefined && inputs[index] !== undefined) + inputs[index].removeEventListener("input", listener); + }); + modalContent.classList.remove("relative", "h-[220px]"); + inputEventListeners.length = 0; + inputs.length = 0; + modalContainer.classList.add("hidden"); + modalContent.innerHTML = ""; + } + + function handleClickOutside(e) { + if (modalBody !== null && !modalBody.contains(e.target)) { + closeModal(); + } + } + + function handleKeyboardEvents(e) { + if (e.key === "Escape") { + closeModal(); + } + if (e.key === "Enter") { + if (e.target.tagName.toLowerCase() === "textarea") { + return; + } else { + modalSubmitButton.click(); + } + } + } + + document.getElementById("modal-title").textContent = id; + document.getElementById("modal-description").textContent = description; + + existingValues = + payload[type].find( + (modifier) => modifier.name === id && modifier.params !== undefined + )?.params ?? []; + + params.map((param, i) => { + function textareaEventListener(e) { + const codeElement = document.querySelector("code"); + let text = e.target.value; + + if (text[text.length - 1] == "\n") { + text += " "; + } + + codeElement.innerHTML = text + .replace(new RegExp("&", "g"), "&") + .replace(new RegExp("<", "g"), "<"); + + Prism.highlightElement(codeElement); + values[i] = text; + syncScroll(e.target); + } + + function textareaKeyEventListener(e) { + if (e.key === "Tab" && !e.shiftKey) { + e.preventDefault(); + e.stopPropagation(); + let text = e.target.value; + const start = e.target.selectionStart; + const end = e.target.selectionEnd; + e.target.value = text.substring(0, start) + "\t" + text.substring(end); + e.target.setSelectionRange(start + 1, start + 1); + e.target.dispatchEvent(new Event("input")); + } + syncScroll(e.target); + } + + function syncScroll(el) { + const codeElement = document.querySelector("code"); + codeElement.scrollTop = el.scrollTop; + codeElement.scrollLeft = el.scrollLeft; + } + + function inputEventListener(e) { + if (e.key !== "Enter") { + values[i] = e.target.value; + } + } + + const label = document.createElement("label"); + label.textContent = param.name; + label.setAttribute("for", `input-${i}`); + let input; + if (param.name === "js") { + input = document.createElement("textarea"); + input.type = "textarea"; + input.setAttribute("spellcheck", "false"); + input.placeholder = "Enter your JavaScript injection code ..."; + input.classList.add( + "h-[200px]", + "w-full", + "font-mono", + "whitespace-nowrap", + "font-semibold", + "absolute", + "text-base", + "leading-6", + "rounded-md", + "ring-1", + "ring-slate-900/10", + "shadow-sm", + "z-10", + "p-4", + "m-0", + "my-2", + "bg-transparent", + "dark:bg-transparent", + "text-transparent", + "overflow-auto", + "resize-none", + "caret-white", + "hover:ring-slate-300", + "hyphens-none" + ); + input.style.tabSize = "4"; + } else { + input = document.createElement("input"); + input.type = "text"; + input.classList.add( + "w-full", + "text-sm", + "leading-6", + "text-slate-400", + "rounded-md", + "ring-1", + "ring-slate-900/10", + "shadow-sm", + "py-1.5", + "pl-2", + "pr-3", + "mt-0", + "hover:ring-slate-300", + "dark:bg-slate-800", + "dark:highlight-white/5", + "overflow-auto" + ); + } + input.id = `input-${i}`; + input.value = existingValues[i] ?? ""; + modalContent.appendChild(label); + modalContent.appendChild(input); + if (input.type === "textarea") { + label.classList.add("sr-only", "hidden"); + preElement = document.createElement("pre"); + codeElement = document.createElement("code"); + preElement.setAttribute("aria-hidden", "true"); + preElement.classList.add( + "bg-[#2d2d2d]", + "dark:bg-[#2d2d2d]", + "h-[200px]", + "w-full", + "rounded-md", + "ring-1", + "ring-slate-900/10", + "shadow-sm", + "p-0", + "m-0", + "my-2", + "font-mono", + "text-base", + "leading-6", + "overflow-auto", + "whitespace-nowrap", + "font-semibold", + "absolute", + "z-0", + "hyphens-none" + ); + modalContent.classList.add("relative", "h-[220px]"); + preElement.setAttribute("tabindex", "-1"); + codeElement.classList.add( + "language-javascript", + "absolute", + "w-full", + "font-mono", + "text-base", + "leading-6", + "z-0", + "p-4", + "-mx-4", + "-my-4", + "h-full", + "whitespace-nowrap", + "overflow-auto", + "hyphens-none" + ); + preElement.appendChild(codeElement); + modalContent.appendChild(preElement); + codeElement.innerHTML = input.value + .replace(new RegExp("&", "g"), "&") + .replace(new RegExp("<", "g"), "<"); + Prism.highlightElement(codeElement); + input.addEventListener("input", textareaEventListener); + input.addEventListener("keydown", textareaKeyEventListener); + input.addEventListener("scroll", () => syncScroll(input)); + inputEventListeners.push( + textareaEventListener, + textareaKeyEventListener, + syncScroll + ); + } else { + input.addEventListener("input", inputEventListener); + inputEventListeners.push(inputEventListener); + } + inputs.push(input); + }); + + modalContainer.classList.remove("hidden"); + document.getElementById("input-0").focus(); + + return new Promise((resolve) => { + modalBody.addEventListener("keydown", handleKeyboardEvents); + modalContainer.addEventListener("click", handleClickOutside); + modalClose.addEventListener("click", () => { + closeModal(); + }); + modalSubmitButton.addEventListener("click", (e) => { + inputs.forEach((input, i) => { + values[i] = input.value; + }); + resolve(values); + closeModal(); + }); + }); +} + +function toggleModifier(type, id, params = []) { + function pillClickHandler(pill) { + toggleModifier(pill.getAttribute("type"), pill.id); + pill.removeEventListener("click", () => pillClickHandler(pill)); + pill.remove(); + } + + function createPill(type, id) { + const pill = document.createElement("span"); + pill.classList.add( + "inline-flex", + "items-center", + "rounded-md", + "bg-slate-100", + "dark:bg-slate-800", + "px-2", + "py-1", + "h-4", + "text-xs", + "font-medium", + "border", + "border-slate-400", + "dark:border-slate-700", + "cursor-pointer" + ); + pill.id = id; + pill.setAttribute("type", type); + pill.textContent = id; + modifierContainer.appendChild(pill); + pill.addEventListener("click", () => pillClickHandler(pill)); + } + + if ( + params === undefined && + payload[type].some((modifier) => modifier.name === id) + ) { + payload[type] = payload[type].filter((modifier) => modifier.name !== id); + const existingPill = document.getElementById(id); + if (existingPill !== null) { + existingPill.removeEventListener("click", () => pillClickHandler(pill)); + existingPill.remove(); + } + } else { + const existingModifier = payload[type].find( + (modifier) => modifier.name === id + ); + if (existingModifier) { + existingModifier.params = params; + } else { + payload[type].push({ name: id, params: params }); + } + const existingPill = document.getElementById(id); + if (existingPill === null) { + createPill(type, id); + } + } + + submitForm(); +} + +function addModifierToNinjaData(id, description, params, type) { + const section = + type === "requestmodifications" + ? "Request Modifiers" + : "Response Modifiers"; + const modifier = { + id: id, + title: id, + section: section, + + handler: () => { + if (Object.keys(params).length === 0) { + toggleModifier(type, id); + } else { + if (params[0].name === "_") { + toggleModifier(type, id, (params = [""])); + } else { + getValues(type, id, description, params).then((values) => { + if (Object.keys(values).length === 0) return; + toggleModifier(type, id, values); + }); + } + } + }, + }; + + ninjaData.push(modifier); +} + +const ninja = document.querySelector("ninja-keys"); +ninja.data = ninjaData; +document.getElementById("btnNinja").addEventListener("click", () => { + ninja.open(); +}); diff --git a/handlers/playground.go b/handlers/playground.go new file mode 100644 index 00000000..4217663d --- /dev/null +++ b/handlers/playground.go @@ -0,0 +1,42 @@ +package handlers + +import ( + _ "embed" + + "github.com/everywall/ladder/proxychain" + ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset" + + "net/http" + + "github.com/gofiber/fiber/v2" +) + +//go:embed playground.html +var playgroundHtml string + +func PlaygroundHandler(path string, opts *ProxyOptions) fiber.Handler { + return func(c *fiber.Ctx) error { + if c.Method() == fiber.MethodGet { + c.Set("Content-Type", "text/html") + + return c.SendString(playgroundHtml) + } else if c.Method() == fiber.MethodPost { + var modificationData ruleset_v2.Rule + if err := c.BodyParser(&modificationData); err != nil { + return err + } + + c.Method(fiber.MethodGet) + + return proxychain. + NewProxyChain(). + SetFiberCtx(c). + WithAPIPath(path). + AddOnceRequestModifications(modificationData.RequestModifications...). + AddOnceResponseModifications(modificationData.ResponseModifications...). + Execute() + } + + return c.Status(http.StatusMethodNotAllowed).SendString("Method not allowed") + } +} diff --git a/handlers/playground.html b/handlers/playground.html new file mode 100644 index 00000000..f8de68e1 --- /dev/null +++ b/handlers/playground.html @@ -0,0 +1,451 @@ + + + + + + ladder | playground + + + + + + + + + + + + + +
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ +
+
+
+ + + + + + +
+ + diff --git a/handlers/proxy.go b/handlers/proxy.go index bedfbde7..2db88274 100644 --- a/handlers/proxy.go +++ b/handlers/proxy.go @@ -1,330 +1,76 @@ package handlers import ( - "fmt" - "io" - "log" - "net/http" - "net/url" - "os" - "regexp" - "strings" + rx "github.com/everywall/ladder/proxychain/requestmodifiers" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" - "ladder/pkg/ruleset" + "github.com/everywall/ladder/proxychain" + ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset" - "github.com/PuerkitoBio/goquery" "github.com/gofiber/fiber/v2" ) -var ( - UserAgent = getenv("USER_AGENT", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") - ForwardedFor = getenv("X_FORWARDED_FOR", "66.249.66.1") - rulesSet = ruleset.NewRulesetFromEnv() - allowedDomains = []string{} -) - -func init() { - allowedDomains = strings.Split(os.Getenv("ALLOWED_DOMAINS"), ",") - if os.Getenv("ALLOWED_DOMAINS_RULESET") == "true" { - allowedDomains = append(allowedDomains, rulesSet.Domains()...) - } -} - -// extracts a URL from the request ctx. If the URL in the request -// is a relative path, it reconstructs the full URL using the referer header. -func extractUrl(c *fiber.Ctx) (string, error) { - // try to extract url-encoded - reqUrl, err := url.QueryUnescape(c.Params("*")) - if err != nil { - // fallback - reqUrl = c.Params("*") - } - - // Extract the actual path from req ctx - urlQuery, err := url.Parse(reqUrl) - if err != nil { - return "", fmt.Errorf("error parsing request URL '%s': %v", reqUrl, err) - } - - isRelativePath := urlQuery.Scheme == "" - - // eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg - if isRelativePath { - // Parse the referer URL from the request header. - refererUrl, err := url.Parse(c.Get("referer")) - if err != nil { - return "", fmt.Errorf("error parsing referer URL from req: '%s': %v", reqUrl, err) - } - - // Extract the real url from referer path - realUrl, err := url.Parse(strings.TrimPrefix(refererUrl.Path, "/")) - if err != nil { - return "", fmt.Errorf("error parsing real URL from referer '%s': %v", refererUrl.Path, err) - } - - // reconstruct the full URL using the referer's scheme, host, and the relative path / queries - fullUrl := &url.URL{ - Scheme: realUrl.Scheme, - Host: realUrl.Host, - Path: urlQuery.Path, - RawQuery: urlQuery.RawQuery, - } - - if os.Getenv("LOG_URLS") == "true" { - log.Printf("modified relative URL: '%s' -> '%s'", reqUrl, fullUrl.String()) - } - return fullUrl.String(), nil - - } - - // default behavior: - // eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg - return urlQuery.String(), nil -} - -func ProxySite(rulesetPath string) fiber.Handler { - if rulesetPath != "" { - rs, err := ruleset.NewRuleset(rulesetPath) - if err != nil { - panic(err) - } - rulesSet = rs - } - - return func(c *fiber.Ctx) error { - // Get the url from the URL - url, err := extractUrl(c) - if err != nil { - log.Println("ERROR In URL extraction:", err) - } - - queries := c.Queries() - body, _, resp, err := fetchSite(url, queries) - if err != nil { - log.Println("ERROR:", err) - c.SendStatus(fiber.StatusInternalServerError) - return c.SendString(err.Error()) - } - - c.Cookie(&fiber.Cookie{}) - c.Set("Content-Type", resp.Header.Get("Content-Type")) - c.Set("Content-Security-Policy", resp.Header.Get("Content-Security-Policy")) - - return c.SendString(body) - } -} - -func modifyURL(uri string, rule ruleset.Rule) (string, error) { - newUrl, err := url.Parse(uri) - if err != nil { - return "", err - } - - for _, urlMod := range rule.URLMods.Domain { - re := regexp.MustCompile(urlMod.Match) - newUrl.Host = re.ReplaceAllString(newUrl.Host, urlMod.Replace) - } - - for _, urlMod := range rule.URLMods.Path { - re := regexp.MustCompile(urlMod.Match) - newUrl.Path = re.ReplaceAllString(newUrl.Path, urlMod.Replace) - } - - v := newUrl.Query() - for _, query := range rule.URLMods.Query { - if query.Value == "" { - v.Del(query.Key) - continue - } - v.Set(query.Key, query.Value) - } - newUrl.RawQuery = v.Encode() - - if rule.GoogleCache { - newUrl, err = url.Parse("https://webcache.googleusercontent.com/search?q=cache:" + newUrl.String()) - if err != nil { - return "", err - } - } - - return newUrl.String(), nil -} - -func fetchSite(urlpath string, queries map[string]string) (string, *http.Request, *http.Response, error) { - urlQuery := "?" - if len(queries) > 0 { - for k, v := range queries { - urlQuery += k + "=" + v + "&" - } - } - urlQuery = strings.TrimSuffix(urlQuery, "&") - urlQuery = strings.TrimSuffix(urlQuery, "?") - - u, err := url.Parse(urlpath) - if err != nil { - return "", nil, nil, err - } - - if len(allowedDomains) > 0 && !StringInSlice(u.Host, allowedDomains) { - return "", nil, nil, fmt.Errorf("domain not allowed. %s not in %s", u.Host, allowedDomains) - } - - if os.Getenv("LOG_URLS") == "true" { - log.Println(u.String() + urlQuery) - } - - // Modify the URI according to ruleset - rule := fetchRule(u.Host, u.Path) - url, err := modifyURL(u.String()+urlQuery, rule) - if err != nil { - return "", nil, nil, err - } - - // Fetch the site - client := &http.Client{} - req, _ := http.NewRequest("GET", url, nil) - - if rule.Headers.UserAgent != "" { - req.Header.Set("User-Agent", rule.Headers.UserAgent) - } else { - req.Header.Set("User-Agent", UserAgent) - } - - if rule.Headers.XForwardedFor != "" { - if rule.Headers.XForwardedFor != "none" { - req.Header.Set("X-Forwarded-For", rule.Headers.XForwardedFor) - } - } else { - req.Header.Set("X-Forwarded-For", ForwardedFor) - } - - if rule.Headers.Referer != "" { - if rule.Headers.Referer != "none" { - req.Header.Set("Referer", rule.Headers.Referer) - } - } else { - req.Header.Set("Referer", u.String()) - } - - if rule.Headers.Cookie != "" { - req.Header.Set("Cookie", rule.Headers.Cookie) - } - - resp, err := client.Do(req) - if err != nil { - return "", nil, nil, err - } - defer resp.Body.Close() - - bodyB, err := io.ReadAll(resp.Body) - if err != nil { - return "", nil, nil, err - } - - if rule.Headers.CSP != "" { - // log.Println(rule.Headers.CSP) - resp.Header.Set("Content-Security-Policy", rule.Headers.CSP) - } - - // log.Print("rule", rule) TODO: Add a debug mode to print the rule - body := rewriteHtml(bodyB, u, rule) - return body, req, resp, nil -} - -func rewriteHtml(bodyB []byte, u *url.URL, rule ruleset.Rule) string { - // Rewrite the HTML - body := string(bodyB) - - // images - imagePattern := `]*\s+)?src="(/)([^"]*)"` - re := regexp.MustCompile(imagePattern) - body = re.ReplaceAllString(body, fmt.Sprintf(`]*\s+)?src="(/)([^"]*)"` - reScript := regexp.MustCompile(scriptPattern) - body = reScript.ReplaceAllString(body, fmt.Sprintf(` - About Us -
- - - `) - u := &url.URL{Host: "example.com"} - - expected := ` - - - Test Page - - - - - About Us -
- - - ` - - actual := rewriteHtml(bodyB, u, ruleset.Rule{}) - assert.Equal(t, expected, actual) -} - -// END: 6f8b3f5d5d5d diff --git a/handlers/raw.go b/handlers/raw.go deleted file mode 100644 index 7baa0e52..00000000 --- a/handlers/raw.go +++ /dev/null @@ -1,21 +0,0 @@ -package handlers - -import ( - "log" - - "github.com/gofiber/fiber/v2" -) - -func Raw(c *fiber.Ctx) error { - // Get the url from the URL - urlQuery := c.Params("*") - - queries := c.Queries() - body, _, _, err := fetchSite(urlQuery, queries) - if err != nil { - log.Println("ERROR:", err) - c.SendStatus(500) - return c.SendString(err.Error()) - } - return c.SendString(body) -} diff --git a/handlers/raw.test.go b/handlers/raw.test.go deleted file mode 100644 index f8867d08..00000000 --- a/handlers/raw.test.go +++ /dev/null @@ -1,60 +0,0 @@ -// BEGIN: 7f8d9e6d4b5c -package handlers - -import ( - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/gofiber/fiber/v2" -) - -func TestRaw(t *testing.T) { - app := fiber.New() - app.Get("/raw/*", Raw) - - testCases := []struct { - name string - url string - expected string - }{ - { - name: "valid url", - url: "https://www.google.com", - expected: "", - }, - { - name: "invalid url", - url: "invalid-url", - expected: "parse invalid-url: invalid URI for request", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/raw/"+tc.url, nil) - resp, err := app.Test(req) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Errorf("expected status OK; got %v", resp.Status) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if !strings.Contains(string(body), tc.expected) { - t.Errorf("expected body to contain %q; got %q", tc.expected, string(body)) - } - }) - } -} - -// END: 7f8d9e6d4b5c diff --git a/handlers/ruleset.go b/handlers/ruleset.go deleted file mode 100644 index 33dcf298..00000000 --- a/handlers/ruleset.go +++ /dev/null @@ -1,23 +0,0 @@ -package handlers - -import ( - "os" - - "github.com/gofiber/fiber/v2" - "gopkg.in/yaml.v3" -) - -func Ruleset(c *fiber.Ctx) error { - if os.Getenv("EXPOSE_RULESET") == "false" { - c.SendStatus(fiber.StatusForbidden) - return c.SendString("Rules Disabled") - } - - body, err := yaml.Marshal(rulesSet) - if err != nil { - c.SendStatus(fiber.StatusInternalServerError) - return c.SendString(err.Error()) - } - - return c.SendString(string(body)) -} diff --git a/handlers/script.go b/handlers/script.go new file mode 100644 index 00000000..4e003270 --- /dev/null +++ b/handlers/script.go @@ -0,0 +1,33 @@ +package handlers + +import ( + "embed" + + "github.com/gofiber/fiber/v2" +) + +//go:embed script.js +var scriptData embed.FS + +//go:embed playground-script.js +var playgroundScriptData embed.FS + +func Script(c *fiber.Ctx) error { + if c.Path() == "/script.js" { + scriptData, err := scriptData.ReadFile("script.js") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + c.Set("Content-Type", "text/javascript") + return c.Send(scriptData) + } + if c.Path() == "/playground-script.js" { + playgroundScriptData, err := playgroundScriptData.ReadFile("playground-script.js") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + c.Set("Content-Type", "text/javascript") + return c.Send(playgroundScriptData) + } + return nil +} diff --git a/handlers/script.js b/handlers/script.js new file mode 100644 index 00000000..9f5ef70c --- /dev/null +++ b/handlers/script.js @@ -0,0 +1,330 @@ +const labels = document.querySelectorAll("label"); +const inputs = document.querySelectorAll('input[type="radio"]'); +const mainElement = document.querySelector("main"); +const inputField = document.getElementById("inputField"); +const clearButton = document.getElementById("clearButton"); + +window.addEventListener("DOMContentLoaded", handleDOMContentLoaded); + +function handleDOMContentLoaded() { + handleFontChange(); + handleFontSizeChange(); + inputs.forEach((input) => { + const storedValue = localStorage.getItem(input.name); + if (storedValue === input.value) { + input.checked = true; + } + }); + window.removeEventListener("DOMContentLoaded", handleDOMContentLoaded); +} + +function clearInput() { + inputField.value = ""; + clearButton.style.display = "none"; + inputField.focus(); +} + +if (inputField !== null && clearButton !== null) { + inputField.addEventListener("input", () => { + const clearButton = document.getElementById("clearButton"); + if (clearButton !== null) { + if (inputField.value.trim().length > 0) { + clearButton.style.display = "block"; + } else { + clearButton.style.display = "none"; + } + } + }); + + inputField.addEventListener("keydown", (event) => { + if (event.code === "Escape") { + clearInput(); + } + }); + + clearButton.addEventListener("click", () => { + clearInput(); + }); +} + +function focusable_children(node) { + const nodes = Array.from( + node.querySelectorAll( + 'a[href], button, input, textarea, select, details, [tabindex]:not([tabindex="-1"])' + ) + ).filter((s) => s.offsetParent !== null); + const index = nodes.indexOf(document.activeElement); + const update = (d) => { + let i = index + d; + i += nodes.length; + i %= nodes.length; + nodes[i].focus(); + }; + return { + next: (selector) => { + const reordered = [ + ...nodes.slice(index + 1), + ...nodes.slice(0, index + 1), + ]; + for (let i = 0; i < reordered.length; i += 1) { + if (!selector || reordered[i].matches(selector)) { + reordered[i].focus(); + return; + } + } + }, + prev: (selector) => { + const reordered = [ + ...nodes.slice(index + 1), + ...nodes.slice(0, index + 1), + ]; + for (let i = reordered.length - 2; i >= 0; i -= 1) { + if (!selector || reordered[i].matches(selector)) { + reordered[i].focus(); + return; + } + } + }, + update, + }; +} + +function trap(node) { + const handle_keydown = (e) => { + if (e.key === "Tab") { + e.preventDefault(); + const group = focusable_children(node); + if (e.shiftKey) { + group.prev(); + } else { + group.next(); + } + } + }; + node.addEventListener("keydown", handle_keydown); + return { + destroy: () => { + node.removeEventListener("keydown", handle_keydown); + }, + }; +} + +const toggleDropdown = () => { + const dropdown = document.getElementById("dropdown"); + const dropdown_button = dropdown.querySelector("button"); + const dropdown_panel = document.getElementById("dropdown_panel"); + const focusTrap = trap(dropdown); + + const closeDropdown = () => { + dropdown_panel.classList.add("hidden"); + dropdown_button.setAttribute("aria-expanded", "false"); + focusTrap.destroy(); + dropdown.removeEventListener("keydown", handleEscapeKey); + document.removeEventListener("click", handleClickOutside); + inputs.forEach((input) => { + input.removeEventListener("change", handleInputChange); + }); + labels.forEach((label) => { + label.removeEventListener("click", handleLabelSelection); + }); + }; + + const handleClickOutside = (e) => { + if (dropdown !== null && !dropdown.contains(e.target)) { + closeDropdown(); + } + }; + + const handleEscapeKey = (e) => { + if (e.key === "Escape") { + dropdown_panel.classList.add("hidden"); + closeDropdown(); + } + }; + + const handleInputChange = (e) => { + if (e.target.checked) { + localStorage.setItem(e.target.name, e.target.value); + switch (e.target.name) { + case "theme": { + handleThemeChange(); + break; + } + case "font": { + handleFontChange(); + break; + } + case "fontsize": { + handleFontSizeChange(); + break; + } + default: { + console.error("Unknown event"); + break; + } + } + } + }; + + const handleLabelSelection = (e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + const input = document.getElementById(e.target.getAttribute("for")); + input.checked = true; + input.dispatchEvent(new Event("change", { bubbles: true })); + } + }; + + if (dropdown_panel.classList.contains("hidden")) { + dropdown_panel.classList.remove("hidden"); + dropdown_button.setAttribute("aria-expanded", "true"); + dropdown.addEventListener("keydown", handleEscapeKey); + inputs.forEach((input) => { + input.addEventListener("change", handleInputChange); + }); + labels.forEach((label) => { + label.addEventListener("keydown", handleLabelSelection); + }); + document.addEventListener("click", handleClickOutside); + } else { + closeDropdown(); + } +}; + +const handleFontChange = () => { + if (mainElement === null) { + return; + } + let font = localStorage.getItem("font"); + if (font === null) { + localStorage.setItem("font", "sans-serif"); + font = "sans-serif"; + } + if (font === "serif") { + mainElement.classList.add("font-serif"); + mainElement.classList.remove("font-sans"); + } else { + mainElement.classList.add("font-sans"); + mainElement.classList.remove("font-serif"); + } +}; + +const changeFontSize = (node, classes) => { + const sizes = [ + "text-xs", + "text-sm", + "text-base", + "text-lg", + "text-xl", + "text-2xl", + "text-3xl", + "text-4xl", + "text-5xl", + "lg:text-4xl", + "lg:text-5xl", + "lg:text-6xl", + ]; + const currentClasses = sizes.filter((size) => node.classList.contains(size)); + node.classList.remove(...currentClasses); + node.classList.add(...classes); +}; + +const handleFontSizeChange = () => { + if (mainElement === null) { + return; + } + let fontSize = localStorage.getItem("fontsize"); + if (fontSize === null) { + localStorage.setItem("fontsize", "text-base"); + fontSize = "text-base"; + } + if (fontSize === "text-sm") { + changeFontSize(document.querySelector("body"), ["text-sm"]); + } else if (fontSize === "text-lg") { + changeFontSize(document.querySelector("body"), ["text-lg"]); + } else { + changeFontSize(document.querySelector("body"), ["text-base"]); + } + + const nodes = document.querySelectorAll( + "h1, h2, h3, h4, h5, h6, code, pre, kbd, table" + ); + if (fontSize === "text-sm") { + changeFontSize(mainElement, ["text-sm"]); + } else if (fontSize === "text-lg") { + changeFontSize(mainElement, ["text-lg"]); + } else { + changeFontSize(mainElement, ["text-base"]); + } + nodes.forEach((node) => { + let classes = ""; + switch (node.tagName) { + case "H1": { + if (fontSize === "text-sm") { + classes = ["text-3xl", "lg:text-4xl"]; + } else if (fontSize === "text-lg") { + classes = ["text-5xl", "lg:text-6xl"]; + } else { + classes = ["text-4xl", "lg:text-5xl"]; + } + break; + } + case "H2": { + if (fontSize === "text-sm") { + classes = ["text-2xl"]; + } else if (fontSize === "text-lg") { + classes = ["text-4xl"]; + } else { + classes = ["text-3xl"]; + } + break; + } + case "H3": { + if (fontSize === "text-sm") { + classes = ["text-xl"]; + } else if (fontSize === "text-lg") { + classes = ["text-3xl"]; + } else { + classes = ["text-2xl"]; + } + break; + } + case "H4": + case "H5": + case "H6": { + if (fontSize === "text-sm") { + classes = ["text-lg"]; + } else if (fontSize === "text-lg") { + classes = ["text-2xl"]; + } else { + classes = ["text-xl"]; + } + break; + } + case "CODE": + case "PRE": + case "KBD": + case "TABLE": { + if (fontSize === "text-sm") { + classes = ["text-xs"]; + } else if (fontSize === "text-lg") { + classes = ["text-base"]; + } else { + classes = ["text-sm"]; + } + break; + } + default: { + if (fontSize === "text-sm") { + classes = ["text-sm"]; + } else if (fontSize === "text-lg") { + classes = ["text-lg"]; + } else { + classes = ["text-base"]; + } + break; + } + } + changeFontSize(node, classes); + }); +}; diff --git a/handlers/styles.css b/handlers/styles.css new file mode 100644 index 00000000..aa8db5d7 --- /dev/null +++ b/handlers/styles.css @@ -0,0 +1 @@ +*,::after,::before{box-sizing:border-box;border-width:0;border-style:solid;border-color:#e5e7eb}::after,::before{--tw-content:''}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-feature-settings:normal;font-variation-settings:normal;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-feature-settings:inherit;font-variation-settings:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0;padding:0}legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}a{--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity));text-decoration-line:underline;text-underline-offset:4px;transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:.3s}a:hover{--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}:is(.dark a){--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}:is(.dark a:hover){--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}h1{scroll-margin:5rem;font-size:2.25rem;line-height:2.5rem;font-weight:800;letter-spacing:-.025em;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark h1){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}@media (min-width:1024px){h1{font-size:3rem;line-height:1}}h2{scroll-margin:5rem;border-bottom-width:1px;padding-bottom:.5rem;font-size:1.875rem;line-height:2.25rem;font-weight:600;letter-spacing:-.025em;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}h2:first-child{margin-top:0}:is(.dark h2){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}h3{scroll-margin:5rem;font-size:1.5rem;line-height:2rem;font-weight:600;letter-spacing:-.025em;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark h3){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}h4,h5,h6{scroll-margin:5rem;font-size:1.25rem;line-height:1.75rem;font-weight:600;letter-spacing:-.025em;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark h4),:is(.dark h5),:is(.dark h6){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}p{line-height:1.75rem;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark p){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}p:not(:first-child){margin-top:1.5rem}code,kbd,pre{position:relative;white-space:break-spaces;overflow-wrap:break-word;border-radius:.25rem;--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity));padding-top:.2rem;padding-bottom:.2rem;font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:.875rem;line-height:1.25rem;font-weight:600}:is(.dark code),:is(.dark kbd),:is(.dark pre){--tw-bg-opacity:1;background-color:rgb(30 41 59 / var(--tw-bg-opacity))}blockquote{margin-top:1.5rem;border-left-width:2px;padding-left:1.5rem;font-style:italic}ul{margin-top:1.5rem;margin-bottom:1.5rem;margin-left:1.5rem;list-style-type:disc;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark ul){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}ul>li{margin-top:.5rem}ol{margin-top:1.5rem;margin-bottom:1.5rem;margin-left:1.5rem;list-style-type:decimal;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark ol){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}ol>li{margin-top:.5rem}dl{margin-top:1.5rem;margin-bottom:1.5rem;font-weight:700;--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark dl){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}dl>dd{margin-left:1.5rem;font-weight:400}dl>dt{margin-top:.75rem}li{--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark li){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}table{width:100%;caption-side:bottom;font-size:.875rem;line-height:1.25rem}thead tr{border-bottom-width:1px}tbody tr:last-child{border-width:0}tfoot{border-top-width:1px;--tw-border-opacity:1;border-color:rgb(148 163 184 / var(--tw-border-opacity));background-color:rgb(51 65 85 / .5);font-weight:500}:is(.dark tfoot){--tw-border-opacity:1;border-color:rgb(51 65 85 / var(--tw-border-opacity));background-color:rgb(226 232 240 / .5)}tfoot:last-child>tr{border-bottom-width:0}tr{border-bottom-width:1px;--tw-border-opacity:1;border-color:rgb(148 163 184 / var(--tw-border-opacity));transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}tr:hover{background-color:rgb(226 232 240 / .5)}tr[data-state=selected]{--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}:is(.dark tr){--tw-border-opacity:1;border-color:rgb(51 65 85 / var(--tw-border-opacity))}:is(.dark tr:hover){background-color:rgb(51 65 85 / .5)}:is(.dark tr[data-state=selected]){--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}th{height:3rem;padding-left:1rem;padding-right:1rem;text-align:left;vertical-align:middle;font-weight:500;--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}:is(.dark th){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}th:has([role=checkbox]){padding-right:0}td{padding:1rem;vertical-align:middle}td:has([role=checkbox]){padding-right:0}caption{margin-top:1rem;font-size:.875rem;line-height:1.25rem;--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}:is(.dark caption){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}img{margin-left:auto;margin-right:auto;height:auto;width:auto;max-width:100%;border-radius:.375rem;-o-object-fit:cover;object-fit:cover;--tw-shadow:0 4px 6px -1px rgb(0 0 0 / 0.1),0 2px 4px -2px rgb(0 0 0 / 0.1);--tw-shadow-colored:0 4px 6px -1px var(--tw-shadow-color),0 2px 4px -2px var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}:is(.dark img){--tw-shadow-color:#334155;--tw-shadow:var(--tw-shadow-colored)}figcaption{margin-top:.5rem;font-size:.875rem;line-height:1.25rem;--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}:is(.dark figcaption){--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}hr{height:1px;width:100%;flex-shrink:0;--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}:is(.dark hr){--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}*,::after,::before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border-width:0}.fixed{position:fixed}.absolute{position:absolute}.relative{position:relative}.inset-0{inset:0px}.inset-x-0{left:0;right:0}.inset-y-0{top:0;bottom:0}.right-0{right:0}.top-0{top:0}.z-0{z-index:0}.z-10{z-index:10}.m-0{margin:0}.-mx-4{margin-left:-1rem;margin-right:-1rem}.-my-4{margin-top:-1rem;margin-bottom:-1rem}.mx-1{margin-left:.25rem;margin-right:.25rem}.mx-4{margin-left:1rem;margin-right:1rem}.mx-auto{margin-left:auto;margin-right:auto}.my-2{margin-top:.5rem;margin-bottom:.5rem}.my-4{margin-top:1rem;margin-bottom:1rem}.my-auto{margin-top:auto;margin-bottom:auto}.-ml-2{margin-left:-.5rem}.-mt-12{margin-top:-3rem}.ml-1{margin-left:.25rem}.ml-2{margin-left:.5rem}.mr-1{margin-right:.25rem}.mr-2{margin-right:.5rem}.mt-0{margin-top:0}.mt-2{margin-top:.5rem}.mt-3{margin-top:.75rem}.mt-48{margin-top:12rem}.mt-5{margin-top:1.25rem}.block{display:block}.flex{display:flex}.inline-flex{display:inline-flex}.table{display:table}.grid{display:grid}.hidden{display:none}.h-10{height:2.5rem}.h-11{height:2.75rem}.h-12{height:3rem}.h-4{height:1rem}.h-48{height:12rem}.h-5{height:1.25rem}.h-6{height:1.5rem}.h-8{height:2rem}.h-\[1px\]{height:1px}.h-\[200px\]{height:200px}.h-\[220px\]{height:220px}.h-\[250px\]{height:250px}.h-\[calc\(100vh-14\.5rem\)\]{height:calc(100vh - 14.5rem)}.h-auto{height:auto}.h-full{height:100%}.h-screen{height:100vh}.max-h-14{max-height:3.5rem}.min-h-full{min-height:100%}.w-10{width:2.5rem}.w-4{width:1rem}.w-5{width:1.25rem}.w-52{width:13rem}.w-6{width:1.5rem}.w-auto{width:auto}.w-full{width:100%}.w-screen{width:100vw}.max-w-3xl{max-width:48rem}.max-w-full{max-width:100%}.flex-1{flex:1 1 0%}.shrink-0{flex-shrink:0}.grow{flex-grow:1}.transform{transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.cursor-default{cursor:default}.cursor-pointer{cursor:pointer}.select-none{-webkit-user-select:none;-moz-user-select:none;user-select:none}.resize-none{resize:none}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.flex-wrap-reverse{flex-wrap:wrap-reverse}.place-items-center{place-items:center}.items-end{align-items:flex-end}.items-center{align-items:center}.justify-center{justify-content:center}.justify-between{justify-content:space-between}.justify-items-center{justify-items:center}.gap-1{gap:.25rem}.gap-2{gap:.5rem}.gap-4{gap:1rem}.gap-6{gap:1.5rem}.gap-x-10{-moz-column-gap:2.5rem;column-gap:2.5rem}.gap-y-4{row-gap:1rem}.space-y-3>:not([hidden])~:not([hidden]){--tw-space-y-reverse:0;margin-top:calc(.75rem * calc(1 - var(--tw-space-y-reverse)));margin-bottom:calc(.75rem * var(--tw-space-y-reverse))}.place-self-end{place-self:end}.overflow-auto{overflow:auto}.overflow-hidden{overflow:hidden}.overflow-y-auto{overflow-y:auto}.overflow-x-hidden{overflow-x:hidden}.hyphens-none{-webkit-hyphens:none;hyphens:none}.whitespace-normal{white-space:normal}.whitespace-nowrap{white-space:nowrap}.break-all{word-break:break-all}.rounded-full{border-radius:9999px}.rounded-lg{border-radius:.5rem}.rounded-md{border-radius:.375rem}.rounded-xl{border-radius:.75rem}.border{border-width:1px}.border-b-2{border-bottom-width:2px}.border-t{border-top-width:1px}.border-t-2{border-top-width:2px}.border-gray-300{--tw-border-opacity:1;border-color:rgb(209 213 219 / var(--tw-border-opacity))}.border-slate-400{--tw-border-opacity:1;border-color:rgb(148 163 184 / var(--tw-border-opacity))}.border-slate-600{--tw-border-opacity:1;border-color:rgb(71 85 105 / var(--tw-border-opacity))}.bg-\[\#2d2d2d\]{--tw-bg-opacity:1;background-color:rgb(45 45 45 / var(--tw-bg-opacity))}.bg-slate-100{--tw-bg-opacity:1;background-color:rgb(241 245 249 / var(--tw-bg-opacity))}.bg-slate-200{--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}.bg-slate-400{--tw-bg-opacity:1;background-color:rgb(148 163 184 / var(--tw-bg-opacity))}.bg-slate-500\/50{background-color:rgb(100 116 139 / .5)}.bg-slate-800{--tw-bg-opacity:1;background-color:rgb(30 41 59 / var(--tw-bg-opacity))}.bg-transparent{background-color:transparent}.bg-white{--tw-bg-opacity:1;background-color:rgb(255 255 255 / var(--tw-bg-opacity))}.object-cover{-o-object-fit:cover;object-fit:cover}.p-0{padding:0}.p-2{padding:.5rem}.p-4{padding:1rem}.px-2{padding-left:.5rem;padding-right:.5rem}.px-4{padding-left:1rem;padding-right:1rem}.px-6{padding-left:1.5rem;padding-right:1.5rem}.px-8{padding-left:2rem;padding-right:2rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-1\.5{padding-top:.375rem;padding-bottom:.375rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-2\.5{padding-top:.625rem;padding-bottom:.625rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.py-4{padding-top:1rem;padding-bottom:1rem}.pb-4{padding-bottom:1rem}.pl-0{padding-left:0}.pl-2{padding-left:.5rem}.pr-2{padding-right:.5rem}.pr-3{padding-right:.75rem}.pt-10{padding-top:2.5rem}.pt-2{padding-top:.5rem}.pt-4{padding-top:1rem}.pt-5{padding-top:1.25rem}.text-left{text-align:left}.text-center{text-align:center}.align-middle{vertical-align:middle}.font-mono{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}.font-sans{font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji"}.font-serif{font-family:ui-serif,Georgia,Cambria,"Times New Roman",Times,serif}.text-2xl{font-size:1.5rem;line-height:2rem}.text-3xl{font-size:1.875rem;line-height:2.25rem}.text-4xl{font-size:2.25rem;line-height:2.5rem}.text-5xl{font-size:3rem;line-height:1}.text-base{font-size:1rem;line-height:1.5rem}.text-lg{font-size:1.125rem;line-height:1.75rem}.text-sm{font-size:.875rem;line-height:1.25rem}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-xs{font-size:.75rem;line-height:1rem}.font-extrabold{font-weight:800}.font-medium{font-weight:500}.font-semibold{font-weight:600}.leading-6{line-height:1.5rem}.leading-8{line-height:2rem}.leading-none{line-height:1}.tracking-tight{letter-spacing:-.025em}.text-\[\#7AA7D1\]{--tw-text-opacity:1;color:rgb(122 167 209 / var(--tw-text-opacity))}.text-red-500{--tw-text-opacity:1;color:rgb(239 68 68 / var(--tw-text-opacity))}.text-slate-200{--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}.text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.text-slate-600{--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}.text-slate-900{--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}.text-transparent{color:transparent}.underline{text-decoration-line:underline}.no-underline{text-decoration-line:none}.underline-offset-2{text-underline-offset:2px}.antialiased{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.caret-white{caret-color:#fff}.shadow-md{--tw-shadow:0 4px 6px -1px rgb(0 0 0 / 0.1),0 2px 4px -2px rgb(0 0 0 / 0.1);--tw-shadow-colored:0 4px 6px -1px var(--tw-shadow-color),0 2px 4px -2px var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.shadow-sm{--tw-shadow:0 1px 2px 0 rgb(0 0 0 / 0.05);--tw-shadow-colored:0 1px 2px 0 var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.shadow-xl{--tw-shadow:0 20px 25px -5px rgb(0 0 0 / 0.1),0 8px 10px -6px rgb(0 0 0 / 0.1);--tw-shadow-colored:0 20px 25px -5px var(--tw-shadow-color),0 8px 10px -6px var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.outline{outline-style:solid}.ring-1{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-slate-900\/10{--tw-ring-color:rgb(15 23 42 / 0.1)}.ring-offset-2{--tw-ring-offset-width:2px}.ring-offset-white{--tw-ring-offset-color:#fff}.filter{filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.transition-all{transition-property:all;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}.transition-colors{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}.transition-opacity{transition-property:opacity;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}.duration-300{transition-duration:.3s}.first-of-type\:rounded-t-md:first-of-type{border-top-left-radius:.375rem;border-top-right-radius:.375rem}.last-of-type\:rounded-b-md:last-of-type{border-bottom-right-radius:.375rem;border-bottom-left-radius:.375rem}.hover\:bg-slate-200:hover{--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}.hover\:bg-slate-200\/90:hover{background-color:rgb(226 232 240 / .9)}.hover\:bg-slate-800\/90:hover{background-color:rgb(30 41 59 / .9)}.hover\:text-blue-500:hover{--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}.hover\:text-slate-400:hover{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.hover\:text-slate-500:hover{--tw-text-opacity:1;color:rgb(100 116 139 / var(--tw-text-opacity))}.hover\:text-slate-900:hover{--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}.hover\:underline:hover{text-decoration-line:underline}.hover\:ring-slate-300:hover{--tw-ring-opacity:1;--tw-ring-color:rgb(203 213 225 / var(--tw-ring-opacity))}.hover\:drop-shadow-\[0_0px_10px_rgba\(122\2c 167\2c 209\2c \.3\)\]:hover{--tw-drop-shadow:drop-shadow(0 0px 10px rgba(122,167,209,.3));filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.hover\:drop-shadow-\[0_0px_4px_rgba\(122\2c 167\2c 209\2c \.3\)\]:hover{--tw-drop-shadow:drop-shadow(0 0px 4px rgba(122,167,209,.3));filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}.focus\:border-\[\#7AA7D1\]:focus{--tw-border-opacity:1;border-color:rgb(122 167 209 / var(--tw-border-opacity))}.focus\:outline-none:focus{outline:2px solid transparent;outline-offset:2px}.focus\:ring:focus{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(3px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.focus-visible\:outline-none:focus-visible{outline:2px solid transparent;outline-offset:2px}.focus-visible\:ring-2:focus-visible{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.focus-visible\:ring-offset-2:focus-visible{--tw-ring-offset-width:2px}.disabled\:pointer-events-none:disabled{pointer-events:none}.disabled\:opacity-50:disabled{opacity:.5}.peer:checked~.peer-checked\:bg-slate-200{--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}:is(.dark .dark\:border-gray-700){--tw-border-opacity:1;border-color:rgb(55 65 81 / var(--tw-border-opacity))}:is(.dark .dark\:border-slate-400){--tw-border-opacity:1;border-color:rgb(148 163 184 / var(--tw-border-opacity))}:is(.dark .dark\:border-slate-700){--tw-border-opacity:1;border-color:rgb(51 65 85 / var(--tw-border-opacity))}:is(.dark .dark\:bg-\[\#2d2d2d\]){--tw-bg-opacity:1;background-color:rgb(45 45 45 / var(--tw-bg-opacity))}:is(.dark .dark\:bg-slate-200){--tw-bg-opacity:1;background-color:rgb(226 232 240 / var(--tw-bg-opacity))}:is(.dark .dark\:bg-slate-700){--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}:is(.dark .dark\:bg-slate-800){--tw-bg-opacity:1;background-color:rgb(30 41 59 / var(--tw-bg-opacity))}:is(.dark .dark\:bg-slate-900){--tw-bg-opacity:1;background-color:rgb(15 23 42 / var(--tw-bg-opacity))}:is(.dark .dark\:bg-transparent){background-color:transparent}:is(.dark .dark\:text-red-400){--tw-text-opacity:1;color:rgb(248 113 113 / var(--tw-text-opacity))}:is(.dark .dark\:text-slate-200){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}:is(.dark .dark\:text-slate-400){--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}:is(.dark .dark\:text-slate-900){--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}:is(.dark .dark\:shadow-slate-700){--tw-shadow-color:#334155;--tw-shadow:var(--tw-shadow-colored)}:is(.dark .dark\:ring-offset-slate-900){--tw-ring-offset-color:#0f172a}:is(.dark .dark\:hover\:bg-slate-200\/90:hover){background-color:rgb(226 232 240 / .9)}:is(.dark .dark\:hover\:bg-slate-500:hover){--tw-bg-opacity:1;background-color:rgb(100 116 139 / var(--tw-bg-opacity))}:is(.dark .dark\:hover\:bg-slate-700:hover){--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}:is(.dark .dark\:hover\:bg-slate-800\/90:hover){background-color:rgb(30 41 59 / .9)}:is(.dark .dark\:hover\:text-blue-500:hover){--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}:is(.dark .dark\:hover\:text-slate-200:hover){--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}:is(.dark .hover\:dark\:text-slate-200):hover{--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}:is(.dark .hover\:dark\:text-slate-300):hover{--tw-text-opacity:1;color:rgb(203 213 225 / var(--tw-text-opacity))}:is(.dark .peer:checked ~ .dark\:peer-checked\:bg-slate-700){--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}@media (min-width:640px){.sm\:my-8{margin-top:2rem;margin-bottom:2rem}.sm\:ml-4{margin-left:1rem}.sm\:mt-0{margin-top:0}.sm\:flex{display:flex}.sm\:w-full{width:100%}.sm\:max-w-lg{max-width:32rem}.sm\:flex-row-reverse{flex-direction:row-reverse}.sm\:p-0{padding:0}.sm\:p-6{padding:1.5rem}.sm\:px-6{padding-left:1.5rem;padding-right:1.5rem}.sm\:pb-4{padding-bottom:1rem}.sm\:text-4xl{font-size:2.25rem;line-height:2.5rem}}@media (min-width:1024px){.lg\:mx-auto{margin-left:auto;margin-right:auto}.lg\:text-4xl{font-size:2.25rem;line-height:2.5rem}.lg\:text-5xl{font-size:3rem;line-height:1}.lg\:text-6xl{font-size:3.75rem;line-height:1}} diff --git a/handlers/styles.go b/handlers/styles.go new file mode 100644 index 00000000..cd326dd0 --- /dev/null +++ b/handlers/styles.go @@ -0,0 +1,23 @@ +package handlers + +import ( + "embed" + + "github.com/gofiber/fiber/v2" +) + +//go:embed styles.css +var cssData embed.FS + +func Styles(c *fiber.Ctx) error { + + cssData, err := cssData.ReadFile("styles.css") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") + } + + c.Set("Content-Type", "text/css") + + return c.Send(cssData) + +} diff --git a/internal/cli/art.go b/internal/cli/art.go new file mode 100644 index 00000000..017bd2cb --- /dev/null +++ b/internal/cli/art.go @@ -0,0 +1,82 @@ +package cli + +import ( + "fmt" + "os" + "strings" + + "golang.org/x/term" +) + +var art string = ` + _____╬═╬____________________________________________ + |_|__╬═╬___|___|___|___| EVERYWALL |___|___|___|___| + |___|╬═╬|___▄▄▌ ▄▄▄· ·▄▄▄▄ ·▄▄▄▄ ▄▄▄ .▄▄▄ __|_| + |_|__╬═╬___|██• ▐█ ▀█ ██▪ ██ ██▪ ██ ▀▄.▀·▀▄ █·|___| + |___|╬═╬|___██▪ ▄█▀▀█ ▐█· ▐█▌▐█· ▐█▌▐▀▀▪▄▐▀▀▄ __|_| + |_|__╬═╬___|▐█▌▐▌▐█ ▪▐▌██. ██ ██. ██ ▐█▄▄▌▐█•█▌|___| + |___|╬═╬|___.▀▀▀ ▀ ▀ ▀▀▀▀▀• ▀▀▀▀▀• ▀▀▀ .▀ ▀__|_| + |_|__╬═╬___|___|___|_ VERSION %-7s__|___|___|___| + |___|╬═╬|____|___|___|___|___|___|___|___|___|___|_| + ╬═╬ + ╬═╬ %s + ` + +func StartupMessage(version string, port string, ruleset string) string { + isTerm := term.IsTerminal(int(os.Stdout.Fd())) + version = strings.Trim(version, " ") + version = strings.Trim(version, "\n") + + var link string + if isTerm { + link = createHyperlink("http://localhost:" + port) + } else { + link = "http://localhost:" + port + } + + buf := fmt.Sprintf(art, version, link) + if isTerm { + buf = blinkChars(buf, '.', '•', '·', '▪') + } + + if ruleset == "" { + buf += "\n [!] no ruleset specified.\n [!] for better performance, use a ruleset using --ruleset\n" + } + if isTerm { + buf = colorizeNonASCII(buf) + } + return buf +} + +func createHyperlink(url string) string { + return fmt.Sprintf("\033[4m%s\033[0m", url) +} + +func colorizeNonASCII(input string) string { + result := "" + for _, r := range input { + if r > 127 { + // If the character is non-ASCII, color it blue + result += fmt.Sprintf("\033[34m%c\033[0m", r) + } else { + // ASCII characters remain unchanged + result += string(r) + } + } + return result +} + +func blinkChars(input string, chars ...rune) string { + result := "" +MAIN: + for _, x := range input { + for _, y := range chars { + if x == y { + result += fmt.Sprintf("\033[5m%s\033[0m", string(x)) + continue MAIN + } + } + result += fmt.Sprintf("%s", string(x)) + } + return result +} diff --git a/handlers/cli/cli.go b/internal/cli/ruleset_merge.go similarity index 58% rename from handlers/cli/cli.go rename to internal/cli/ruleset_merge.go index ca7b7e00..40999c68 100644 --- a/handlers/cli/cli.go +++ b/internal/cli/ruleset_merge.go @@ -5,9 +5,7 @@ import ( "io" "os" - "ladder/pkg/ruleset" - - "golang.org/x/term" + ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset" ) // HandleRulesetMerge merges a set of ruleset files, specified by the rulesetPath or RULESET env variable, into either YAML or Gzip format. @@ -21,7 +19,7 @@ import ( // // Returns: // - An error if the ruleset loading or merging process fails, otherwise nil. -func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, output *os.File) error { +func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, output *os.File) error { if !mergeRulesets { return nil } @@ -35,55 +33,15 @@ func HandleRulesetMerge(rulesetPath string, mergeRulesets bool, useGzip bool, ou os.Exit(1) } - rs, err := ruleset.NewRuleset(rulesetPath) + rs, err := ruleset_v2.NewRuleset(rulesetPath) if err != nil { fmt.Println(err) os.Exit(1) } - if useGzip { - return gzipMerge(rs, output) - } - return yamlMerge(rs, output) } -// gzipMerge takes a RuleSet and an optional output file path pointer. It compresses the RuleSet into Gzip format. -// If the output file path is provided, the compressed data is written to this file. Otherwise, it prints a warning -// and outputs the binary data to stdout -// -// Parameters: -// - rs: The ruleset.RuleSet to be compressed. -// - output: The output for the gzip data. If nil, stdout will be used. -// -// Returns: -// - An error if compression or file writing fails, otherwise nil. -func gzipMerge(rs ruleset.RuleSet, output io.Writer) error { - gzip, err := rs.GzipYaml() - if err != nil { - return err - } - - if output != nil { - _, err = io.Copy(output, gzip) - if err != nil { - return err - } - } - - if term.IsTerminal(int(os.Stdout.Fd())) { - println("warning: binary output can mess up your terminal. Use '--merge-rulesets-output ' or pipe it to a file.") - os.Exit(1) - } - - _, err = io.Copy(os.Stdout, gzip) - if err != nil { - return err - } - - return nil -} - // yamlMerge takes a RuleSet and an optional output file path pointer. It converts the RuleSet into YAML format. // If the output file path is provided, the YAML data is written to this file. If not, the YAML data is printed to stdout. // @@ -93,8 +51,8 @@ func gzipMerge(rs ruleset.RuleSet, output io.Writer) error { // // Returns: // - An error if YAML conversion or file writing fails, otherwise nil. -func yamlMerge(rs ruleset.RuleSet, output io.Writer) error { - yaml, err := rs.Yaml() +func yamlMerge(rs ruleset_v2.Ruleset, output io.Writer) error { + yaml, err := rs.YAML() if err != nil { return err } diff --git a/package.json b/package.json index ebe7d425..1ba2e1ed 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { - "scripts": { - "build": "pnpx tailwindcss -i ./styles/input.css -o ./styles/output.css --build && pnpx minify ./styles/output.css > ./cmd/styles.css" - }, - "devDependencies": { - "minify": "^10.5.2", - "tailwindcss": "^3.3.5" - } -} \ No newline at end of file + "scripts": { + "build": "pnpx tailwindcss -i ./styles/input.css -o ./styles/output.css --build --minify", + "watch": "pnpx tailwindcss -i ./styles/input.css -o ./styles/output.css --build --minify --watch" + }, + "devDependencies": { + "tailwindcss": "^3.3.6" + } +} diff --git a/pkg/ruleset/ruleset.go b/pkg/ruleset/ruleset.go deleted file mode 100644 index 7f3079f1..00000000 --- a/pkg/ruleset/ruleset.go +++ /dev/null @@ -1,310 +0,0 @@ -package ruleset - -import ( - "compress/gzip" - "errors" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - - "gopkg.in/yaml.v3" -) - -type Regex struct { - Match string `yaml:"match"` - Replace string `yaml:"replace"` -} -type KV struct { - Key string `yaml:"key"` - Value string `yaml:"value"` -} - -type RuleSet []Rule - -type Rule struct { - Domain string `yaml:"domain,omitempty"` - Domains []string `yaml:"domains,omitempty"` - Paths []string `yaml:"paths,omitempty"` - Headers struct { - UserAgent string `yaml:"user-agent,omitempty"` - XForwardedFor string `yaml:"x-forwarded-for,omitempty"` - Referer string `yaml:"referer,omitempty"` - Cookie string `yaml:"cookie,omitempty"` - CSP string `yaml:"content-security-policy,omitempty"` - } `yaml:"headers,omitempty"` - GoogleCache bool `yaml:"googleCache,omitempty"` - RegexRules []Regex `yaml:"regexRules,omitempty"` - - URLMods struct { - Domain []Regex `yaml:"domain,omitempty"` - Path []Regex `yaml:"path,omitempty"` - Query []KV `yaml:"query,omitempty"` - } `yaml:"urlMods,omitempty"` - - Injections []struct { - Position string `yaml:"position,omitempty"` - Append string `yaml:"append,omitempty"` - Prepend string `yaml:"prepend,omitempty"` - Replace string `yaml:"replace,omitempty"` - } `yaml:"injections,omitempty"` -} - -var remoteRegex = regexp.MustCompile(`^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)`) - -// NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable. -// It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set. -// If the RULESET is set but the rules cannot be loaded, it panics. -func NewRulesetFromEnv() RuleSet { - rulesPath, ok := os.LookupEnv("RULESET") - if !ok { - log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.") - return RuleSet{} - } - - ruleSet, err := NewRuleset(rulesPath) - if err != nil { - log.Println(err) - } - - return ruleSet -} - -// NewRuleset loads a RuleSet from a given string of rule paths, separated by semicolons. -// It supports loading rules from both local file paths and remote URLs. -// Returns a RuleSet and an error if any issues occur during loading. -func NewRuleset(rulePaths string) (RuleSet, error) { - var ruleSet RuleSet - - var errs []error - - rp := strings.Split(rulePaths, ";") - for _, rule := range rp { - var err error - - rulePath := strings.Trim(rule, " ") - isRemote := remoteRegex.MatchString(rulePath) - - if isRemote { - err = ruleSet.loadRulesFromRemoteFile(rulePath) - } else { - err = ruleSet.loadRulesFromLocalDir(rulePath) - } - - if err != nil { - e := fmt.Errorf("WARN: failed to load ruleset from '%s'", rulePath) - errs = append(errs, errors.Join(e, err)) - - continue - } - } - - if len(errs) != 0 { - e := fmt.Errorf("WARN: failed to load %d rulesets", len(rp)) - errs = append(errs, e) - - // panic if the user specified a local ruleset, but it wasn't found on disk - // don't fail silently - for _, err := range errs { - if errors.Is(os.ErrNotExist, err) { - e := fmt.Errorf("PANIC: ruleset '%s' not found", err) - panic(errors.Join(e, err)) - } - } - - // else, bubble up any errors, such as syntax or remote host issues - return ruleSet, errors.Join(errs...) - } - - ruleSet.PrintStats() - - return ruleSet, nil -} - -// ================== RULESET loading logic =================================== - -// loadRulesFromLocalDir loads rules from a local directory specified by the path. -// It walks through the directory, loading rules from YAML files. -// Returns an error if the directory cannot be accessed -// If there is an issue loading any file, it will be skipped -func (rs *RuleSet) loadRulesFromLocalDir(path string) error { - _, err := os.Stat(path) - if err != nil { - return err - } - - yamlRegex := regexp.MustCompile(`.*\.ya?ml`) - - err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if isYaml := yamlRegex.MatchString(path); !isYaml { - return nil - } - - err = rs.loadRulesFromLocalFile(path) - if err != nil { - log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err) - return nil - } - - log.Printf("INFO: loaded ruleset %s\n", path) - - return nil - }) - - if err != nil { - return err - } - - return nil -} - -// loadRulesFromLocalFile loads rules from a local YAML file specified by the path. -// Returns an error if the file cannot be read or if there's a syntax error in the YAML. -func (rs *RuleSet) loadRulesFromLocalFile(path string) error { - yamlFile, err := os.ReadFile(path) - if err != nil { - e := fmt.Errorf("failed to read rules from local file: '%s'", path) - return errors.Join(e, err) - } - - var r RuleSet - err = yaml.Unmarshal(yamlFile, &r) - - if err != nil { - e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s'", path) - ee := errors.Join(e, err) - - if _, ok := os.LookupEnv("DEBUG"); ok { - debugPrintRule(string(yamlFile), ee) - } - - return ee - } - - *rs = append(*rs, r...) - - return nil -} - -// loadRulesFromRemoteFile loads rules from a remote URL. -// It supports plain and gzip compressed content. -// Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML. -func (rs *RuleSet) loadRulesFromRemoteFile(rulesURL string) error { - var r RuleSet - - resp, err := http.Get(rulesURL) - if err != nil { - e := fmt.Errorf("failed to load rules from remote url '%s'", rulesURL) - return errors.Join(e, err) - } - - defer resp.Body.Close() - - if resp.StatusCode >= 400 { - e := fmt.Errorf("failed to load rules from remote url (%s) on '%s'", resp.Status, rulesURL) - return errors.Join(e, err) - } - - var reader io.Reader - - isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip" - - if isGzip { - reader, err = gzip.NewReader(resp.Body) - - if err != nil { - return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err) - } - } else { - reader = resp.Body - } - - err = yaml.NewDecoder(reader).Decode(&r) - - if err != nil { - e := fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error", rulesURL, resp.Status) - ee := errors.Join(e, err) - - return ee - } - - *rs = append(*rs, r...) - - return nil -} - -// ================= utility methods ========================== - -// Yaml returns the ruleset as a Yaml string -func (rs *RuleSet) Yaml() (string, error) { - y, err := yaml.Marshal(rs) - if err != nil { - return "", err - } - - return string(y), nil -} - -// GzipYaml returns an io.Reader that streams the Gzip-compressed YAML representation of the RuleSet. -func (rs *RuleSet) GzipYaml() (io.Reader, error) { - pr, pw := io.Pipe() - - go func() { - defer pw.Close() - - gw := gzip.NewWriter(pw) - defer gw.Close() - - if err := yaml.NewEncoder(gw).Encode(rs); err != nil { - gw.Close() // Ensure to close the gzip writer - pw.CloseWithError(err) - return - } - }() - - return pr, nil -} - -// Domains extracts and returns a slice of all domains present in the RuleSet. -func (rs *RuleSet) Domains() []string { - var domains []string - for _, rule := range *rs { - domains = append(domains, rule.Domain) - domains = append(domains, rule.Domains...) - } - return domains -} - -// DomainCount returns the count of unique domains present in the RuleSet. -func (rs *RuleSet) DomainCount() int { - return len(rs.Domains()) -} - -// Count returns the total number of rules in the RuleSet. -func (rs *RuleSet) Count() int { - return len(*rs) -} - -// PrintStats logs the number of rules and domains loaded in the RuleSet. -func (rs *RuleSet) PrintStats() { - log.Printf("INFO: Loaded %d rules for %d domains\n", rs.Count(), rs.DomainCount()) -} - -// debugPrintRule is a utility function for printing a rule and associated error for debugging purposes. -func debugPrintRule(rule string, err error) { - fmt.Println("------------------------------ BEGIN DEBUG RULESET -----------------------------") - fmt.Printf("%s\n", err.Error()) - fmt.Println("--------------------------------------------------------------------------------") - fmt.Println(rule) - fmt.Println("------------------------------ END DEBUG RULESET -------------------------------") -} diff --git a/pkg/ruleset/ruleset_test.go b/pkg/ruleset/ruleset_test.go deleted file mode 100644 index 42f115cb..00000000 --- a/pkg/ruleset/ruleset_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package ruleset - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/gofiber/fiber/v2" - "github.com/stretchr/testify/assert" -) - -var ( - validYAML = ` -- domain: example.com - regexRules: - - match: "^http:" - replace: "https:"` - - invalidYAML = ` -- domain: [thisIsATestYamlThatIsMeantToFail.example] - regexRules: - - match: "^http:" - replace: "https:" - - match: "[incomplete"` -) - -func TestLoadRulesFromRemoteFile(t *testing.T) { - app := fiber.New() - defer app.Shutdown() - - app.Get("/valid-config.yml", func(c *fiber.Ctx) error { - c.SendString(validYAML) - return nil - }) - - app.Get("/invalid-config.yml", func(c *fiber.Ctx) error { - c.SendString(invalidYAML) - return nil - }) - - app.Get("/valid-config.gz", func(c *fiber.Ctx) error { - c.Set("Content-Type", "application/octet-stream") - - rs, err := loadRuleFromString(validYAML) - if err != nil { - t.Errorf("failed to load valid yaml from string: %s", err.Error()) - } - - s, err := rs.GzipYaml() - if err != nil { - t.Errorf("failed to load gzip serialize yaml: %s", err.Error()) - } - - err = c.SendStream(s) - if err != nil { - t.Errorf("failed to stream gzip serialized yaml: %s", err.Error()) - } - return nil - }) - - // Start the server in a goroutine - go func() { - if err := app.Listen("127.0.0.1:9999"); err != nil { - t.Errorf("Server failed to start: %s", err.Error()) - } - }() - - // Wait for the server to start - time.Sleep(time.Second * 1) - - rs, err := NewRuleset("http://127.0.0.1:9999/valid-config.yml") - if err != nil { - t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error()) - } - - assert.Equal(t, rs[0].Domain, "example.com") - - rs, err = NewRuleset("http://127.0.0.1:9999/valid-config.gz") - if err != nil { - t.Errorf("failed to load gzipped ruleset from http server: %s", err.Error()) - } - - assert.Equal(t, rs[0].Domain, "example.com") - - os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.gz") - - rs = NewRulesetFromEnv() - if !assert.Equal(t, rs[0].Domain, "example.com") { - t.Error("expected no errors loading ruleset from gzip url using environment variable, but got one") - } -} - -func loadRuleFromString(yaml string) (RuleSet, error) { - // Create a temporary file and load it - tmpFile, _ := os.CreateTemp("", "ruleset*.yaml") - - defer os.Remove(tmpFile.Name()) - - tmpFile.WriteString(yaml) - - rs := RuleSet{} - err := rs.loadRulesFromLocalFile(tmpFile.Name()) - - return rs, err -} - -// TestLoadRulesFromLocalFile tests the loading of rules from a local YAML file. -func TestLoadRulesFromLocalFile(t *testing.T) { - rs, err := loadRuleFromString(validYAML) - if err != nil { - t.Errorf("Failed to load rules from valid YAML: %s", err) - } - - assert.Equal(t, rs[0].Domain, "example.com") - assert.Equal(t, rs[0].RegexRules[0].Match, "^http:") - assert.Equal(t, rs[0].RegexRules[0].Replace, "https:") - - _, err = loadRuleFromString(invalidYAML) - if err == nil { - t.Errorf("Expected an error when loading invalid YAML, but got none") - } -} - -// TestLoadRulesFromLocalDir tests the loading of rules from a local nested directory full of yaml rulesets -func TestLoadRulesFromLocalDir(t *testing.T) { - // Create a temporary directory - baseDir, err := os.MkdirTemp("", "ruleset_test") - if err != nil { - t.Fatalf("Failed to create temporary directory: %s", err) - } - - defer os.RemoveAll(baseDir) - - // Create a nested subdirectory - nestedDir := filepath.Join(baseDir, "nested") - err = os.Mkdir(nestedDir, 0o755) - - if err != nil { - t.Fatalf("Failed to create nested directory: %s", err) - } - - // Create a nested subdirectory - nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice") - err = os.Mkdir(nestedTwiceDir, 0o755) - if err != nil { - t.Fatalf("Failed to create twice-nested directory: %s", err) - } - - testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"} - for _, fileName := range testCases { - filePath := filepath.Join(nestedDir, "2x-"+fileName) - os.WriteFile(filePath, []byte(validYAML), 0o644) - - filePath = filepath.Join(nestedDir, fileName) - os.WriteFile(filePath, []byte(validYAML), 0o644) - - filePath = filepath.Join(baseDir, "base-"+fileName) - os.WriteFile(filePath, []byte(validYAML), 0o644) - } - - rs := RuleSet{} - err = rs.loadRulesFromLocalDir(baseDir) - - assert.NoError(t, err) - assert.Equal(t, rs.Count(), len(testCases)*3) - - for _, rule := range rs { - assert.Equal(t, rule.Domain, "example.com") - assert.Equal(t, rule.RegexRules[0].Match, "^http:") - assert.Equal(t, rule.RegexRules[0].Replace, "https:") - } -} diff --git a/proxychain/codegen/README.md b/proxychain/codegen/README.md new file mode 100644 index 00000000..2fbe8143 --- /dev/null +++ b/proxychain/codegen/README.md @@ -0,0 +1,16 @@ +## TLDR +- If you create, delete or rename any request/response modifier, run `go run codegen.go`, so that ruleset unmarshaling will work properly. + +## Overview + +The `codegen.go` file is a utility for the rulesets that automatically generates Go code that maps functional options names found in response/request modifiers to corresponding factory functions. This generation is crucial for the serialization of rulesets from JSON or YAML into functional options suitable for use in proxychains. The tool processes Go files containing modifier functions and generates the necessary mappings. + +- The generated mappings will be written in `proxychain/ruleset/rule_reqmod_types.gen.go` and `proxychain/ruleset/rule_resmod_types.gen.go`. +- These files are used in UnmarshalJSON and UnmarshalYAML methods of the rule type, found in `proxychain/ruleset/rule.go` + + +## Usage +```sh +go run codegen.go +``` + diff --git a/proxychain/codegen/codegen.go b/proxychain/codegen/codegen.go new file mode 100644 index 00000000..6d1292f9 --- /dev/null +++ b/proxychain/codegen/codegen.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + + //"io/fs" + "os" + "path/filepath" + "strings" + //"strings" +) + +func responseModToFactoryMap(fn *ast.FuncDecl) (modMap string) { + paramCount := len(fn.Type.Params.List) + name := fn.Name.Name + var x string + switch paramCount { + case 0: + x = fmt.Sprintf(" rsmModMap[\"%s\"] = func(_ ...string) proxychain.ResponseModification {\n return tx.%s()\n }\n", name, name) + default: + p := []string{} + for i := 0; i < paramCount; i++ { + p = append(p, fmt.Sprintf("params[%d]", i)) + } + params := strings.Join(p, ", ") + x = fmt.Sprintf(" rsmModMap[\"%s\"] = func(params ...string) proxychain.ResponseModification {\n return tx.%s(%s)\n }\n", name, name, params) + } + return x +} + +func responseModCodeGen(dir string) (code string, err error) { + fset := token.NewFileSet() + + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + factoryMaps := []string{} + for _, file := range files { + if !shouldGenCodeFor(file) { + continue + } + + // Parse each Go file + node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments) + if err != nil { + return "", err + } + + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && fn.Recv == nil && fn.Name.IsExported() { + factoryMaps = append(factoryMaps, responseModToFactoryMap(fn)) + } + return true + }) + + } + + code = fmt.Sprintf(` +package ruleset_v2 +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go +// The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" +) + +type ResponseModifierFactory func(params ...string) proxychain.ResponseModification + +var rsmModMap map[string]ResponseModifierFactory + +func init() { + rsmModMap = make(map[string]ResponseModifierFactory) + + %s +}`, strings.Join(factoryMaps, "\n")) + // fmt.Println(code) + return code, nil +} + +func requestModToFactoryMap(fn *ast.FuncDecl) (modMap string) { + paramCount := len(fn.Type.Params.List) + name := fn.Name.Name + var x string + switch paramCount { + case 0: + x = fmt.Sprintf(" rqmModMap[\"%s\"] = func(_ ...string) proxychain.RequestModification {\n return rx.%s()\n }\n", name, name) + default: + p := []string{} + for i := 0; i < paramCount; i++ { + p = append(p, fmt.Sprintf("params[%d]", i)) + } + params := strings.Join(p, ", ") + x = fmt.Sprintf(" rqmModMap[\"%s\"] = func(params ...string) proxychain.RequestModification {\n return rx.%s(%s)\n }\n", name, name, params) + } + return x +} + +func requestModCodeGen(dir string) (code string, err error) { + fset := token.NewFileSet() + + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + factoryMaps := []string{} + for _, file := range files { + if !shouldGenCodeFor(file) { + continue + } + + // Parse each Go file + node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments) + if err != nil { + return "", err + } + + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && fn.Recv == nil && fn.Name.IsExported() { + factoryMaps = append(factoryMaps, requestModToFactoryMap(fn)) + } + return true + }) + + } + + code = fmt.Sprintf(` +package ruleset_v2 +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go +// The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain" + rx "github.com/everywall/ladder/proxychain/requestmodifiers" +) + +type RequestModifierFactory func(params ...string) proxychain.RequestModification + +var rqmModMap map[string]RequestModifierFactory + +func init() { + rqmModMap = make(map[string]RequestModifierFactory) + + %s +}`, strings.Join(factoryMaps, "\n")) + // fmt.Println(code) + return code, nil +} + +func shouldGenCodeFor(file fs.DirEntry) bool { + if file.IsDir() { + return false + } + if filepath.Ext(file.Name()) != ".go" { + return false + } + if strings.HasSuffix(file.Name(), "_test.go") { + return false + } + return true +} + +func main() { + rqmCode, err := requestModCodeGen("../requestmodifiers/") + if err != nil { + panic(err) + } + // fmt.Println(rqmCode) + + fq, err := os.Create("../ruleset/rule_reqmod_types.gen.go") + if err != nil { + panic(err) + } + _, err = io.WriteString(fq, rqmCode) + if err != nil { + panic(err) + } + + rsmCode, err := responseModCodeGen("../responsemodifiers/") + if err != nil { + panic(err) + } + // fmt.Println(rsmCode) + + fs, err := os.Create("../ruleset/rule_resmod_types.gen.go") + if err != nil { + panic(err) + } + _, err = io.WriteString(fs, rsmCode) + if err != nil { + panic(err) + } +} diff --git a/proxychain/proxychain.go b/proxychain/proxychain.go new file mode 100644 index 00000000..2043b2a8 --- /dev/null +++ b/proxychain/proxychain.go @@ -0,0 +1,555 @@ +package proxychain + +import ( + "errors" + "fmt" + "io" + "log" + "net/url" + "strings" + + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" + profiles "github.com/bogdanfinn/tls-client/profiles" + + "github.com/gofiber/fiber/v2" +) + +/* +ProxyChain manages the process of forwarding an HTTP request to an upstream server, +applying request and response modifications along the way. + + - It accepts incoming HTTP requests (as a Fiber *ctx), and applies + request modifiers (ReqMods) and response modifiers (ResMods) before passing the + upstream response back to the client. + + - ProxyChains can be reused to avoid memory allocations. However, they are not concurrent-safe + so a ProxyChainPool should be used with mutexes to avoid memory errors. + +--- + +# EXAMPLE + +``` + +import ( + + rx "ladder/pkg/proxychain/requestmodifiers" + tx "ladder/pkg/proxychain/responsemodifiers" + "ladder/pkg/proxychain/responsemodifiers/rewriters" + "ladder/internal/proxychain" + +) + +proxychain.NewProxyChain(). + + SetFiberCtx(c). + SetRequestModifications( + rx.BlockOutgoingCookies(), + rx.SpoofOrigin(), + rx.SpoofReferrer(), + ). + SetResultModifications( + tx.BlockIncomingCookies(), + tx.RewriteHTMLResourceURLs() + ). + Execute() + +``` + + client ladder service upstream + +┌─────────┐ ┌────────────────────────┐ ┌─────────┐ +│ │GET │ │ │ │ +│ req────┼───► ProxyChain │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ apply │ │ │ +│ │ │ RequestModifications │ │ │ +│ │ │ │ │ │ │ +│ │ │ ▼ │ │ │ +│ │ │ send GET │ │ │ +│ │ │ Request req────────┼─► │ │ +│ │ │ │ │ │ +│ │ │ 200 OK │ │ │ +│ │ │ ┌────────────────┼─response │ +│ │ │ ▼ │ │ │ +│ │ │ apply │ │ │ +│ │ │ ResultModifications │ │ │ +│ │ │ │ │ │ │ +│ │◄───┼───────┘ │ │ │ +│ │ │ 200 OK │ │ │ +│ │ │ │ │ │ +└─────────┘ └────────────────────────┘ └─────────┘ +*/ +type ProxyChain struct { + Context *fiber.Ctx + Client HTTPClient + onceClient HTTPClient + Request *http.Request + Response *http.Response + requestModifications []RequestModification + onceRequestModifications []RequestModification + onceResponseModifications []ResponseModification + responseModifications []ResponseModification + debugMode bool + abortErr error + APIPrefix string +} + +// a ProxyStrategy is a pre-built proxychain with purpose-built defaults +type ProxyStrategy ProxyChain + +// A RequestModification is a function that should operate on the +// ProxyChain Req or Client field, using the fiber ctx as needed. +type RequestModification func(*ProxyChain) error + +// A ResponseModification is a function that should operate on the +// ProxyChain Res (http result) & Body (buffered http response body) field +type ResponseModification func(*ProxyChain) error + +// abstraction over HTTPClient +type HTTPClient interface { + GetCookies(u *url.URL) []*http.Cookie + SetCookies(u *url.URL, cookies []*http.Cookie) + SetCookieJar(jar http.CookieJar) + GetCookieJar() http.CookieJar + SetProxy(proxyURL string) error + GetProxy() string + SetFollowRedirect(followRedirect bool) + GetFollowRedirect() bool + CloseIdleConnections() + Do(req *http.Request) (*http.Response, error) + Get(url string) (resp *http.Response, err error) + Head(url string) (resp *http.Response, err error) + Post(url, contentType string, body io.Reader) (resp *http.Response, err error) +} + +// SetRequestModifications sets the ProxyChain's request modifiers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) SetRequestModifications(mods ...RequestModification) *ProxyChain { + chain.requestModifications = mods + return chain +} + +// AddRequestModifications adds more request modifiers to the ProxyChain +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) AddRequestModifications(mods ...RequestModification) *ProxyChain { + chain.requestModifications = append(chain.requestModifications, mods...) + return chain +} + +// AddOnceRequestModifications adds a request modifier to the ProxyChain that should only fire once +// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied. +func (chain *ProxyChain) AddOnceRequestModifications(mods ...RequestModification) *ProxyChain { + chain.onceRequestModifications = append(chain.onceRequestModifications, mods...) + return chain +} + +// AddOnceResponseModifications adds a response modifier to the ProxyChain that should only fire once +// the modifier will not fire until ProxyChain.Execute() is run and will be removed after it has been applied. +func (chain *ProxyChain) AddOnceResponseModifications(mods ...ResponseModification) *ProxyChain { + chain.onceResponseModifications = append(chain.onceResponseModifications, mods...) + return chain +} + +// AddResponseModifications sets the ProxyChain's response modifiers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) AddResponseModifications(mods ...ResponseModification) *ProxyChain { + chain.responseModifications = append(chain.responseModifications, mods...) + return chain +} + +// SetResponseModifications sets the ProxyChain's response modifiers +// the modifier will not fire until ProxyChain.Execute() is run. +func (chain *ProxyChain) SetResponseModifications(mods ...ResponseModification) *ProxyChain { + chain.responseModifications = mods + return chain +} + +// WithAPIPath trims the path during URL extraction. +// example: using path = "api/outline/", a path like "http://localhost:8080/api/outline/https://example.com" becomes "https://example.com" +func (chain *ProxyChain) WithAPIPath(path string) *ProxyChain { + chain.APIPrefix = path + chain.APIPrefix = strings.TrimSuffix(chain.APIPrefix, "*") + return chain +} + +func (chain *ProxyChain) _initializeRequest() (*http.Request, error) { + if chain.Context == nil { + chain.abortErr = chain.abort(errors.New("no context set")) + return nil, chain.abortErr + } + // initialize a request (without url) + req, err := http.NewRequest(chain.Context.Method(), "", nil) + if err != nil { + return nil, err + } + chain.Request = req + switch chain.Context.Method() { + case "GET": + case "DELETE": + case "HEAD": + case "OPTIONS": + break + case "POST": + case "PUT": + case "PATCH": + // stream content of body from client request to upstream request + chain.Request.Body = io.NopCloser(chain.Context.Request().BodyStream()) + default: + return nil, fmt.Errorf("unsupported request method from client: '%s'", chain.Context.Method()) + } + + return req, nil +} + +// reconstructURLFromReferer reconstructs the URL using the referer's scheme, host, and the relative path / queries +func reconstructURLFromReferer(referer *url.URL, relativeURL *url.URL) (*url.URL, error) { + // Extract the real url from referer path + realURL, err := url.Parse(strings.TrimPrefix(referer.Path, "/")) + if err != nil { + return nil, fmt.Errorf("error parsing real URL from referer '%s': %v", referer.Path, err) + } + + if realURL.Scheme == "" || realURL.Host == "" { + return nil, fmt.Errorf("invalid referer URL: '%s' on request '%s", referer.String(), relativeURL.String()) + } + + log.Printf("rewrite relative URL using referer: '%s' -> '%s'\n", relativeURL.String(), realURL.String()) + + return &url.URL{ + Scheme: referer.Scheme, + Host: referer.Host, + Path: realURL.Path, + RawQuery: realURL.RawQuery, + }, nil +} + +// prevents calls like: http://localhost:8080/http://localhost:8080 +func preventRecursiveProxyRequest(urlQuery *url.URL, baseProxyURL string) *url.URL { + u := urlQuery.String() + isRecursive := strings.HasPrefix(u, baseProxyURL) || u == baseProxyURL + if !isRecursive { + return urlQuery + } + + fixedURL, err := url.Parse(strings.TrimPrefix(strings.TrimPrefix(urlQuery.String(), baseProxyURL), "/")) + if err != nil { + log.Printf("proxychain: failed to fix recursive request: '%s' -> '%s\n'", baseProxyURL, u) + return urlQuery + } + return preventRecursiveProxyRequest(fixedURL, baseProxyURL) +} + +// extractURL extracts a URL from the request ctx +func (chain *ProxyChain) extractURL() (*url.URL, error) { + isLocal := strings.HasPrefix(chain.Context.BaseURL(), "http://localhost") || strings.HasPrefix(chain.Context.BaseURL(), "http://127.0.0.1") + isReqPath := strings.HasPrefix(chain.Context.Path(), "/http") + isAPI := strings.HasPrefix(chain.Context.Path(), "/api") + isOutline := strings.HasPrefix(chain.Context.Path(), "/outline") + + if isLocal || isReqPath || isAPI || isOutline { + return chain.extractURLFromPath() + } + + u, err := url.Parse(chain.Context.BaseURL()) + if err != nil { + return &url.URL{}, err + } + parts := strings.Split(u.Hostname(), ".") + if len(parts) < 2 { + fmt.Println("path") + return chain.extractURLFromPath() + } + + return chain.extractURLFromSubdomain() +} + +// extractURLFromPath extracts a URL from the request ctx if subdomains are used. +func (chain *ProxyChain) extractURLFromSubdomain() (*url.URL, error) { + u, err := url.Parse(chain.Context.BaseURL()) + if err != nil { + return &url.URL{}, err + } + parts := strings.Split(u.Hostname(), ".") + if len(parts) < 2 { + // no subdomain set, fallback to path extraction + //panic("asdf") + return chain.extractURLFromPath() + } + subdomain := strings.Join(parts[:len(parts)-2], ".") + subURL := subdomain + subURL = strings.ReplaceAll(subURL, "--", "|") + subURL = strings.ReplaceAll(subURL, "-", ".") + subURL = strings.ReplaceAll(subURL, "|", "-") + return url.Parse(fmt.Sprintf("https://%s/%s", subURL, u.Path)) +} + +// extractURLFromPath extracts a URL from the request ctx. If the URL in the request +// is a relative path, it reconstructs the full URL using the referer header. +func (chain *ProxyChain) extractURLFromPath() (*url.URL, error) { + reqURL := chain.Context.Params("*") + + reqURL = strings.TrimPrefix(reqURL, chain.APIPrefix) + + // sometimes client requests doubleroot '//' + // there is a bug somewhere else, but this is a workaround until we find it + if strings.HasPrefix(reqURL, "/") || strings.HasPrefix(reqURL, `%2F`) { + reqURL = strings.TrimPrefix(reqURL, "/") + reqURL = strings.TrimPrefix(reqURL, `%2F`) + } + + // unescape url query + uReqURL, err := url.QueryUnescape(reqURL) + if err == nil { + reqURL = uReqURL + } + + urlQuery, err := url.Parse(reqURL) + if err != nil { + return nil, fmt.Errorf("error parsing request URL '%s': %v", reqURL, err) + } + + // prevent recursive proxy requests + fullURL := chain.Context.Request().URI() + proxyURL := fmt.Sprintf("%s://%s", fullURL.Scheme(), fullURL.Host()) + urlQuery = preventRecursiveProxyRequest(urlQuery, proxyURL) + + // Handle standard paths + // eg: https://localhost:8080/https://realsite.com/images/foobar.jpg -> https://realsite.com/images/foobar.jpg + isRelativePath := urlQuery.Scheme == "" + if !isRelativePath { + return urlQuery, nil + } + + // Handle relative URLs + // eg: https://localhost:8080/images/foobar.jpg -> https://realsite.com/images/foobar.jpg + referer, err := url.Parse(chain.Context.Get("referer")) + relativePath := urlQuery + if err != nil { + return nil, fmt.Errorf("error parsing referer URL from req: '%s': %v", relativePath, err) + } + return reconstructURLFromReferer(referer, relativePath) +} + +// SetFiberCtx takes the request ctx from the client +// for the modifiers and execute function to use. +// it must be set everytime a new request comes through +// if the upstream request url cannot be extracted from the ctx, +// a 500 error will be sent back to the client +func (chain *ProxyChain) SetFiberCtx(ctx *fiber.Ctx) *ProxyChain { + chain.Context = ctx + + // initialize the request and prepare it for modification + req, err := chain._initializeRequest() + if err != nil { + chain.abortErr = chain.abort(err) + } + chain.Request = req + + // extract the URL for the request and add it to the new request + url, err := chain.extractURL() + if err != nil { + chain.abortErr = chain.abort(err) + } else { + chain.Request.URL = url + fmt.Printf("extracted URL: %s\n", chain.Request.URL) + } + + return chain +} + +func (chain *ProxyChain) validateCtxIsSet() error { + if chain.Context != nil { + return nil + } + err := errors.New("proxyChain was called without setting a fiber Ctx. Use ProxyChain.SetFiberCtx()") + chain.abortErr = chain.abort(err) + return chain.abortErr +} + +// SetHTTPClient sets a new upstream http client transport +// useful for modifying TLS +func (chain *ProxyChain) SetHTTPClient(httpClient HTTPClient) *ProxyChain { + chain.Client = httpClient + return chain +} + +// SetOnceHTTPClient sets a new upstream http client transport temporarily +// and clears it once it is used. +func (chain *ProxyChain) SetOnceHTTPClient(httpClient HTTPClient) *ProxyChain { + chain.onceClient = httpClient + return chain +} + +// SetVerbose changes the logging behavior to print +// the modification steps and applied rulesets for debugging +func (chain *ProxyChain) SetDebugLogging(isDebugMode bool) *ProxyChain { + if isDebugMode { + log.Println("DEBUG MODE ENABLED") + } + chain.debugMode = isDebugMode + return chain +} + +// abort proxychain and return 500 error to client +// this will prevent Execute from firing and reset the state +// returns the initial error enriched with context +func (chain *ProxyChain) abort(err error) error { + // defer chain._reset() + chain.abortErr = err + // chain.Context.Response().SetStatusCode(500) + // var e error + // if chain.Request.URL != nil { + // e = fmt.Errorf("ProxyChain error for '%s': %s", chain.Request.URL.String(), err.Error()) + // } else { + // e = fmt.Errorf("ProxyChain error: '%s'", err.Error()) + // } + // chain.Context.SendString(e.Error()) // <- RenderErrorPage middleware to render error + // log.Println(e.Error()) + return err +} + +// internal function to reset state of ProxyChain for reuse +func (chain *ProxyChain) _reset() { + chain.abortErr = nil + chain.Request = nil + // chain.Response = nil + chain.Context = nil + chain.onceResponseModifications = []ResponseModification{} + chain.onceRequestModifications = []RequestModification{} + // chain.onceClient = nil +} + +// NewProxyChain initializes a new ProxyChain +func NewProxyChain() *ProxyChain { + chain := new(ProxyChain) + + options := []tls_client.HttpClientOption{ + tls_client.WithTimeoutSeconds(20), + //tls_client.WithRandomTLSExtensionOrder(), + tls_client.WithClientProfile(profiles.Chrome_117), + // tls_client.WithNotFollowRedirects(), + // tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument + } + client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) + if err != nil { + panic(err) + } + chain.Client = client + + return chain +} + +/// ======================================================================================================== + +// _execute sends the request for the ProxyChain and returns the raw body only +// the caller is responsible for returning a response back to the requestor +// the caller is also responsible for calling chain._reset() when they are done with the body +func (chain *ProxyChain) _execute() (io.Reader, error) { + // ================== PREFLIGHT CHECKS ============================= + if chain.validateCtxIsSet() != nil || chain.abortErr != nil { + return nil, chain.abortErr + } + if chain.Request == nil { + return nil, errors.New("proxychain request not yet initialized") + } + if chain.Request.URL.Scheme == "" { + return nil, errors.New("request url not set or invalid. Check ProxyChain ReqMods for issues") + } + + // ======== REQUEST MODIFICATIONS :: [client -> ladder] -> upstream -> ladder -> client ============================= + // Apply requestModifications to proxychain + for _, applyRequestModificationsTo := range chain.requestModifications { + err := applyRequestModificationsTo(chain) + if err != nil { + return nil, chain.abort(err) + } + } + + // Apply onceRequestModifications to proxychain and clear them + for len(chain.onceRequestModifications) > 0 { + i := 0 + modFn := chain.onceRequestModifications[i] + err := modFn(chain) + if err != nil { + return nil, chain.abort(err) + } + // pop modFn off slice + chain.onceRequestModifications = append(chain.onceRequestModifications[:i], chain.onceRequestModifications[i+1:]...) + } + + // ======== SEND REQUEST UPSTREAM :: client -> [ladder -> upstream] -> ladder -> client ============================= + // Send Request Upstream + if chain.onceClient != nil { + // if chain.SetOnceClient() is used, use that client instead of the + // default http client temporarily. + resp, err := chain.onceClient.Do(chain.Request) + if err != nil { + return nil, chain.abort(err) + } + chain.Response = resp + // chain.onceClient = nil + } else { + resp, err := chain.Client.Do(chain.Request) + if err != nil { + return nil, chain.abort(err) + } + chain.Response = resp + } + + // ======== APPLY RESPONSE MODIFIERS :: client -> ladder -> [upstream -> ladder] -> client ============================= + // Apply ResponseModifiers to proxychain + for _, applyResultModificationsTo := range chain.responseModifications { + err := applyResultModificationsTo(chain) + if err != nil { + return nil, chain.abort(err) + } + } + + // Apply onceResponseModifications to proxychain and clear them + for len(chain.onceResponseModifications) > 0 { + i := 0 + modFn := chain.onceResponseModifications[i] + err := modFn(chain) + if err != nil { + return nil, chain.abort(err) + } + // pop modFn off slice + chain.onceResponseModifications = append(chain.onceResponseModifications[:i], chain.onceResponseModifications[i+1:]...) + } + + // ======== RETURN BODY TO CLIENT :: client -> ladder -> upstream -> [ladder -> client] ============================= + return chain.Response.Body, nil +} + +// Execute sends the request for the ProxyChain and returns the request to the sender +// and resets the fields so that the ProxyChain can be reused. +// if any step in the ProxyChain fails, the request will abort and a 500 error will +// be returned to the client +func (chain *ProxyChain) Execute() error { + defer chain._reset() + body, err := chain._execute() + if err != nil { + log.Println(err) + return err + } + if chain.Context == nil { + return errors.New("no context set") + } + + // TODO: this seems broken + // in case api user did not set or forward content-type, we do it for them + /* + ct := string(chain.Context.Response().Header.Peek("content-type")) + if ct == "" { + chain.Context.Set("content-type", chain.Response.Header.Get("content-type")) + } + */ + + // Return request back to client + return chain.Context.SendStream(body) + + // return chain.Context.SendStream(body) +} diff --git a/proxychain/proxychain_pool.go b/proxychain/proxychain_pool.go new file mode 100644 index 00000000..ac5aa0ff --- /dev/null +++ b/proxychain/proxychain_pool.go @@ -0,0 +1,11 @@ +package proxychain + +import ( + "net/url" +) + +type Pool map[url.URL]ProxyChain + +func NewPool() Pool { + return map[url.URL]ProxyChain{} +} diff --git a/proxychain/requestmodifiers/add_cache_buster_query.go b/proxychain/requestmodifiers/add_cache_buster_query.go new file mode 100644 index 00000000..d36fdd37 --- /dev/null +++ b/proxychain/requestmodifiers/add_cache_buster_query.go @@ -0,0 +1,29 @@ +package requestmodifiers + +import ( + "math/rand" + + "github.com/everywall/ladder/proxychain" +) + +// AddCacheBusterQuery modifies query params to add a random parameter key +// In order to get the upstream network stack to serve a fresh copy of the page. +func AddCacheBusterQuery() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + ModifyQueryParams("ord", randomString(15)), + ) + + return nil + } +} + +func randomString(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789." + + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return string(b) +} diff --git a/proxychain/requestmodifiers/bot/bot.go b/proxychain/requestmodifiers/bot/bot.go new file mode 100644 index 00000000..8c5df4e2 --- /dev/null +++ b/proxychain/requestmodifiers/bot/bot.go @@ -0,0 +1,142 @@ +package bot + +import ( + "encoding/json" + "fmt" + "io" + "math/big" + "math/bits" + "math/rand" + "net" + "net/http" + "time" +) + +type Bot interface { + UpdatePool() error + GetRandomIdentity() string +} + +type bot struct { + UserAgent string + Fingerprint string + IPPool botPool +} + +type botPool struct { + Timestamp string `json:"creationTime"` + Prefixes []botPrefix `json:"prefixes"` +} + +type botPrefix struct { + IPv6 string `json:"ipv6Prefix,omitempty"` + IPv4 string `json:"ipv4Prefix,omitempty"` +} + +// TODO: move pointers around, not global variables +var GoogleBot = bot{ + UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36", + + // https://github.com/trisulnsm/trisul-scripts/blob/master/lua/frontend_scripts/reassembly/ja3/prints/ja3fingerprint.json + Fingerprint: "769,49195-49199-49196-49200-52393-52392-52244-52243-49161-49171-49162-49172-156-157-47-53-10,65281-0-23-35-13-5-18-16-11-10-21,29-23-24,0", + + IPPool: botPool{ + Timestamp: "2023-11-28T23:00:56.000000", + Prefixes: []botPrefix{ + { + IPv4: "34.100.182.96/28", + }, + }, + }, +} + +var BingBot = bot{ + UserAgent: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/79.0.3945.120 Safari/537.36", + IPPool: botPool{ + Timestamp: "2023-03-08T10:00:00.121331", + Prefixes: []botPrefix{ + { + IPv4: "207.46.13.0/24", + }, + }, + }, +} + +func (b *bot) UpdatePool(url string) error { + client := &http.Client{Timeout: 10 * time.Second} + + resp, err := client.Get(url) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to update googlebot IP pool: status code %s", resp.Status) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + err = json.Unmarshal(body, &b.IPPool) + + return err +} + +func (b *bot) GetRandomIP() string { + count := len(b.IPPool.Prefixes) + + var prefix botPrefix + + if count == 1 { + prefix = b.IPPool.Prefixes[0] + } else { + idx := rand.Intn(count) + prefix = b.IPPool.Prefixes[idx] + } + + if prefix.IPv4 != "" { + ip, err := randomIPFromSubnet(prefix.IPv4) + if err == nil { + return ip.String() + } + } + + if prefix.IPv6 != "" { + ip, err := randomIPFromSubnet(prefix.IPv6) + if err == nil { + return ip.String() + } + } + + // fallback to default IP which is known to work + ip, _ := randomIPFromSubnet(b.IPPool.Prefixes[0].IPv4) + + return ip.String() +} + +func randomIPFromSubnet(c string) (net.IP, error) { + ip, ipnet, err := net.ParseCIDR(c) + if err != nil { + return nil, err + } + + // int representation of byte mask + mask := big.NewInt(0).SetBytes(ipnet.Mask).Uint64() + + // how many unset bits there are at the end of the mask + offset := bits.TrailingZeros8(byte(0) ^ byte(mask)) + + // total number of ips available in the block + offset *= offset + + toAdd := rand.Intn(offset) + + last := len(ip) - 1 + ip[last] = ip[last] + byte(toAdd) + + return ip, nil +} diff --git a/proxychain/requestmodifiers/bot/bot_test.go b/proxychain/requestmodifiers/bot/bot_test.go new file mode 100644 index 00000000..74dbfc3d --- /dev/null +++ b/proxychain/requestmodifiers/bot/bot_test.go @@ -0,0 +1,36 @@ +package bot + +import ( + "net" + "testing" +) + +func TestRandomIPFromSubnet(t *testing.T) { + err := GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + if err != nil { + t.Error(err) + } + + for _, prefix := range GoogleBot.IPPool.Prefixes { + subnet := prefix.IPv4 + if prefix.IPv6 != "" { + subnet = prefix.IPv6 + } + + t.Run(subnet, func(t *testing.T) { + _, ipnet, err := net.ParseCIDR(subnet) + if err != nil { + t.Error(err) + } + + ip, err := randomIPFromSubnet(subnet) + if err != nil { + t.Error(err) + } + + if !ipnet.Contains(ip) { + t.Fail() + } + }) + } +} diff --git a/proxychain/requestmodifiers/forward_request_headers.go b/proxychain/requestmodifiers/forward_request_headers.go new file mode 100644 index 00000000..1e8a4398 --- /dev/null +++ b/proxychain/requestmodifiers/forward_request_headers.go @@ -0,0 +1,45 @@ +package requestmodifiers + +import ( + "strings" + //"fmt" + "github.com/everywall/ladder/proxychain" +) + +var forwardBlacklist map[string]bool + +func init() { + forwardBlacklist = map[string]bool{ + "host": true, + "connection": true, + "keep-alive": true, + "content-length": true, + "content-encoding": true, + "transfer-encoding": true, + "referer": true, + "x-forwarded-for": true, + "x-real-ip": true, + "forwarded": true, + "accept-encoding": true, + } +} + +// ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server +func ForwardRequestHeaders() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + forwardHeaders := func(key, value []byte) { + k := strings.ToLower(string(key)) + v := string(value) + if forwardBlacklist[k] { + return + } + // fmt.Println(k, v) + chain.Request.Header.Set(k, v) + } + + chain.Context.Request(). + Header.VisitAll(forwardHeaders) + + return nil + } +} diff --git a/proxychain/requestmodifiers/masquerade_as_trusted_bot.go b/proxychain/requestmodifiers/masquerade_as_trusted_bot.go new file mode 100644 index 00000000..eb8db8d4 --- /dev/null +++ b/proxychain/requestmodifiers/masquerade_as_trusted_bot.go @@ -0,0 +1,127 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain/requestmodifiers/bot" + + "github.com/everywall/ladder/proxychain" +) + +// MasqueradeAsGoogleBot modifies user agent and x-forwarded for +// to appear to be a Google Bot +func MasqueradeAsGoogleBot() proxychain.RequestModification { + ip := bot.GoogleBot.GetRandomIP() + + return masqueradeAsTrustedBot(bot.GoogleBot.UserAgent, ip, bot.GoogleBot.Fingerprint) +} + +// MasqueradeAsBingBot modifies user agent and x-forwarded for +// to appear to be a Bing Bot +func MasqueradeAsBingBot() proxychain.RequestModification { + ip := bot.BingBot.GetRandomIP() + + return masqueradeAsTrustedBot(bot.BingBot.Fingerprint, ip, "") +} + +// MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for +// to appear to be a archive.org (wayback machine) Bot +func MasqueradeAsWaybackMachineBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" + const botIP string = "207.241.235.164" + return masqueradeAsTrustedBot(botUA, botIP, "") +} + +// MasqueradeAsFacebookBot modifies user agent and x-forwarded for +// to appear to be a Facebook Bot (link previews?) +func MasqueradeAsFacebookBot() proxychain.RequestModification { + const botUA string = "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" + // 31.13.97.0/24, 31.13.99.0/24, 31.13.100.0/24, 66.220.144.0/20, 69.63.189.0/24, 69.63.190.0/24, 69.171.224.0/20, 69.171.240.0/21, 69.171.248.0/24, 173.252.73.0/24, 173.252.74.0/24, 173.252.77.0/24, 173.252.100.0/22, 173.252.104.0/21, 173.252.112.0/24, 2a03:2880:10::/48, 2a03:2880:10ff::/48, 2a03:2880:11::/48, 2a03:2880:11ff::/48, 2a03:2880:20::/48, 2a03:2880:20ff::/48, 2a03:2880:21ff::/48, 2a03:2880:30ff::/48, 2a03:2880:31ff::/48, 2a03:2880:1010::/48, 2a03:2880:1020::/48, 2a03:2880:2020::/48, 2a03:2880:2050::/48, 2a03:2880:2040::/48, 2a03:2880:2110::/48, 2a03:2880:2130::/48, 2a03:2880:3010::/48, 2a03:2880:3020::/48 + const botIP string = "31.13.99.8" + const ja3 string = "771,49199-49195-49171-49161-49200-49196-49172-49162-51-57-50-49169-49159-47-53-10-5-4-255,0-11-10-13-13172-16,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2" + return masqueradeAsTrustedBot(botUA, botIP, ja3) +} + +// MasqueradeAsYandexBot modifies user agent and x-forwarded for +// to appear to be a Yandex Spider Bot +func MasqueradeAsYandexBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" + // 100.43.90.0/24, 37.9.115.0/24, 37.140.165.0/24, 77.88.22.0/25, 77.88.29.0/24, 77.88.31.0/24, 77.88.59.0/24, 84.201.146.0/24, 84.201.148.0/24, 84.201.149.0/24, 87.250.243.0/24, 87.250.253.0/24, 93.158.147.0/24, 93.158.148.0/24, 93.158.151.0/24, 93.158.153.0/32, 95.108.128.0/24, 95.108.138.0/24, 95.108.150.0/23, 95.108.158.0/24, 95.108.156.0/24, 95.108.188.128/25, 95.108.234.0/24, 95.108.248.0/24, 100.43.80.0/24, 130.193.62.0/24, 141.8.153.0/24, 178.154.165.0/24, 178.154.166.128/25, 178.154.173.29, 178.154.200.158, 178.154.202.0/24, 178.154.205.0/24, 178.154.239.0/24, 178.154.243.0/24, 37.9.84.253, 199.21.99.99, 178.154.162.29, 178.154.203.251, 178.154.211.250, 178.154.171.0/24, 178.154.200.0/24, 178.154.244.0/24, 178.154.246.0/24, 95.108.181.0/24, 95.108.246.252, 5.45.254.0/24, 5.255.253.0/24, 37.140.141.0/24, 37.140.188.0/24, 100.43.81.0/24, 100.43.85.0/24, 100.43.91.0/24, 199.21.99.0/24, 2a02:6b8:b000::/32, 2a02:6b8:b010::/32, 2a02:6b8:b011::/32, 2a02:6b8:c0e::/32 + const botIP string = "37.9.115.9" + const ja3 string = "769,49200-49196-49192-49188-49172-49162-165-163-161-159-107-106-105-104-57-56-55-54-136-135-134-133-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-164-162-160-158-103-64-63-62-51-50-49-48-154-153-152-151-69-68-67-66-49201-49197-49193-49189-49166-49156-156-60-47-150-65-7-49169-49159-49164-49154-5-4-49170-49160-22-19-16-13-49165-49155-10-255,0-11-10-35-13-15,23-25-28-27-24-26-22-14-13-11-12-9-10,0-1-2" + return masqueradeAsTrustedBot(botUA, botIP, ja3) +} + +// MasqueradeAsBaiduBot modifies user agent and x-forwarded for +// to appear to be a Baidu Spider Bot +func MasqueradeAsBaiduBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" + // 180.76.15.0/24, 119.63.196.0/24, 115.239.212./24, 119.63.199.0/24, 122.81.208.0/22, 123.125.71.0/24, 180.76.4.0/24, 180.76.5.0/24, 180.76.6.0/24, 185.10.104.0/24, 220.181.108.0/24, 220.181.51.0/24, 111.13.102.0/24, 123.125.67.144/29, 123.125.67.152/31, 61.135.169.0/24, 123.125.68.68/30, 123.125.68.72/29, 123.125.68.80/28, 123.125.68.96/30, 202.46.48.0/20, 220.181.38.0/24, 123.125.68.80/30, 123.125.68.84/31, 123.125.68.0/24 + const botIP string = "180.76.15.7" + return masqueradeAsTrustedBot(botUA, botIP, "") +} + +// MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for +// to appear to be a DuckDuckGo Bot +func MasqueradeAsDuckDuckBot() proxychain.RequestModification { + const botUA string = "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)" + // 46.51.197.88, 46.51.197.89, 50.18.192.250, 50.18.192.251, 107.21.1.61, 176.34.131.233, 176.34.135.167, 184.72.106.52, 184.72.115.86 + const botIP string = "46.51.197.88" + return masqueradeAsTrustedBot(botUA, botIP, "") +} + +// MasqueradeAsYahooBot modifies user agent and x-forwarded for +// to appear to be a Yahoo Bot +func MasqueradeAsYahooBot() proxychain.RequestModification { + const botUA string = "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" + // 5.255.250.0/24, 37.9.87.0/24, 67.195.37.0/24, 67.195.50.0/24, 67.195.110.0/24, 67.195.111.0/24, 67.195.112.0/23, 67.195.114.0/24, 67.195.115.0/24, 68.180.224.0/21, 72.30.132.0/24, 72.30.142.0/24, 72.30.161.0/24, 72.30.196.0/24, 72.30.198.0/24, 74.6.254.0/24, 74.6.8.0/24, 74.6.13.0/24, 74.6.17.0/24, 74.6.18.0/24, 74.6.22.0/24, 74.6.27.0/24, 74.6.168.0/24, 77.88.5.0/24, 77.88.47.0/24, 93.158.161.0/24, 98.137.72.0/24, 98.137.206.0/24, 98.137.207.0/24, 98.139.168.0/24, 114.111.95.0/24, 124.83.159.0/24, 124.83.179.0/24, 124.83.223.0/24, 141.8.144.0/24, 183.79.63.0/24, 183.79.92.0/24, 203.216.255.0/24, 211.14.11.0/24 + const ja3 = "769,49200-49196-49192-49188-49172-49162-163-159-107-106-57-56-136-135-49202-49198-49194-49190-49167-49157-157-61-53-132-49199-49195-49191-49187-49171-49161-162-158-103-64-51-50-49170-49160-154-153-69-68-22-19-49201-49197-49193-49189-49166-49156-49165-49155-156-60-47-150-65-10-7-49169-49159-49164-49154-5-4-255,0-11-10-13-15,25-24-23,0-1-2" + const botIP string = "37.9.87.5" + return masqueradeAsTrustedBot(botUA, botIP, ja3) +} + +func masqueradeAsTrustedBot(botUA string, botIP string, ja3 string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofUserAgent(botUA), + + // general / nginx + SetRequestHeader("X-Forwarded-For", botIP), + SetRequestHeader("X-Real-IP", botIP), + SetRequestHeader("True-Client-IP", botIP), + SetRequestHeader("WL-Proxy-Client-IP", botIP), + SetRequestHeader("X-Cluster-Client-IP", botIP), + /* + // akamai + SetRequestHeader("True-Client-IP", botIP), + + // cloudflare + // TODO: this seems to cause issues with CF... figure out workaround or remove + Error 1000 + Ray ID: xxxxxxxxxxxxxxxx • + 2023-12-01 20:09:22 UTC + DNS points to prohibited IP + What happened? + You've requested a page on a website (xxxxxxxxxxxxxxxxxxx) that is on the Cloudflare network. Unfortunately, it is resolving to an IP address that is creating a conflict within Cloudflare's system + + SetRequestHeader("CF-Connecting-IP", botIP), + + // weblogic + SetRequestHeader("WL-Proxy-Client-IP", botIP), + // azure + SetRequestHeader("X-Cluster-Client-IP", botIP), + */ + + DeleteRequestHeader("referrer"), + DeleteRequestHeader("origin"), + ) + + /* + if ja3 != "" { + chain.AddOnceRequestModifications( + SpoofJA3fingerprint(ja3, botUA), + ) + } + */ + + return nil + } +} diff --git a/proxychain/requestmodifiers/modify_domain_with_regex.go b/proxychain/requestmodifiers/modify_domain_with_regex.go new file mode 100644 index 00000000..21ea69e7 --- /dev/null +++ b/proxychain/requestmodifiers/modify_domain_with_regex.go @@ -0,0 +1,19 @@ +package requestmodifiers + +import ( + "fmt" + "regexp" + + "github.com/everywall/ladder/proxychain" +) + +func ModifyDomainWithRegex(matchRegex string, replacement string) proxychain.RequestModification { + match, err := regexp.Compile(matchRegex) + return func(px *proxychain.ProxyChain) error { + if err != nil { + return fmt.Errorf("RequestModification :: ModifyDomainWithRegex error => invalid match regex: %s - %s", matchRegex, err.Error()) + } + px.Request.URL.Host = match.ReplaceAllString(px.Request.URL.Host, replacement) + return nil + } +} diff --git a/proxychain/requestmodifiers/modify_outgoing_cookies.go b/proxychain/requestmodifiers/modify_outgoing_cookies.go new file mode 100644 index 00000000..71869ec0 --- /dev/null +++ b/proxychain/requestmodifiers/modify_outgoing_cookies.go @@ -0,0 +1,100 @@ +package requestmodifiers + +import ( + //"net/http" + //http "github.com/Danny-Dasilva/fhttp" + http "github.com/bogdanfinn/fhttp" + + "github.com/everywall/ladder/proxychain" +) + +// SetOutgoingCookie modifes a specific cookie name +// by modifying the request cookie headers going to the upstream server. +// If the cookie name does not already exist, it is created. +func SetOutgoingCookie(name string, val string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + cookies := chain.Request.Cookies() + hasCookie := false + for _, cookie := range cookies { + if cookie.Name != name { + continue + } + hasCookie = true + cookie.Value = val + } + + if hasCookie { + return nil + } + + chain.Request.AddCookie(&http.Cookie{ + Domain: chain.Request.URL.Host, + Name: name, + Value: val, + }) + + return nil + } +} + +// SetOutgoingCookies modifies a client request's cookie header +// to a raw Cookie string, overwriting existing cookies +func SetOutgoingCookies(cookies string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.Request.Header.Set("Cookies", cookies) + return nil + } +} + +// DeleteOutgoingCookie modifies the http request's cookies header to +// delete a specific request cookie going to the upstream server. +// If the cookie does not exist, it does not do anything. +func DeleteOutgoingCookie(name string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + cookies := chain.Request.Cookies() + chain.Request.Header.Del("Cookies") + + for _, cookie := range cookies { + if cookie.Name == name { + chain.Request.AddCookie(cookie) + } + } + return nil + } +} + +// DeleteOutgoingCookies removes the cookie header entirely, +// preventing any cookies from reaching the upstream server. +func DeleteOutgoingCookies() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Del("Cookie") + return nil + } +} + +// DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client +// to the upstream proxy server. Cookies whose names are in the whitelist are not removed. +func DeleteOutgoingCookiesExcept(whitelist ...string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + // Convert whitelist slice to a map for efficient lookups + whitelistMap := make(map[string]struct{}) + for _, cookieName := range whitelist { + whitelistMap[cookieName] = struct{}{} + } + + // Get all cookies from the request header + cookies := px.Request.Cookies() + + // Clear the original Cookie header + px.Request.Header.Del("Cookie") + + // Re-add cookies that are in the whitelist + for _, cookie := range cookies { + if _, found := whitelistMap[cookie.Name]; found { + px.Request.AddCookie(cookie) + } + } + + return nil + } +} diff --git a/proxychain/requestmodifiers/modify_path_with_regex.go b/proxychain/requestmodifiers/modify_path_with_regex.go new file mode 100644 index 00000000..2c0f20b3 --- /dev/null +++ b/proxychain/requestmodifiers/modify_path_with_regex.go @@ -0,0 +1,19 @@ +package requestmodifiers + +import ( + "fmt" + "regexp" + + "github.com/everywall/ladder/proxychain" +) + +func ModifyPathWithRegex(matchRegex string, replacement string) proxychain.RequestModification { + match, err := regexp.Compile(matchRegex) + return func(px *proxychain.ProxyChain) error { + if err != nil { + return fmt.Errorf("RequestModification :: ModifyPathWithRegex error => invalid match regex: %s - %s", matchRegex, err.Error()) + } + px.Request.URL.Path = match.ReplaceAllString(px.Request.URL.Path, replacement) + return nil + } +} diff --git a/proxychain/requestmodifiers/modify_query_params.go b/proxychain/requestmodifiers/modify_query_params.go new file mode 100644 index 00000000..681ce8b9 --- /dev/null +++ b/proxychain/requestmodifiers/modify_query_params.go @@ -0,0 +1,28 @@ +package requestmodifiers + +import ( + //"fmt" + "net/url" + + "github.com/everywall/ladder/proxychain" +) + +// ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. +// If the query param key doesn't exist, it is created. +func ModifyQueryParams(key string, value string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + q := chain.Request.URL.Query() + chain.Request.URL.RawQuery = modifyQueryParams(key, value, q) + //fmt.Println(chain.Request.URL.String()) + return nil + } +} + +func modifyQueryParams(key string, value string, q url.Values) string { + if value == "" { + q.Del(key) + return q.Encode() + } + q.Set(key, value) + return q.Encode() +} diff --git a/proxychain/requestmodifiers/modify_request_headers.go b/proxychain/requestmodifiers/modify_request_headers.go new file mode 100644 index 00000000..8938fa9f --- /dev/null +++ b/proxychain/requestmodifiers/modify_request_headers.go @@ -0,0 +1,23 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SetRequestHeader modifies a specific outgoing header +// This is the header that the upstream server will see. +func SetRequestHeader(name string, val string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set(name, val) + return nil + } +} + +// DeleteRequestHeader modifies a specific outgoing header +// This is the header that the upstream server will see. +func DeleteRequestHeader(name string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Del(name) + return nil + } +} diff --git a/proxychain/requestmodifiers/request_archive_is.go b/proxychain/requestmodifiers/request_archive_is.go new file mode 100644 index 00000000..625925a1 --- /dev/null +++ b/proxychain/requestmodifiers/request_archive_is.go @@ -0,0 +1,48 @@ +package requestmodifiers + +import ( + "fmt" + "net/url" + "regexp" + + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" +) + +const archivistUrl string = "https://archive.is/latest" + +// RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is +func RequestArchiveIs() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + rURL := preventRecursiveArchivistURLs(chain.Request.URL.String()) + chain.Request.URL.RawQuery = "" + newURL, err := url.Parse(fmt.Sprintf("%s/%s", archivistUrl, rURL)) + if err != nil { + return err + } + + // archivist seems to sabotage requests from cloudflare's DNS + // bypass this just in case + chain.AddOnceRequestModifications(ResolveWithGoogleDoH()) + + chain.Request.URL = newURL + + // cleanup archivst headers + script := `[...document.querySelector("body > center").childNodes].filter(e => e.id != "SOLID").forEach(e => e.remove())` + chain.AddOnceResponseModifications( + tx.InjectScriptAfterDOMContentLoaded(script), + ) + return nil + } +} + +// https://archive.is/20200421201055/https://rt.live/ -> http://rt.live/ +func preventRecursiveArchivistURLs(url string) string { + re := regexp.MustCompile(`https?:\/\/archive\.is\/\d+\/(https?:\/\/.*)`) + match := re.FindStringSubmatch(url) + if match != nil { + return match[1] + } + return url +} diff --git a/proxychain/requestmodifiers/request_google_cache.go b/proxychain/requestmodifiers/request_google_cache.go new file mode 100644 index 00000000..6baa1deb --- /dev/null +++ b/proxychain/requestmodifiers/request_google_cache.go @@ -0,0 +1,22 @@ +package requestmodifiers + +import ( + "net/url" + + "github.com/everywall/ladder/proxychain" +) + +const googleCacheUrl string = "https://webcache.googleusercontent.com/search?q=cache:" + +// RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version. +func RequestGoogleCache() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + encodedURL := url.QueryEscape(px.Request.URL.String()) + newURL, err := url.Parse(googleCacheUrl + encodedURL) + if err != nil { + return err + } + px.Request.URL = newURL + return nil + } +} diff --git a/proxychain/requestmodifiers/request_wayback_machine.go b/proxychain/requestmodifiers/request_wayback_machine.go new file mode 100644 index 00000000..81ead322 --- /dev/null +++ b/proxychain/requestmodifiers/request_wayback_machine.go @@ -0,0 +1,44 @@ +package requestmodifiers + +import ( + "net/url" + "regexp" + + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" +) + +const waybackUrl string = "https://web.archive.org/web/" + +// RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version. +func RequestWaybackMachine() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.Request.URL.RawQuery = "" + rURL := preventRecursiveWaybackURLs(chain.Request.URL.String()) + newURLString := waybackUrl + rURL + newURL, err := url.Parse(newURLString) + if err != nil { + return err + } + chain.Request.URL = newURL + + // cleanup wayback headers + script := `["wm-ipp-print", "wm-ipp-base"].forEach(id => { try { document.getElementById(id).remove() } catch{ } })` + chain.AddOnceResponseModifications( + tx.InjectScriptAfterDOMContentLoaded(script), + ) + + return nil + } +} + +func preventRecursiveWaybackURLs(url string) string { + re := regexp.MustCompile(`https:\/\/web\.archive\.org\/web\/\d+\/\*(https?:\/\/.*)`) + + match := re.FindStringSubmatch(url) + if match != nil { + return match[1] + } + return url +} diff --git a/proxychain/requestmodifiers/resolve_with_google_doh.go b/proxychain/requestmodifiers/resolve_with_google_doh.go new file mode 100644 index 00000000..0a698708 --- /dev/null +++ b/proxychain/requestmodifiers/resolve_with_google_doh.go @@ -0,0 +1,94 @@ +package requestmodifiers + +import ( + "context" + "encoding/json" + "fmt" + "net" + "time" + + http "github.com/bogdanfinn/fhttp" + + /* + tls_client "github.com/bogdanfinn/tls-client" + //"net/http" + */ + + "github.com/everywall/ladder/proxychain" +) + +// resolveWithGoogleDoH resolves DNS using Google's DNS-over-HTTPS +func resolveWithGoogleDoH(host string) (string, error) { + url := "https://dns.google/resolve?name=" + host + "&type=A" + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result struct { + Answer []struct { + Data string `json:"data"` + } `json:"Answer"` + } + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return "", err + } + + // Get the first A record + if len(result.Answer) > 0 { + return result.Answer[0].Data, nil + } + return "", fmt.Errorf("no DoH DNS record found for %s", host) +} + +type CustomDialer struct { + *net.Dialer +} + +func newCustomDialer(timeout, keepAlive time.Duration) *CustomDialer { + return &CustomDialer{ + Dialer: &net.Dialer{ + Timeout: timeout, + KeepAlive: keepAlive, + }, + } +} + +func (cd *CustomDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + port = "443" + } + + resolvedHost, err := resolveWithGoogleDoH(host) + if err != nil { + return nil, err + } + return cd.Dialer.DialContext(ctx, network, net.JoinHostPort(resolvedHost, port)) +} + +// ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL +// using Google's DNS over HTTPs service +func ResolveWithGoogleDoH() proxychain.RequestModification { + ///customDialer := NewCustomDialer(10*time.Second, 10*time.Second) + return func(chain *proxychain.ProxyChain) error { + /* + options := []tls_client.HttpClientOption{ + tls_client.WithTimeoutSeconds(30), + tls_client.WithRandomTLSExtensionOrder(), + tls_client.WithDialer(*customDialer.Dialer), + //tls_client.WithClientProfile(profiles.Chrome_105), + } + + client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) + if err != nil { + return err + } + + chain.SetOnceHTTPClient(client) + */ + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_origin.go b/proxychain/requestmodifiers/spoof_origin.go new file mode 100644 index 00000000..b871048f --- /dev/null +++ b/proxychain/requestmodifiers/spoof_origin.go @@ -0,0 +1,24 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofOrigin modifies the origin header +// if the upstream server returns a Vary header +// it means you might get a different response if you change this +func SpoofOrigin(url string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("origin", url) + return nil + } +} + +// HideOrigin modifies the origin header +// so that it is the original origin, not the proxy +func HideOrigin() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("origin", px.Request.URL.String()) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer.go b/proxychain/requestmodifiers/spoof_referrer.go new file mode 100644 index 00000000..4735bf3e --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer.go @@ -0,0 +1,39 @@ +package requestmodifiers + +import ( + "fmt" + + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrer modifies the referrer header. +// It is useful if the page can be accessed from a search engine +// or social media site, but not by browsing the website itself. +// if url is "", then the referrer header is removed. +func SpoofReferrer(url string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + // change refer on client side js + script := fmt.Sprintf(`document.referrer = "%s"`, url) + chain.AddOnceResponseModifications( + tx.InjectScriptBeforeDOMContentLoaded(script), + ) + + if url == "" { + chain.Request.Header.Del("referrer") + return nil + } + chain.Request.Header.Set("referrer", url) + return nil + } +} + +// HideReferrer modifies the referrer header +// so that it is the original referrer, not the proxy +func HideReferrer() proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("referrer", px.Request.URL.String()) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go b/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go new file mode 100644 index 00000000..aa9aaedd --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go @@ -0,0 +1,43 @@ +package requestmodifiers + +import ( + "fmt" + "math/rand" + "strings" + "time" + + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromBaiduSearch modifies the referrer header +// pretending to be from a BaiduSearch +func SpoofReferrerFromBaiduSearch() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + // https://www.baidu.com/link?url=5biIeDvUIihawf3Zbbysach2Xn4H3w3FzO6LZKgSs-B5Yt4M4RUFikokOk5zetf2&wd=&eqid=9da80d8208009b8480000706655d5ed6 + referrer := fmt.Sprintf("https://baidu.com/link?url=%s", generateRandomBaiduURL()) + chain.Request.Header.Set("referrer", referrer) + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + chain.Request.Header.Set("sec-fetch-mode", "navigate") + return nil + } +} + +// utility functions ================== + +func generateRandomString(charset string, length int) string { + var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano())) + var stringBuilder strings.Builder + for i := 0; i < length; i++ { + stringBuilder.WriteByte(charset[seededRand.Intn(len(charset))]) + } + return stringBuilder.String() +} + +func generateRandomBaiduURL() string { + const alphanumericCharset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + const hexCharset = "0123456789abcdef" + randomAlphanumeric := generateRandomString(alphanumericCharset, 30) // Length before "-" + randomHex := generateRandomString(hexCharset, 16) // Length of eqid + return randomAlphanumeric + "-" + "&wd=&eqid=" + randomHex +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go b/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go new file mode 100644 index 00000000..25c01a53 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go @@ -0,0 +1,20 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromBingSearch modifies the referrer header +// pretending to be from a bing search site +func SpoofReferrerFromBingSearch() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://www.bing.com/"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ModifyQueryParams("utm_source", "bing"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_google_search.go b/proxychain/requestmodifiers/spoof_referrer_from_google_search.go new file mode 100644 index 00000000..0d8b5fd9 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_google_search.go @@ -0,0 +1,20 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromGoogleSearch modifies the referrer header +// pretending to be from a google search site +func SpoofReferrerFromGoogleSearch() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://www.google.com"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ModifyQueryParams("utm_source", "google"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go b/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go new file mode 100644 index 00000000..7def19cf --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go @@ -0,0 +1,21 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromLinkedInPost modifies the referrer header +// pretending to be from a linkedin post +func SpoofReferrerFromLinkedInPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://www.linkedin.com/"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ModifyQueryParams("utm_campaign", "post"), + ModifyQueryParams("utm_medium", "web"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go b/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go new file mode 100644 index 00000000..17d765f0 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go @@ -0,0 +1,23 @@ +package requestmodifiers + +import ( + "fmt" + + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromNaverSearch modifies the referrer header +// pretending to be from a Naver search (popular in South Korea) +func SpoofReferrerFromNaverSearch() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + referrer := fmt.Sprintf( + "https://search.naver.com/search.naver?where=nexearch&sm=top_hty&fbm=0&ie=utf8&query=%s", + chain.Request.URL.Host, + ) + chain.Request.Header.Set("referrer", referrer) + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + chain.Request.Header.Set("sec-fetch-mode", "navigate") + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go b/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go new file mode 100644 index 00000000..929a26d0 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go @@ -0,0 +1,17 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromPinterestPost modifies the referrer header +// pretending to be from a pinterest post +func SpoofReferrerFromPinterestPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.Request.Header.Set("referrer", "https://www.pinterest.com/") + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + chain.Request.Header.Set("sec-fetch-mode", "navigate") + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go b/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go new file mode 100644 index 00000000..6892838d --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go @@ -0,0 +1,16 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromQQPost modifies the referrer header +// pretending to be from a QQ post (popular social media in China) +func SpoofReferrerFromQQPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.Request.Header.Set("referrer", "https://new.qq.com/") + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go b/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go new file mode 100644 index 00000000..31c88007 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go @@ -0,0 +1,17 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromRedditPost modifies the referrer header +// pretending to be from a reddit post +func SpoofReferrerFromRedditPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.Request.Header.Set("referrer", "https://www.reddit.com/") + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + chain.Request.Header.Set("sec-fetch-mode", "navigate") + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go b/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go new file mode 100644 index 00000000..4860097b --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go @@ -0,0 +1,19 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromTumblrPost modifies the referrer header +// pretending to be from a tumblr post +func SpoofReferrerFromTumblrPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://www.tumblr.com/"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go b/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go new file mode 100644 index 00000000..eeff10c6 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go @@ -0,0 +1,19 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromTwitterPost modifies the referrer header +// pretending to be from a twitter post +func SpoofReferrerFromTwitterPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://t.co/"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go b/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go new file mode 100644 index 00000000..ddc96279 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go @@ -0,0 +1,19 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromVkontaktePost modifies the referrer header +// pretending to be from a vkontakte post (popular in Russia) +func SpoofReferrerFromVkontaktePost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceRequestModifications( + SpoofReferrer("https://away.vk.com/"), + SetRequestHeader("sec-fetch-site", "cross-site"), + SetRequestHeader("sec-fetch-dest", "document"), + SetRequestHeader("sec-fetch-mode", "navigate"), + ) + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go b/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go new file mode 100644 index 00000000..d8453905 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go @@ -0,0 +1,21 @@ +package requestmodifiers + +import ( + "fmt" + "math/rand" + + "github.com/everywall/ladder/proxychain" +) + +// SpoofReferrerFromWeiboPost modifies the referrer header +// pretending to be from a Weibo post (popular in China) +func SpoofReferrerFromWeiboPost() proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + referrer := fmt.Sprintf("http://weibo.com/u/%d", rand.Intn(90001)) + chain.Request.Header.Set("referrer", referrer) + chain.Request.Header.Set("sec-fetch-site", "cross-site") + chain.Request.Header.Set("sec-fetch-dest", "document") + chain.Request.Header.Set("sec-fetch-mode", "navigate") + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_user_agent.go b/proxychain/requestmodifiers/spoof_user_agent.go new file mode 100644 index 00000000..e902f9ad --- /dev/null +++ b/proxychain/requestmodifiers/spoof_user_agent.go @@ -0,0 +1,41 @@ +package requestmodifiers + +import ( + _ "embed" + "strings" + + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" +) + +// https://github.com/faisalman/ua-parser-js/tree/master +// update using: +// git submodule update --remote --merge +// +//go:embed vendor/ua-parser-js/dist/ua-parser.min.js +var UAParserJS string + +// note: spoof_user_agent.js has a dependency on ua-parser.min.js +// ua-parser.min.js should be loaded first. +// +//go:embed spoof_user_agent.js +var spoofUserAgentJS string + +// SpoofUserAgent modifies the user agent +func SpoofUserAgent(ua string) proxychain.RequestModification { + return func(chain *proxychain.ProxyChain) error { + // modify ua headers + chain.AddOnceRequestModifications( + SetRequestHeader("user-agent", ua), + ) + + script := strings.ReplaceAll(spoofUserAgentJS, "{{USER_AGENT}}", ua) + chain.AddOnceResponseModifications( + tx.InjectScriptBeforeDOMContentLoaded(script), + tx.InjectScriptBeforeDOMContentLoaded(UAParserJS), + ) + + return nil + } +} diff --git a/proxychain/requestmodifiers/spoof_user_agent.js b/proxychain/requestmodifiers/spoof_user_agent.js new file mode 100644 index 00000000..d37b93d3 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_user_agent.js @@ -0,0 +1,100 @@ +(() => { + const UA = "{{USER_AGENT}}"; + + // monkey-patch navigator.userAgent + { + const { get } = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "userAgent", + ); + Object.defineProperty(Navigator.prototype, "userAgent", { + get: new Proxy(get, { + apply() { + return UA; + }, + }), + }); + } + + // monkey-patch navigator.appVersion + { + const { get } = Object.getOwnPropertyDescriptor( + Navigator.prototype, + "appVersion", + ); + Object.defineProperty(Navigator.prototype, "appVersion", { + get: new Proxy(get, { + apply() { + return UA.replace("Mozilla/", ""); + }, + }), + }); + } + + // monkey-patch navigator.UserAgentData + // Assuming UAParser is already loaded and available + function spoofUserAgentData(uaString) { + // Parse the user-agent string + const parser = new UAParser(uaString); + const parsedData = parser.getResult(); + + // Extracted data + const platform = parsedData.os.name; + const browserName = parsedData.browser.name; + const browserMajorVersion = parsedData.browser.major; + const isMobile = + /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test( + uaString, + ); + + // Overwrite navigator.userAgentData + self.NavigatorUAData = self.NavigatorUAData || new class NavigatorUAData { + brands = [{ + brand: browserName, + version: browserMajorVersion, + }]; + mobile = isMobile; + platform = platform; + toJSON() { + return { + brands: this.brands, + mobile: this.mobile, + platform: this.platform, + }; + } + getHighEntropyValues(hints) { + const result = this.toJSON(); + // Add additional high entropy values based on hints + // Modify these as per your requirements + if (hints.includes("architecture")) { + result.architecture = "x86"; + } + if (hints.includes("bitness")) { + result.bitness = "64"; + } + if (hints.includes("model")) { + result.model = ""; + } + if (hints.includes("platformVersion")) { + result.platformVersion = "10.0.0"; // Example value + } + if (hints.includes("uaFullVersion")) { + result.uaFullVersion = browserMajorVersion; + } + if (hints.includes("fullVersionList")) { + result.fullVersionList = this.brands; + } + return Promise.resolve(result); + } + }(); + + // Apply the monkey patch + Object.defineProperty(navigator, "userAgentData", { + value: new self.NavigatorUAData(), + writable: false, + }); + } + + spoofUserAgentData(UA); + // TODO: use hideMonkeyPatch to hide overrides +})(); diff --git a/proxychain/requestmodifiers/spoof_x_forwarded_for.go b/proxychain/requestmodifiers/spoof_x_forwarded_for.go new file mode 100644 index 00000000..c7762bf4 --- /dev/null +++ b/proxychain/requestmodifiers/spoof_x_forwarded_for.go @@ -0,0 +1,14 @@ +package requestmodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// SpoofXForwardedFor modifies the X-Forwarded-For header +// in some cases, a forward proxy may interpret this as the source IP +func SpoofXForwardedFor(ip string) proxychain.RequestModification { + return func(px *proxychain.ProxyChain) error { + px.Request.Header.Set("X-FORWARDED-FOR", ip) + return nil + } +} diff --git a/proxychain/requestmodifiers/vendor/ua-parser-js b/proxychain/requestmodifiers/vendor/ua-parser-js new file mode 160000 index 00000000..3622b614 --- /dev/null +++ b/proxychain/requestmodifiers/vendor/ua-parser-js @@ -0,0 +1 @@ +Subproject commit 3622b614a71749e6d96f241d6810d6086075e2a4 diff --git a/proxychain/responsemodifiers/api/error_api.go b/proxychain/responsemodifiers/api/error_api.go new file mode 100644 index 00000000..333564fb --- /dev/null +++ b/proxychain/responsemodifiers/api/error_api.go @@ -0,0 +1,56 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "reflect" +) + +type Error struct { + Success bool `json:"success"` + Error ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Message string `json:"message"` + Type string `json:"type"` + Cause string `json:"cause"` +} + +func CreateAPIErrReader(err error) io.ReadCloser { + if err == nil { + return io.NopCloser(bytes.NewBufferString(`{"success":false, "error": "No error provided"}`)) + } + + baseErr := getBaseError(err) + apiErr := Error{ + Success: false, + Error: ErrorDetails{ + Message: err.Error(), + Type: reflect.TypeOf(err).String(), + Cause: baseErr.Error(), + }, + } + + // Serialize the APIError into JSON + jsonData, jsonErr := json.Marshal(apiErr) + if jsonErr != nil { + return io.NopCloser(bytes.NewBufferString(`{"success":false, "error": "Failed to serialize error"}`)) + } + + // Return the JSON data as an io.ReadCloser + return io.NopCloser(bytes.NewBuffer(jsonData)) +} + +func getBaseError(err error) error { + for { + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + return err + } + + err = unwrapped + } +} diff --git a/proxychain/responsemodifiers/api/outline_api.go b/proxychain/responsemodifiers/api/outline_api.go new file mode 100644 index 00000000..f17f42e5 --- /dev/null +++ b/proxychain/responsemodifiers/api/outline_api.go @@ -0,0 +1,174 @@ +package api + +import ( + "github.com/go-shiori/dom" + "github.com/markusmobius/go-trafilatura" + "golang.org/x/net/html" +) + +// ======================================================================================= +// credit @joncrangle https://github.com/everywall/ladder/issues/38#issuecomment-1831252934 + +type ImageContent struct { + Type string `json:"type"` + URL string `json:"url"` + Alt string `json:"alt"` + Caption string `json:"caption"` +} + +type LinkContent struct { + Type string `json:"type"` + Href string `json:"href"` + Data string `json:"data"` +} + +type TextContent struct { + Type string `json:"type"` + Data string `json:"data"` +} + +type ListContent struct { + Type string `json:"type"` + ListItems []ListItemContent `json:"listItems"` +} + +type ListItemContent struct { + Data string `json:"data"` +} + +type JSONDocument struct { + Success bool `json:"success"` + Error ErrorDetails `json:"error"` + Metadata struct { + Title string `json:"title"` + Author string `json:"author"` + URL string `json:"url"` + Hostname string `json:"hostname"` + Image string `json:"image"` + Description string `json:"description"` + Sitename string `json:"sitename"` + Date string `json:"date"` + Categories []string `json:"categories"` + Tags []string `json:"tags"` + License string `json:"license"` + } `json:"metadata"` + Content []interface{} `json:"content"` + Comments string `json:"comments"` +} + +func ExtractResultToAPIResponse(extract *trafilatura.ExtractResult) *JSONDocument { + jsonDoc := &JSONDocument{} + + // Populate success + jsonDoc.Success = true + + // Populate metadata + jsonDoc.Metadata.Title = extract.Metadata.Title + jsonDoc.Metadata.Author = extract.Metadata.Author + jsonDoc.Metadata.URL = extract.Metadata.URL + jsonDoc.Metadata.Hostname = extract.Metadata.Hostname + jsonDoc.Metadata.Description = extract.Metadata.Description + jsonDoc.Metadata.Image = extract.Metadata.Image + jsonDoc.Metadata.Sitename = extract.Metadata.Sitename + jsonDoc.Metadata.Date = extract.Metadata.Date.Format("2006-01-02") + jsonDoc.Metadata.Categories = extract.Metadata.Categories + jsonDoc.Metadata.Tags = extract.Metadata.Tags + jsonDoc.Metadata.License = extract.Metadata.License + jsonDoc.Metadata.Hostname = extract.Metadata.Hostname + + // Populate content + if extract.ContentNode != nil { + jsonDoc.Content = parseContent(extract.ContentNode) + } + + // Populate comments + if extract.CommentsNode != nil { + jsonDoc.Comments = dom.OuterHTML(extract.CommentsNode) + } + + return jsonDoc +} + +func parseContent(node *html.Node) []interface{} { + var content []interface{} + + for child := node.FirstChild; child != nil; child = child.NextSibling { + switch child.Data { + case "img": + image := ImageContent{ + Type: "img", + URL: dom.GetAttribute(child, "src"), + Alt: dom.GetAttribute(child, "alt"), + Caption: dom.GetAttribute(child, "caption"), + } + content = append(content, image) + + case "a": + link := LinkContent{ + Type: "a", + Href: dom.GetAttribute(child, "href"), + Data: dom.InnerText(child), + } + content = append(content, link) + + case "h1": + text := TextContent{ + Type: "h1", + Data: dom.InnerText(child), + } + content = append(content, text) + + case "h2": + text := TextContent{ + Type: "h2", + Data: dom.InnerText(child), + } + content = append(content, text) + + case "h3": + text := TextContent{ + Type: "h3", + Data: dom.InnerText(child), + } + content = append(content, text) + + case "h4": + text := TextContent{ + Type: "h4", + Data: dom.InnerText(child), + } + content = append(content, text) + + case "h5": + text := TextContent{ + Type: "h5", + Data: dom.InnerText(child), + } + content = append(content, text) + + case "ul", "ol": + list := ListContent{ + Type: child.Data, + ListItems: []ListItemContent{}, + } + for listItem := child.FirstChild; listItem != nil; listItem = listItem.NextSibling { + if listItem.Data == "li" { + listItemContent := ListItemContent{ + Data: dom.InnerText(listItem), + } + list.ListItems = append(list.ListItems, listItemContent) + } + } + content = append(content, list) + + default: + text := TextContent{ + Type: "p", + Data: dom.InnerText(child), + } + content = append(content, text) + } + } + + return content +} diff --git a/proxychain/responsemodifiers/api_content.go b/proxychain/responsemodifiers/api_content.go new file mode 100644 index 00000000..c715bf3e --- /dev/null +++ b/proxychain/responsemodifiers/api_content.go @@ -0,0 +1,48 @@ +package responsemodifiers + +import ( + "bytes" + "encoding/json" + "io" + + "github.com/markusmobius/go-trafilatura" + + "github.com/everywall/ladder/proxychain/responsemodifiers/api" + + "github.com/everywall/ladder/proxychain" +) + +// APIContent creates an JSON representation of the article and returns it as an API response. +func APIContent() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // we set content-type twice here, in case another response modifier + // tries to forward over the original headers + chain.Context.Set("content-type", "application/json") + chain.Response.Header.Set("content-type", "application/json") + + // extract dom contents + opts := trafilatura.Options{ + IncludeImages: true, + IncludeLinks: true, + // FavorPrecision: true, + FallbackCandidates: nil, // TODO: https://github.com/markusmobius/go-trafilatura/blob/main/examples/chained/main.go + // implement fallbacks from "github.com/markusmobius/go-domdistiller" and "github.com/go-shiori/go-readability" + OriginalURL: chain.Request.URL, + } + + result, err := trafilatura.Extract(chain.Response.Body, opts) + if err != nil { + chain.Response.Body = api.CreateAPIErrReader(err) + return nil + } + + res := api.ExtractResultToAPIResponse(result) + jsonData, err := json.MarshalIndent(res, "", " ") + if err != nil { + return err + } + + chain.Response.Body = io.NopCloser(bytes.NewReader(jsonData)) + return nil + } +} diff --git a/proxychain/responsemodifiers/api_content_test.go b/proxychain/responsemodifiers/api_content_test.go new file mode 100644 index 00000000..9c60c576 --- /dev/null +++ b/proxychain/responsemodifiers/api_content_test.go @@ -0,0 +1,70 @@ +package responsemodifiers + +import ( + "encoding/json" + "fmt" + "io" + "net/url" + "testing" + + "github.com/everywall/ladder/proxychain/responsemodifiers/api" +) + +func TestCreateAPIErrReader(t *testing.T) { + _, baseErr := url.Parse("://this is an invalid url") + wrappedErr := fmt.Errorf("wrapped error: %w", baseErr) + + readCloser := api.CreateAPIErrReader(wrappedErr) + defer readCloser.Close() + + // Read and unmarshal the JSON output + data, err := io.ReadAll(readCloser) + if err != nil { + t.Fatalf("Failed to read from ReadCloser: %v", err) + } + fmt.Println(string(data)) + + var apiErr api.Error + err = json.Unmarshal(data, &apiErr) + if err != nil { + t.Fatalf("Failed to unmarshal JSON: %v", err) + } + + // Verify the structure of the APIError + if apiErr.Success { + t.Errorf("Expected Success to be false, got true") + } + + if apiErr.Error.Message != wrappedErr.Error() { + t.Errorf("Expected error message to be '%v', got '%v'", wrappedErr.Error(), apiErr.Error.Message) + } +} + +func TestCreateAPIErrReader2(t *testing.T) { + _, baseErr := url.Parse("://this is an invalid url") + + readCloser := api.CreateAPIErrReader(baseErr) + defer readCloser.Close() + + // Read and unmarshal the JSON output + data, err := io.ReadAll(readCloser) + if err != nil { + t.Fatalf("Failed to read from ReadCloser: %v", err) + } + fmt.Println(string(data)) + + var apiErr api.Error + err = json.Unmarshal(data, &apiErr) + if err != nil { + t.Fatalf("Failed to unmarshal JSON: %v", err) + } + + // Verify the structure of the APIError + if apiErr.Success { + t.Errorf("Expected Success to be false, got true") + } + + if apiErr.Error.Message != baseErr.Error() { + t.Errorf("Expected error message to be '%v', got '%v'", baseErr.Error(), apiErr.Error.Message) + } +} diff --git a/proxychain/responsemodifiers/block_element_removal.go b/proxychain/responsemodifiers/block_element_removal.go new file mode 100644 index 00000000..7dd8b767 --- /dev/null +++ b/proxychain/responsemodifiers/block_element_removal.go @@ -0,0 +1,43 @@ +package responsemodifiers + +import ( + _ "embed" + "strings" + + "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters" + + "github.com/everywall/ladder/proxychain" +) + +//go:embed vendor/block_element_removal.js +var blockElementRemoval string + +// BlockElementRemoval prevents paywall javascript from removing a +// particular element by detecting the removal, then immediately reinserting it. +// This is useful when a page will return a "fake" 404, after flashing the content briefly. +// If the /outline/ API works, but the regular API doesn't, try this modifier. +func BlockElementRemoval(cssSelector string) proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // don't add rewriter if it's not even html + ct := chain.Response.Header.Get("content-type") + if !strings.HasPrefix(ct, "text/html") { + return nil + } + + params := map[string]string{ + // ie: "div.article-content" + "{{CSS_SELECTOR}}": cssSelector, + } + + rr := rewriters.NewScriptInjectorRewriterWithParams( + blockElementRemoval, + rewriters.BeforeDOMContentLoaded, + params, + ) + + htmlRewriter := rewriters.NewHTMLRewriter(chain.Response.Body, rr) + chain.Response.Body = htmlRewriter + + return nil + } +} diff --git a/proxychain/responsemodifiers/block_third_party_scripts.go b/proxychain/responsemodifiers/block_third_party_scripts.go new file mode 100644 index 00000000..9af1d0c3 --- /dev/null +++ b/proxychain/responsemodifiers/block_third_party_scripts.go @@ -0,0 +1,34 @@ +package responsemodifiers + +import ( + _ "embed" + "fmt" + "strings" + + "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters" + + "github.com/everywall/ladder/proxychain" +) + +// BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading. +func BlockThirdPartyScripts() proxychain.ResponseModification { + // TODO: monkey patch fetch and XMLHttpRequest to firewall 3P JS as well. + return func(chain *proxychain.ProxyChain) error { + // don't add rewriter if it's not even html + ct := chain.Response.Header.Get("content-type") + if !strings.HasPrefix(ct, "text/html") { + return nil + } + + // proxyURL is the URL of the ladder: http://localhost:8080 (ladder) + originalURI := chain.Context.Request().URI() + proxyURL := fmt.Sprintf("%s://%s", originalURI.Scheme(), originalURI.Host()) + + // replace http.Response.Body with a readcloser that wraps the original, modifying the html attributes + rr := rewriters.NewBlockThirdPartyScriptsRewriter(chain.Request.URL, proxyURL) + blockJSRewriter := rewriters.NewHTMLRewriter(chain.Response.Body, rr) + chain.Response.Body = blockJSRewriter + + return nil + } +} diff --git a/proxychain/responsemodifiers/bypass_cors.go b/proxychain/responsemodifiers/bypass_cors.go new file mode 100644 index 00000000..4a02c4eb --- /dev/null +++ b/proxychain/responsemodifiers/bypass_cors.go @@ -0,0 +1,21 @@ +package responsemodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// BypassCORS modifies response headers to prevent the browser +// from enforcing any CORS restrictions. This should run at the end of the chain. +func BypassCORS() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceResponseModifications( + SetResponseHeader("Access-Control-Allow-Origin", "*"), + SetResponseHeader("Access-Control-Expose-Headers", "*"), + SetResponseHeader("Access-Control-Allow-Credentials", "true"), + SetResponseHeader("Access-Control-Allow-Methods", "GET, PUT, POST, DELETE, HEAD, OPTIONS, PATCH"), + SetResponseHeader("Access-Control-Allow-Headers", "*"), + DeleteResponseHeader("X-Frame-Options"), + ) + return nil + } +} diff --git a/proxychain/responsemodifiers/bypass_csp.go b/proxychain/responsemodifiers/bypass_csp.go new file mode 100644 index 00000000..90e1da85 --- /dev/null +++ b/proxychain/responsemodifiers/bypass_csp.go @@ -0,0 +1,30 @@ +package responsemodifiers + +import ( + "github.com/everywall/ladder/proxychain" +) + +// TODO: handle edge case where CSP is specified in meta tag: +// + +// BypassContentSecurityPolicy modifies response headers to prevent the browser +// from enforcing any CSP restrictions. This should run at the end of the chain. +func BypassContentSecurityPolicy() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + chain.AddOnceResponseModifications( + DeleteResponseHeader("Content-Security-Policy"), + DeleteResponseHeader("Content-Security-Policy-Report-Only"), + DeleteResponseHeader("X-Content-Security-Policy"), + DeleteResponseHeader("X-WebKit-CSP"), + ) + return nil + } +} + +// SetContentSecurityPolicy modifies response headers to a specific CSP +func SetContentSecurityPolicy(csp string) proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + chain.Response.Header.Set("Content-Security-Policy", csp) + return nil + } +} diff --git a/proxychain/responsemodifiers/delete_localstorage_data.go b/proxychain/responsemodifiers/delete_localstorage_data.go new file mode 100644 index 00000000..6268ef22 --- /dev/null +++ b/proxychain/responsemodifiers/delete_localstorage_data.go @@ -0,0 +1,28 @@ +package responsemodifiers + +import ( + _ "embed" + "strings" + + "github.com/everywall/ladder/proxychain" +) + +// DeleteLocalStorageData deletes localstorage cookies. +// If the page works once in a fresh incognito window, but fails +// for subsequent loads, try this response modifier alongside +// DeleteSessionStorageData and DeleteIncomingCookies +func DeleteLocalStorageData() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // don't add rewriter if it's not even html + ct := chain.Response.Header.Get("content-type") + if !strings.HasPrefix(ct, "text/html") { + return nil + } + + chain.AddOnceResponseModifications( + InjectScriptBeforeDOMContentLoaded(`window.sessionStorage.clear()`), + InjectScriptAfterDOMContentLoaded(`window.sessionStorage.clear()`), + ) + return nil + } +} diff --git a/proxychain/responsemodifiers/delete_sessionstorage_data.go b/proxychain/responsemodifiers/delete_sessionstorage_data.go new file mode 100644 index 00000000..b55e20c2 --- /dev/null +++ b/proxychain/responsemodifiers/delete_sessionstorage_data.go @@ -0,0 +1,28 @@ +package responsemodifiers + +import ( + _ "embed" + "strings" + + "github.com/everywall/ladder/proxychain" +) + +// DeleteSessionStorageData deletes localstorage cookies. +// If the page works once in a fresh incognito window, but fails +// for subsequent loads, try this response modifier alongside +// DeleteLocalStorageData and DeleteIncomingCookies +func DeleteSessionStorageData() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // don't add rewriter if it's not even html + ct := chain.Response.Header.Get("content-type") + if !strings.HasPrefix(ct, "text/html") { + return nil + } + + chain.AddOnceResponseModifications( + InjectScriptBeforeDOMContentLoaded(`window.sessionStorage.clear()`), + InjectScriptAfterDOMContentLoaded(`window.sessionStorage.clear()`), + ) + return nil + } +} diff --git a/proxychain/responsemodifiers/forward_response_headers.go b/proxychain/responsemodifiers/forward_response_headers.go new file mode 100644 index 00000000..8f36ca94 --- /dev/null +++ b/proxychain/responsemodifiers/forward_response_headers.go @@ -0,0 +1,53 @@ +package responsemodifiers + +import ( + "fmt" + "net/url" + "strings" + + "github.com/everywall/ladder/proxychain" +) + +var forwardBlacklist map[string]bool + +func init() { + forwardBlacklist = map[string]bool{ + "content-length": true, + "content-encoding": true, + "transfer-encoding": true, + "strict-transport-security": true, + "connection": true, + "keep-alive": true, + } +} + +// ForwardResponseHeaders forwards the response headers from the upstream server to the client +func ForwardResponseHeaders() proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // fmt.Println(chain.Response.Header) + for uname, headers := range chain.Response.Header { + name := strings.ToLower(uname) + if forwardBlacklist[name] { + continue + } + + // patch location header to forward to proxy instead + if name == "location" { + u, err := url.Parse(chain.Context.BaseURL()) + if err != nil { + return err + } + newLocation := fmt.Sprintf("%s://%s/%s", u.Scheme, u.Host, headers[0]) + chain.Context.Set("location", newLocation) + } + + // forward headers + for _, value := range headers { + fmt.Println(name, value) + chain.Context.Set(name, value) + } + } + + return nil + } +} diff --git a/proxychain/responsemodifiers/generate_readable_outline.go b/proxychain/responsemodifiers/generate_readable_outline.go new file mode 100644 index 00000000..21033ef1 --- /dev/null +++ b/proxychain/responsemodifiers/generate_readable_outline.go @@ -0,0 +1,269 @@ +package responsemodifiers + +import ( + "bytes" + "embed" + "fmt" + "html/template" + "io" + "log" + "math" + "net/url" + "strings" + "time" + + "github.com/everywall/ladder/proxychain" + "github.com/markusmobius/go-trafilatura" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +//go:embed vendor/generate_readable_outline.html +var templateFS embed.FS + +// GenerateReadableOutline creates an reader-friendly distilled representation of the article. +// This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM. +func GenerateReadableOutline() proxychain.ResponseModification { + // get template only once, and resuse for subsequent calls + f := "vendor/generate_readable_outline.html" + tmpl, err := template.ParseFS(templateFS, f) + if err != nil { + panic(fmt.Errorf("tx.GenerateReadableOutline Error: %s not found", f)) + } + + return func(chain *proxychain.ProxyChain) error { + // =========================================================== + // 1. extract dom contents using reading mode algo + // =========================================================== + opts := trafilatura.Options{ + IncludeImages: false, + IncludeLinks: true, + FavorRecall: true, + Deduplicate: true, + FallbackCandidates: nil, // TODO: https://github.com/markusmobius/go-trafilatura/blob/main/examples/chained/main.go + // implement fallbacks from "github.com/markusmobius/go-domdistiller" and "github.com/go-shiori/go-readability" + OriginalURL: chain.Request.URL, + } + + extract, err := trafilatura.Extract(chain.Response.Body, opts) + if err != nil { + return err + } + + // ============================================================================ + // 2. render generate_readable_outline.html template using metadata from step 1 + // ============================================================================ + + // render DOM to string without H1 title + removeFirstH1(extract.ContentNode) + // rewrite all links to stay on /outline/ path + rewriteHrefLinks(extract.ContentNode, chain.Context.BaseURL(), chain.APIPrefix) + var b bytes.Buffer + html.Render(&b, extract.ContentNode) + distilledHTML := b.String() + + siteName := strings.Split(extract.Metadata.Sitename, ";")[0] + title := strings.Split(extract.Metadata.Title, "|")[0] + fmtDate := createWikipediaDateLink(extract.Metadata.Date) + readingTime := formatDuration(estimateReadingTime(extract.ContentText)) + + // populate template parameters + data := map[string]interface{}{ + "Success": true, + "Image": extract.Metadata.Image, + "Description": extract.Metadata.Description, + "Sitename": siteName, + "Hostname": extract.Metadata.Hostname, + "Url": "/" + chain.Request.URL.String(), + "Title": title, + "Date": fmtDate, + "Author": createDDGFeelingLuckyLinks(extract.Metadata.Author, extract.Metadata.Hostname), + "Body": distilledHTML, + "ReadingTime": readingTime, + } + + // ============================================================================ + // 3. queue sending the response back to the client by replacing the response body + // (the response body will be read as a stream in proxychain.Execute() later on.) + // ============================================================================ + pr, pw := io.Pipe() // pipe io.writer contents into io.reader + + // Use a goroutine for writing to the pipe so we don't deadlock the request + go func() { + defer pw.Close() + + err := tmpl.Execute(pw, data) // <- render template + if err != nil { + log.Printf("WARN: GenerateReadableOutline template rendering error: %s\n", err) + } + }() + + chain.Context.Set("content-type", "text/html") + chain.Response.Body = pr // <- replace response body reader with our new reader from pipe + return nil + } +} + +// ============================================= +// DOM Rendering helpers +// ============================================= + +func removeFirstH1(n *html.Node) { + var recurse func(*html.Node) bool + recurse = func(n *html.Node) bool { + if n.Type == html.ElementNode && n.DataAtom == atom.H1 { + return true // Found the first H1, return true to stop + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if recurse(c) { + n.RemoveChild(c) + return false // Removed first H1, no need to continue + } + } + return false + } + recurse(n) +} + +func rewriteHrefLinks(n *html.Node, baseURL string, apiPath string) { + u, err := url.Parse(baseURL) + if err != nil { + log.Printf("GenerateReadableOutline :: rewriteHrefLinks error - %s\n", err) + } + apiPath = strings.Trim(apiPath, "/") + proxyURL := fmt.Sprintf("%s://%s", u.Scheme, u.Host) + newProxyURL := fmt.Sprintf("%s/%s", proxyURL, apiPath) + + var recurse func(*html.Node) bool + recurse = func(n *html.Node) bool { + if n.Type == html.ElementNode && n.DataAtom == atom.A { + for i := range n.Attr { + attr := n.Attr[i] + if attr.Key != "href" { + continue + } + // rewrite url on a.href: http://localhost:8080/https://example.com -> http://localhost:8080/outline/https://example.com + attr.Val = strings.Replace(attr.Val, proxyURL, newProxyURL, 1) + // rewrite relative URLs too + if strings.HasPrefix(attr.Val, "/") { + attr.Val = fmt.Sprintf("/%s%s", apiPath, attr.Val) + } + n.Attr[i].Val = attr.Val + log.Println(attr.Val) + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + recurse(c) + } + return false + } + recurse(n) +} + +// createWikipediaDateLink takes in a date +// and returns an link pointing to the current events page for that day +func createWikipediaDateLink(t time.Time) string { + url := fmt.Sprintf("https://en.wikipedia.org/wiki/Portal:Current_events#%s", t.Format("2006_January_02")) + date := t.Format("January 02, 2006") + return fmt.Sprintf("%s", url, date) +} + +// createDDGFeelingLuckyLinks takes in comma or semicolon separated terms, +// then turns them into links searching for the term using DuckDuckGo's I'm +// feeling lucky feature. It will redirect the user immediately to the first search result. +func createDDGFeelingLuckyLinks(searchTerms string, siteHostname string) string { + + siteHostname = strings.TrimSpace(siteHostname) + semiColonSplit := strings.Split(searchTerms, ";") + + var links []string + for i, termGroup := range semiColonSplit { + commaSplit := strings.Split(termGroup, ",") + for _, term := range commaSplit { + trimmedTerm := strings.TrimSpace(term) + if trimmedTerm == "" { + continue + } + + ddgQuery := fmt.Sprintf(` site:%s intitle:"%s"`, strings.TrimPrefix(siteHostname, "www."), trimmedTerm) + + encodedTerm := `\%s:` + url.QueryEscape(ddgQuery) + //ddgURL := `https://html.duckduckgo.com/html/?q=` + encodedTerm + ddgURL := `https://www.duckduckgo.com/?q=` + encodedTerm + + link := fmt.Sprintf("%s", ddgURL, trimmedTerm) + links = append(links, link) + } + + // If it's not the last element in semiColonSplit, add a comma to the last link + if i < len(semiColonSplit)-1 { + links[len(links)-1] = links[len(links)-1] + "," + } + } + + return strings.Join(links, " ") +} + +// estimateReadingTime estimates how long the given text will take to read using the given configuration. +func estimateReadingTime(text string) time.Duration { + if len(text) == 0 { + return 0 + } + + // Init options with default values. + WordsPerMinute := 200 + WordBound := func(b byte) bool { + return b == ' ' || b == '\n' || b == '\r' || b == '\t' + } + + words := 0 + start := 0 + end := len(text) - 1 + + // Fetch bounds. + for WordBound(text[start]) { + start++ + } + for WordBound(text[end]) { + end-- + } + + // Calculate the number of words. + for i := start; i <= end; { + for i <= end && !WordBound(text[i]) { + i++ + } + + words++ + + for i <= end && WordBound(text[i]) { + i++ + } + } + + // Reading time stats. + minutes := math.Ceil(float64(words) / float64(WordsPerMinute)) + duration := time.Duration(math.Ceil(minutes) * float64(time.Minute)) + + return duration + +} + +func formatDuration(d time.Duration) string { + // Check if the duration is less than one minute + if d < time.Minute { + seconds := int(d.Seconds()) + return fmt.Sprintf("%d seconds", seconds) + } + + // Convert the duration to minutes + minutes := int(d.Minutes()) + + // Format the string for one or more minutes + if minutes == 1 { + return "1 minute" + } else { + return fmt.Sprintf("%d minutes", minutes) + } +} diff --git a/proxychain/responsemodifiers/inject_script.go b/proxychain/responsemodifiers/inject_script.go new file mode 100644 index 00000000..6383b495 --- /dev/null +++ b/proxychain/responsemodifiers/inject_script.go @@ -0,0 +1,42 @@ +package responsemodifiers + +import ( + _ "embed" + "strings" + + "github.com/everywall/ladder/proxychain/responsemodifiers/rewriters" + + "github.com/everywall/ladder/proxychain" +) + +// injectScript modifies HTTP responses +// to execute javascript at a particular time. +func injectScript(js string, execTime rewriters.ScriptExecTime) proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + // don't add rewriter if it's not even html + ct := chain.Response.Header.Get("content-type") + if !strings.HasPrefix(ct, "text/html") { + return nil + } + + rr := rewriters.NewScriptInjectorRewriter(js, execTime) + htmlRewriter := rewriters.NewHTMLRewriter(chain.Response.Body, rr) + chain.Response.Body = htmlRewriter + return nil + } +} + +// InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head) +func InjectScriptBeforeDOMContentLoaded(js string) proxychain.ResponseModification { + return injectScript(js, rewriters.BeforeDOMContentLoaded) +} + +// InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head) +func InjectScriptAfterDOMContentLoaded(js string) proxychain.ResponseModification { + return injectScript(js, rewriters.AfterDOMContentLoaded) +} + +// InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded) +func InjectScriptAfterDOMIdle(js string) proxychain.ResponseModification { + return injectScript(js, rewriters.AfterDOMIdle) +} diff --git a/proxychain/responsemodifiers/modify_incoming_cookies.go b/proxychain/responsemodifiers/modify_incoming_cookies.go new file mode 100644 index 00000000..4c77468e --- /dev/null +++ b/proxychain/responsemodifiers/modify_incoming_cookies.go @@ -0,0 +1,111 @@ +package responsemodifiers + +import ( + "fmt" + + http "github.com/bogdanfinn/fhttp" + //"net/http" + //http "github.com/Danny-Dasilva/fhttp" + + "github.com/everywall/ladder/proxychain" +) + +// DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server +// back down to the client. +func DeleteIncomingCookies(_ ...string) proxychain.ResponseModification { + return func(chain *proxychain.ProxyChain) error { + chain.Response.Header.Del("Set-Cookie") + chain.AddOnceResponseModifications( + InjectScriptBeforeDOMContentLoaded(`document.cookie = ""`), + InjectScriptAfterDOMContentLoaded(`document.cookie = ""`), + ) + return nil + } +} + +// DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server +// to the client. Cookies whose names are in the whitelist are not removed. +func DeleteIncomingCookiesExcept(whitelist ...string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + // Convert whitelist slice to a map for efficient lookups + whitelistMap := make(map[string]struct{}) + for _, cookieName := range whitelist { + whitelistMap[cookieName] = struct{}{} + } + + // If the response has no cookies, return early + if px.Response.Header == nil { + return nil + } + + // Filter the cookies in the response + filteredCookies := []string{} + for _, cookieStr := range px.Response.Header["Set-Cookie"] { + cookie := parseCookie(cookieStr) + + if _, found := whitelistMap[cookie.Name]; found { + filteredCookies = append(filteredCookies, cookieStr) + } + } + + // Update the Set-Cookie header with the filtered cookies + if len(filteredCookies) > 0 { + px.Response.Header["Set-Cookie"] = filteredCookies + } else { + px.Response.Header.Del("Set-Cookie") + } + + return nil + } +} + +// parseCookie parses a cookie string and returns an http.Cookie object. +func parseCookie(cookieStr string) *http.Cookie { + header := http.Header{} + header.Add("Set-Cookie", cookieStr) + request := http.Request{Header: header} + return request.Cookies()[0] +} + +// SetIncomingCookies adds a raw cookie string being sent from the proxy server down to the client +func SetIncomingCookies(cookies string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + px.Response.Header.Set("Set-Cookie", cookies) + return nil + } +} + +// SetIncomingCookie modifies a specific cookie in the response from the proxy server to the client. +func SetIncomingCookie(name string, val string) proxychain.ResponseModification { + return func(px *proxychain.ProxyChain) error { + if px.Response.Header == nil { + return nil + } + + updatedCookies := []string{} + found := false + + // Iterate over existing cookies and modify the one that matches the cookieName + for _, cookieStr := range px.Response.Header["Set-Cookie"] { + cookie := parseCookie(cookieStr) + if cookie.Name == name { + // Replace the cookie with the new value + updatedCookies = append(updatedCookies, fmt.Sprintf("%s=%s", name, val)) + found = true + } else { + // Keep the cookie as is + updatedCookies = append(updatedCookies, cookieStr) + } + } + + // If the specified cookie wasn't found, add it + if !found { + updatedCookies = append(updatedCookies, fmt.Sprintf("%s=%s", name, val)) + } + + // Update the Set-Cookie header + px.Response.Header["Set-Cookie"] = updatedCookies + + return nil + } +} diff --git a/proxychain/responsemodifiers/modify_incoming_scripts_with_regex.go b/proxychain/responsemodifiers/modify_incoming_scripts_with_regex.go new file mode 100644 index 00000000..65082603 --- /dev/null +++ b/proxychain/responsemodifiers/modify_incoming_scripts_with_regex.go @@ -0,0 +1,53 @@ +package responsemodifiers + +import ( + "bytes" + "io" + "regexp" + "strings" + + "github.com/everywall/ladder/proxychain" +) + +// ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline \n", r.scriptMD5, r.script) + + case r.execTime == AfterDOMContentLoaded: + return "", fmt.Sprintf("\n", r.scriptMD5, r.script) + + case r.execTime == AfterDOMIdle: + s := strings.Replace(afterDomIdleScriptInjector, `'{{AFTER_DOM_IDLE_SCRIPT}}'`, r.script, 1) + return "", fmt.Sprintf("\n\n", r.scriptMD5, s) + + default: + return "", "" + } +} + +// GenerateMD5Hash takes a string and returns its MD5 hash as a hexadecimal string +func generateMD5Hash(input string) string { + hasher := md5.New() + hasher.Write([]byte(input)) + return hex.EncodeToString(hasher.Sum(nil)) +} + +// applies parameters by string replacement of the template script +func (r *ScriptInjectorRewriter) applyParams(params map[string]string) { + // Sort the keys by length in descending order + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + + sort.Slice(keys, func(i, j int) bool { + return len(keys[i]) > len(keys[j]) + }) + + for _, key := range keys { + r.script = strings.ReplaceAll(r.script, key, params[key]) + } +} + +// NewScriptInjectorRewriter implements a HtmlTokenRewriter +// and injects JS into the page for execution at a particular time +func NewScriptInjectorRewriter(script string, execTime ScriptExecTime) *ScriptInjectorRewriter { + scriptMD5 := generateMD5Hash(script) + executeOnceScript := fmt.Sprintf(`if (!document.getElementById("x-%s")) { %s; document.getElementById("%s").id = "x-%s" };`, scriptMD5, script, scriptMD5, scriptMD5) + + return &ScriptInjectorRewriter{ + execTime: execTime, + script: executeOnceScript, + scriptMD5: scriptMD5, + } +} + +// NewScriptInjectorRewriterWith implements a HtmlTokenRewriter +// and injects JS into the page for execution at a particular time +// accepting arguments into the script, which will be added via a string replace +// the params map represents the key-value pair of the params. +// the key will be string replaced with the value +func NewScriptInjectorRewriterWithParams(script string, execTime ScriptExecTime, params map[string]string) *ScriptInjectorRewriter { + rr := NewScriptInjectorRewriter(script, execTime) + rr.applyParams(params) + return rr +} diff --git a/proxychain/responsemodifiers/vendor/block_element_removal.js b/proxychain/responsemodifiers/vendor/block_element_removal.js new file mode 100644 index 00000000..da8350ed --- /dev/null +++ b/proxychain/responsemodifiers/vendor/block_element_removal.js @@ -0,0 +1,35 @@ +/** + * Monitors and restores specific DOM elements if they are removed. + * + * This self-invoking function creates a MutationObserver to watch for removal of elements matching + * "{{CSS_SELECTOR}}". If such an element is removed, it logs the event and attempts to restore the + * element after a 50ms delay. The restored element is reinserted at its original location or prepended + * to the document body if the original location is unavailable. + */ +(function() { + function handleMutation(mutationList) { + for (const mutation of mutationList) { + if (mutation.type === "childList") { + for (const node of Array.from(mutation.removedNodes)) { + if (node.outerHTML && node.querySelector("{{CSS_SELECTOR}}")) { + console.log( + "proxychain: prevented removal of element containing 'article-content'", + ); + console.log(node.outerHTML); + setTimeout(() => { + let e = document.querySelector("{{CSS_SELECTOR}}"); + if (e != null) { + e.replaceWith(node); + } else { + document.body.prepend(node); + } + }, 50); + } + } + } + } + } + + const observer = new MutationObserver(handleMutation); + observer.observe(document, { childList: true, subtree: true }); +})(); diff --git a/proxychain/responsemodifiers/vendor/ddg-tracker-surrogates b/proxychain/responsemodifiers/vendor/ddg-tracker-surrogates new file mode 160000 index 00000000..ba0d8cef --- /dev/null +++ b/proxychain/responsemodifiers/vendor/ddg-tracker-surrogates @@ -0,0 +1 @@ +Subproject commit ba0d8cefe4432723ec75b998241efd2454dff35a diff --git a/proxychain/responsemodifiers/vendor/generate_readable_outline.html b/proxychain/responsemodifiers/vendor/generate_readable_outline.html new file mode 100644 index 00000000..5619e96c --- /dev/null +++ b/proxychain/responsemodifiers/vendor/generate_readable_outline.html @@ -0,0 +1,461 @@ + + + + + + + + + ladder | {{.Title}} + + + +
+
+ + +
+ +
+
+ +
+ {{if not .Success}} +

Error

+

+ There was a problem querying + {{.Params}} +

+ {{.Error}} + {{else}} + +
+

+ + {{.Title}} + +

+ {{if ne .Date ""}} + {{.Date}} + {{end}} {{if ne .Author ""}} + {{.Author}} + {{end}} +
+ +

+ + {{.Title}} + +

+ +
+
+ {{if ne .Author ""}} + {{.Author}} | + + {{end}} {{if ne .Date ""}} + {{.Date}} + {{end}} +
+ +
+ Reading Time: {{.ReadingTime}} +
+
+
+
+
+ {{.Description}} +
+
+ {{.Description}} +
+
+
+ +
{{.Body}}
+ {{end}} +
+ + +
+ + diff --git a/proxychain/responsemodifiers/vendor/patch_dynamic_resource_urls.js b/proxychain/responsemodifiers/vendor/patch_dynamic_resource_urls.js new file mode 100644 index 00000000..74eba115 --- /dev/null +++ b/proxychain/responsemodifiers/vendor/patch_dynamic_resource_urls.js @@ -0,0 +1,368 @@ +// Overrides the global fetch and XMLHttpRequest open methods to modify the request URLs. +// Also overrides the attribute setter prototype to modify the request URLs +// fetch("/relative_script.js") -> fetch("http://localhost:8080/relative_script.js") +(() => { + // ============== PARAMS =========================== + // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar + // proxyOrigin is http://localhost:8080 + const proxyOrigin = "{{PROXY_ORIGIN}}"; + //const proxyOrigin = globalThis.window.location.origin; + + // if the original request was: http://localhost:8080/http://proxiedsite.com/foo/bar + // origin is http://proxiedsite.com + const origin = "{{ORIGIN}}"; + //const origin = (new URL(decodeURIComponent(globalThis.window.location.pathname.substring(1)))).origin + // ============== END PARAMS ====================== + + const blacklistedSchemes = [ + "ftp:", + "mailto:", + "tel:", + "file:", + "blob:", + "javascript:", + "about:", + "magnet:", + "ws:", + "wss:", + ]; + + function rewriteURL(url) { + if (!url) return url; + + // fetch url might be string, url, or request object + // handle all three by downcasting to string + const isStr = typeof url === "string"; + if (!isStr) { + x = String(url); + if (x == "[object Request]") { + url = url.url; + } else { + url = String(url); + } + } + + const oldUrl = url; + + // don't rewrite special URIs + if (blacklistedSchemes.includes(url)) return url; + + // don't rewrite invalid URIs + try { + new URL(url, origin); + } catch { + return url; + } + + // don't double rewrite + if (url.startsWith(`${proxyOrigin}/http://`)) return url; + if (url.startsWith(`${proxyOrigin}/https://`)) return url; + if (url.startsWith(`/${proxyOrigin}`)) return url; + if (url.startsWith(`/${origin}`)) return url; + if (url.startsWith(`/http://`)) return url; + if (url.startsWith(`/https://`)) return url; + if (url.startsWith(`/http%3A%2F%2F`)) return url; + if (url.startsWith(`/https%3A%2F%2F`)) return url; + if (url.startsWith(`/%2Fhttp`)) return url; + + //console.log(`proxychain: origin: ${origin} // proxyOrigin: ${proxyOrigin} // original: ${oldUrl}`) + + //originDomain = origin.replace("https://", ""); + let scheme = origin.split(":")[0]; + + if (url.startsWith("//")) { + url = `/${scheme}://${encodeURIComponent(url.substring(2))}`; + } else if (url.startsWith("/")) { + url = `/${origin}/${encodeURIComponent(url.substring(1))}`; + } else if ( + url.startsWith(proxyOrigin) && !url.startsWith(`${proxyOrigin}/http`) + ) { + // edge case where client js uses current url host to write an absolute path + url = "".replace(proxyOrigin, `${proxyOrigin}/${origin}`); + } else if (url.startsWith(origin)) { + url = `/${encodeURIComponent(url)}`; + } else if (url.startsWith("http://") || url.startsWith("https://")) { + url = `/${proxyOrigin}/${encodeURIComponent(url)}`; + } + console.log(`proxychain: rewrite JS URL: ${oldUrl} -> ${url}`); + return url; + } + + /* + // sometimes anti-bot protections like cloudflare or akamai bot manager check if JS is hooked + function hideMonkeyPatch(objectOrName, method, originalToString) { + let obj; + let isGlobalFunction = false; + + if (typeof objectOrName === "string") { + obj = globalThis[objectOrName]; + isGlobalFunction = (typeof obj === "function") && + (method === objectOrName); + } else { + obj = objectOrName; + } + + if (isGlobalFunction) { + const originalFunction = obj; + globalThis[objectOrName] = function(...args) { + return originalFunction.apply(this, args); + }; + globalThis[objectOrName].toString = () => originalToString; + } else if (obj && typeof obj[method] === "function") { + const originalMethod = obj[method]; + obj[method] = function(...args) { + return originalMethod.apply(this, args); + }; + obj[method].toString = () => originalToString; + } else { + console.warn( + `proxychain: cannot hide monkey patch: ${method} is not a function on the provided object.`, + ); + } + } + */ + function hideMonkeyPatch(objectOrName, method, originalToString) { + return; + } + + // monkey patch fetch + const oldFetch = fetch; + fetch = async (url, init) => { + return oldFetch(rewriteURL(url), init); + }; + hideMonkeyPatch("fetch", "fetch", "function fetch() { [native code] }"); + + // monkey patch xmlhttprequest + const oldOpen = XMLHttpRequest.prototype.open; + XMLHttpRequest.prototype.open = function( + method, + url, + async = true, + user = null, + password = null, + ) { + return oldOpen.call(this, method, rewriteURL(url), async, user, password); + }; + hideMonkeyPatch( + XMLHttpRequest.prototype, + "open", + 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}', + ); + + const oldSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.send = function(method, url) { + return oldSend.call(this, method, rewriteURL(url)); + }; + hideMonkeyPatch( + XMLHttpRequest.prototype, + "send", + 'function(){if("function"==typeof eo)return eo.apply(this,arguments)}', + ); + + // monkey patch service worker registration + /* + const oldRegister = ServiceWorkerContainer.prototype.register; + ServiceWorkerContainer.prototype.register = function (scriptURL, options) { + return oldRegister.call(this, rewriteURL(scriptURL), options); + }; + hideMonkeyPatch( + ServiceWorkerContainer.prototype, + "register", + "function register() { [native code] }", + ); + */ + + // monkey patch URL.toString() method + const oldToString = URL.prototype.toString; + URL.prototype.toString = function() { + let originalURL = oldToString.call(this); + return rewriteURL(originalURL); + }; + hideMonkeyPatch( + URL.prototype, + "toString", + "function toString() { [native code] }", + ); + + // monkey patch URL.toJSON() method + const oldToJson = URL.prototype.toString; + URL.prototype.toString = function() { + let originalURL = oldToJson.call(this); + return rewriteURL(originalURL); + }; + hideMonkeyPatch( + URL.prototype, + "toString", + "function toJSON() { [native code] }", + ); + + // Monkey patch URL.href getter and setter + const originalHrefDescriptor = Object.getOwnPropertyDescriptor( + URL.prototype, + "href", + ); + Object.defineProperty(URL.prototype, "href", { + get: function() { + let originalHref = originalHrefDescriptor.get.call(this); + return rewriteURL(originalHref); + }, + set: function(newValue) { + originalHrefDescriptor.set.call(this, rewriteURL(newValue)); + }, + }); + + // TODO: do one more pass of this by manually traversing the DOM + // AFTER all the JS and page has loaded just in case + + // Monkey patch setter + const elements = [ + { tag: "a", attribute: "href" }, + { tag: "img", attribute: "src" }, + // { tag: 'img', attribute: 'srcset' }, // TODO: handle srcset + { tag: "script", attribute: "src" }, + { tag: "link", attribute: "href" }, + { tag: "link", attribute: "icon" }, + { tag: "iframe", attribute: "src" }, + { tag: "audio", attribute: "src" }, + { tag: "video", attribute: "src" }, + { tag: "source", attribute: "src" }, + // { tag: 'source', attribute: 'srcset' }, // TODO: handle srcset + { tag: "embed", attribute: "src" }, + { tag: "embed", attribute: "pluginspage" }, + { tag: "html", attribute: "manifest" }, + { tag: "object", attribute: "src" }, + { tag: "input", attribute: "src" }, + { tag: "track", attribute: "src" }, + { tag: "form", attribute: "action" }, + { tag: "area", attribute: "href" }, + { tag: "base", attribute: "href" }, + { tag: "blockquote", attribute: "cite" }, + { tag: "del", attribute: "cite" }, + { tag: "ins", attribute: "cite" }, + { tag: "q", attribute: "cite" }, + { tag: "button", attribute: "formaction" }, + { tag: "input", attribute: "formaction" }, + { tag: "meta", attribute: "content" }, + { tag: "object", attribute: "data" }, + ]; + + elements.forEach(({ tag, attribute }) => { + const proto = document.createElement(tag).constructor.prototype; + const descriptor = Object.getOwnPropertyDescriptor(proto, attribute); + if (descriptor && descriptor.set) { + Object.defineProperty(proto, attribute, { + ...descriptor, + set(value) { + // calling rewriteURL will end up calling a setter for href, + // leading to a recusive loop and a Maximum call stack size exceeded + // error, so we guard against this with a local semaphore flag + const isRewritingSetKey = Symbol.for("isRewritingSet"); + if (!this[isRewritingSetKey]) { + this[isRewritingSetKey] = true; + descriptor.set.call(this, rewriteURL(value)); + //descriptor.set.call(this, value); + this[isRewritingSetKey] = false; + } else { + // Directly set the value without rewriting + descriptor.set.call(this, value); + } + }, + get() { + const isRewritingGetKey = Symbol.for("isRewritingGet"); + if (!this[isRewritingGetKey]) { + this[isRewritingGetKey] = true; + let oldURL = descriptor.get.call(this); + let newURL = rewriteURL(oldURL); + this[isRewritingGetKey] = false; + return newURL; + } else { + return descriptor.get.call(this); + } + }, + }); + } + }); + + // monkey-patching Element.setAttribute + const originalSetAttribute = Element.prototype.setAttribute; + Element.prototype.setAttribute = function(name, value) { + const isMatchingElement = elements.some((element) => { + return this.tagName.toLowerCase() === element.tag && + name.toLowerCase() === element.attribute; + }); + if (isMatchingElement) { + value = rewriteURL(value); + } + originalSetAttribute.call(this, name, value); + }; + + // sometimes, libraries will set the Element.innerHTML or Element.outerHTML directly with a string instead of setters. + // in this case, we intercept it, create a fake DOM, parse it and then rewrite all attributes that could + // contain a URL. Then we return the replacement innerHTML/outerHTML with redirected links. + function rewriteInnerHTML(html, elements) { + const isRewritingHTMLKey = Symbol.for("isRewritingHTML"); + + // Check if already processing + if (document[isRewritingHTMLKey]) { + return html; + } + + const tempContainer = document.createElement("div"); + document[isRewritingHTMLKey] = true; + + try { + tempContainer.innerHTML = html; + + // Create a map for quick lookup + const elementsMap = new Map(elements.map((e) => [e.tag, e.attribute])); + + // Loop-based DOM traversal + const nodes = [...tempContainer.querySelectorAll("*")]; + for (const node of nodes) { + const attribute = elementsMap.get(node.tagName.toLowerCase()); + if (attribute && node.hasAttribute(attribute)) { + const originalUrl = node.getAttribute(attribute); + const rewrittenUrl = rewriteURL(originalUrl); + node.setAttribute(attribute, rewrittenUrl); + } + } + + return tempContainer.innerHTML; + } finally { + // Clear the flag + document[isRewritingHTMLKey] = false; + } + } + + // Store original setters + const originalSetters = {}; + + ["innerHTML", "outerHTML"].forEach((property) => { + const descriptor = Object.getOwnPropertyDescriptor( + Element.prototype, + property, + ); + if (descriptor && descriptor.set) { + originalSetters[property] = descriptor.set; + + Object.defineProperty(Element.prototype, property, { + ...descriptor, + set(value) { + const isRewritingHTMLKey = Symbol.for("isRewritingHTML"); + if (!this[isRewritingHTMLKey]) { + this[isRewritingHTMLKey] = true; + try { + // Use custom logic + descriptor.set.call(this, rewriteInnerHTML(value, elements)); + } finally { + this[isRewritingHTMLKey] = false; + } + } else { + // Use original setter in recursive call + originalSetters[property].call(this, value); + } + }, + }); + } + }); +})(); diff --git a/proxychain/responsemodifiers/vendor/patch_google_analytics.js b/proxychain/responsemodifiers/vendor/patch_google_analytics.js new file mode 100644 index 00000000..31c3dcd7 --- /dev/null +++ b/proxychain/responsemodifiers/vendor/patch_google_analytics.js @@ -0,0 +1,109 @@ +// uBlock Origin - a browser extension to block requests. +// Copyright (C) 2019-present Raymond Hill +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see {http://www.gnu.org/licenses/}. +// +// Home: https://github.com/gorhill/uBlock + +(function() { + "use strict"; + // https://developers.google.com/analytics/devguides/collection/analyticsjs/ + const noopfn = function() { + }; + // + const Tracker = function() { + }; + const p = Tracker.prototype; + p.get = noopfn; + p.set = noopfn; + p.send = noopfn; + // + const w = window; + const gaName = w.GoogleAnalyticsObject || "ga"; + const gaQueue = w[gaName]; + // https://github.com/uBlockOrigin/uAssets/pull/4115 + const ga = function() { + const len = arguments.length; + if (len === 0) return; + const args = Array.from(arguments); + let fn; + let a = args[len - 1]; + if (a instanceof Object && a.hitCallback instanceof Function) { + fn = a.hitCallback; + } else if (a instanceof Function) { + fn = () => { + a(ga.create()); + }; + } else { + const pos = args.indexOf("hitCallback"); + if (pos !== -1 && args[pos + 1] instanceof Function) { + fn = args[pos + 1]; + } + } + if (fn instanceof Function === false) return; + try { + fn(); + } catch (ex) { + } + }; + ga.create = function() { + return new Tracker(); + }; + ga.getByName = function() { + return new Tracker(); + }; + ga.getAll = function() { + return [new Tracker()]; + }; + ga.remove = noopfn; + // https://github.com/uBlockOrigin/uAssets/issues/2107 + ga.loaded = true; + w[gaName] = ga; + // https://github.com/gorhill/uBlock/issues/3075 + const dl = w.dataLayer; + if (dl instanceof Object) { + if (dl.hide instanceof Object && typeof dl.hide.end === "function") { + dl.hide.end(); + dl.hide.end = () => { }; + } + if (typeof dl.push === "function") { + const doCallback = function(item) { + if (item instanceof Object === false) return; + if (typeof item.eventCallback !== "function") return; + setTimeout(item.eventCallback, 1); + item.eventCallback = () => { }; + }; + dl.push = new Proxy(dl.push, { + apply: function(target, thisArg, args) { + doCallback(args[0]); + return Reflect.apply(target, thisArg, args); + }, + }); + if (Array.isArray(dl)) { + const q = dl.slice(); + for (const item of q) { + doCallback(item); + } + } + } + } + // empty ga queue + if (gaQueue instanceof Function && Array.isArray(gaQueue.q)) { + const q = gaQueue.q.slice(); + gaQueue.q.length = 0; + for (const entry of q) { + ga(...entry); + } + } +})(); diff --git a/proxychain/ruleset/rule.go b/proxychain/ruleset/rule.go new file mode 100644 index 00000000..221a4113 --- /dev/null +++ b/proxychain/ruleset/rule.go @@ -0,0 +1,137 @@ +package ruleset_v2 + +import ( + //"bytes" + "encoding/json" + "fmt" + + //"gopkg.in/yaml.v3" + "github.com/everywall/ladder/proxychain" +) + +type Rule struct { + Domains []string + RequestModifications []proxychain.RequestModification + _rqms []_rqm // internal represenation of RequestModifications + ResponseModifications []proxychain.ResponseModification + _rsms []_rsm // internal represenation of ResponseModifications +} + +// internal represenation of ResponseModifications +type _rsm struct { + Name string `json:"name" yaml:"name"` + Params []string `json:"params" yaml:"params"` +} + +// internal represenation of RequestModifications +type _rqm struct { + Name string `json:"name" yaml:"name"` + Params []string `json:"params" yaml:"params"` +} + +// implement type encoding/json/Marshaler +func (rule *Rule) UnmarshalJSON(data []byte) error { + type Aux struct { + Domains []string `json:"domains"` + RequestModifications []_rqm `json:"requestmodifications"` + ResponseModifications []_rsm `json:"responsemodifications"` + } + + aux := &Aux{} + if err := json.Unmarshal(data, aux); err != nil { + return err + } + + rule.Domains = aux.Domains + rule._rqms = aux.RequestModifications + rule._rsms = aux.ResponseModifications + + // convert requestModification function call string into actual functional option + for _, rqm := range aux.RequestModifications { + f, exists := rqmModMap[rqm.Name] + if !exists { + return fmt.Errorf("Rule::UnmarshalJSON => requestModifier '%s' does not exist, please check spelling", rqm.Name) + } + rule.RequestModifications = append(rule.RequestModifications, f(rqm.Params...)) + } + + // convert responseModification function call string into actual functional option + for _, rsm := range aux.ResponseModifications { + f, exists := rsmModMap[rsm.Name] + if !exists { + return fmt.Errorf("Rule::UnmarshalJSON => responseModifier '%s' does not exist, please check spelling", rsm.Name) + } + rule.ResponseModifications = append(rule.ResponseModifications, f(rsm.Params...)) + } + + return nil +} + +func (rule *Rule) MarshalJSON() ([]byte, error) { + aux := struct { + Domains []string `json:"domains"` + RequestModifications []_rqm `json:"requestmodifications"` + ResponseModifications []_rsm `json:"responsemodifications"` + }{ + Domains: rule.Domains, + RequestModifications: rule._rqms, + ResponseModifications: rule._rsms, + } + + return json.MarshalIndent(aux, "", " ") +} + +// ============================================================ +// YAML + +// implement type yaml marshaller +func (rule *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error { + type Aux struct { + Domains []string `yaml:"domains"` + RequestModifications []_rqm `yaml:"requestmodifications"` + ResponseModifications []_rsm `yaml:"responsemodifications"` + } + + var aux Aux + if err := unmarshal(&aux); err != nil { + return err + } + + rule.Domains = aux.Domains + rule._rqms = aux.RequestModifications + rule._rsms = aux.ResponseModifications + + // convert requestModification function call string into actual functional option + for _, rqm := range aux.RequestModifications { + f, exists := rqmModMap[rqm.Name] + if !exists { + return fmt.Errorf("Rule::UnmarshalYAML => requestModifier '%s' does not exist, please check spelling", rqm.Name) + } + rule.RequestModifications = append(rule.RequestModifications, f(rqm.Params...)) + } + + // convert responseModification function call string into actual functional option + for _, rsm := range aux.ResponseModifications { + f, exists := rsmModMap[rsm.Name] + if !exists { + return fmt.Errorf("Rule::UnmarshalYAML => responseModifier '%s' does not exist, please check spelling", rsm.Name) + } + rule.ResponseModifications = append(rule.ResponseModifications, f(rsm.Params...)) + } + + return nil +} + +func (rule *Rule) MarshalYAML() (interface{}, error) { + type Aux struct { + Domains []string `yaml:"domains"` + RequestModifications []_rqm `yaml:"requestmodifications"` + ResponseModifications []_rsm `yaml:"responsemodifications"` + } + + return &Aux{ + Domains: rule.Domains, + RequestModifications: rule._rqms, + ResponseModifications: rule._rsms, + }, nil +} diff --git a/proxychain/ruleset/rule_reqmod_types.gen.go b/proxychain/ruleset/rule_reqmod_types.gen.go new file mode 100644 index 00000000..a4445513 --- /dev/null +++ b/proxychain/ruleset/rule_reqmod_types.gen.go @@ -0,0 +1,187 @@ + +package ruleset_v2 +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go +// The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain" + rx "github.com/everywall/ladder/proxychain/requestmodifiers" +) + +type RequestModifierFactory func(params ...string) proxychain.RequestModification + +var rqmModMap map[string]RequestModifierFactory + +func init() { + rqmModMap = make(map[string]RequestModifierFactory) + + rqmModMap["AddCacheBusterQuery"] = func(_ ...string) proxychain.RequestModification { + return rx.AddCacheBusterQuery() + } + + rqmModMap["ForwardRequestHeaders"] = func(_ ...string) proxychain.RequestModification { + return rx.ForwardRequestHeaders() + } + + rqmModMap["MasqueradeAsGoogleBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsGoogleBot() + } + + rqmModMap["MasqueradeAsBingBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsBingBot() + } + + rqmModMap["MasqueradeAsWaybackMachineBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsWaybackMachineBot() + } + + rqmModMap["MasqueradeAsFacebookBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsFacebookBot() + } + + rqmModMap["MasqueradeAsYandexBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsYandexBot() + } + + rqmModMap["MasqueradeAsBaiduBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsBaiduBot() + } + + rqmModMap["MasqueradeAsDuckDuckBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsDuckDuckBot() + } + + rqmModMap["MasqueradeAsYahooBot"] = func(_ ...string) proxychain.RequestModification { + return rx.MasqueradeAsYahooBot() + } + + rqmModMap["ModifyDomainWithRegex"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyDomainWithRegex(params[0], params[1]) + } + + rqmModMap["SetOutgoingCookie"] = func(params ...string) proxychain.RequestModification { + return rx.SetOutgoingCookie(params[0], params[1]) + } + + rqmModMap["SetOutgoingCookies"] = func(params ...string) proxychain.RequestModification { + return rx.SetOutgoingCookies(params[0]) + } + + rqmModMap["DeleteOutgoingCookie"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookie(params[0]) + } + + rqmModMap["DeleteOutgoingCookies"] = func(_ ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookies() + } + + rqmModMap["DeleteOutgoingCookiesExcept"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteOutgoingCookiesExcept(params[0]) + } + + rqmModMap["ModifyPathWithRegex"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyPathWithRegex(params[0], params[1]) + } + + rqmModMap["ModifyQueryParams"] = func(params ...string) proxychain.RequestModification { + return rx.ModifyQueryParams(params[0], params[1]) + } + + rqmModMap["SetRequestHeader"] = func(params ...string) proxychain.RequestModification { + return rx.SetRequestHeader(params[0], params[1]) + } + + rqmModMap["DeleteRequestHeader"] = func(params ...string) proxychain.RequestModification { + return rx.DeleteRequestHeader(params[0]) + } + + rqmModMap["RequestArchiveIs"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestArchiveIs() + } + + rqmModMap["RequestGoogleCache"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestGoogleCache() + } + + rqmModMap["RequestWaybackMachine"] = func(_ ...string) proxychain.RequestModification { + return rx.RequestWaybackMachine() + } + + rqmModMap["ResolveWithGoogleDoH"] = func(_ ...string) proxychain.RequestModification { + return rx.ResolveWithGoogleDoH() + } + + rqmModMap["SpoofOrigin"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofOrigin(params[0]) + } + + rqmModMap["HideOrigin"] = func(_ ...string) proxychain.RequestModification { + return rx.HideOrigin() + } + + rqmModMap["SpoofReferrer"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofReferrer(params[0]) + } + + rqmModMap["HideReferrer"] = func(_ ...string) proxychain.RequestModification { + return rx.HideReferrer() + } + + rqmModMap["SpoofReferrerFromBaiduSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromBaiduSearch() + } + + rqmModMap["SpoofReferrerFromBingSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromBingSearch() + } + + rqmModMap["SpoofReferrerFromGoogleSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromGoogleSearch() + } + + rqmModMap["SpoofReferrerFromLinkedInPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromLinkedInPost() + } + + rqmModMap["SpoofReferrerFromNaverSearch"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromNaverSearch() + } + + rqmModMap["SpoofReferrerFromPinterestPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromPinterestPost() + } + + rqmModMap["SpoofReferrerFromQQPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromQQPost() + } + + rqmModMap["SpoofReferrerFromRedditPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromRedditPost() + } + + rqmModMap["SpoofReferrerFromTumblrPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromTumblrPost() + } + + rqmModMap["SpoofReferrerFromTwitterPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromTwitterPost() + } + + rqmModMap["SpoofReferrerFromVkontaktePost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromVkontaktePost() + } + + rqmModMap["SpoofReferrerFromWeiboPost"] = func(_ ...string) proxychain.RequestModification { + return rx.SpoofReferrerFromWeiboPost() + } + + rqmModMap["SpoofUserAgent"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofUserAgent(params[0]) + } + + rqmModMap["SpoofXForwardedFor"] = func(params ...string) proxychain.RequestModification { + return rx.SpoofXForwardedFor(params[0]) + } + +} \ No newline at end of file diff --git a/proxychain/ruleset/rule_resmod_types.gen.go b/proxychain/ruleset/rule_resmod_types.gen.go new file mode 100644 index 00000000..59bfe847 --- /dev/null +++ b/proxychain/ruleset/rule_resmod_types.gen.go @@ -0,0 +1,111 @@ + +package ruleset_v2 +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/proxychain/codegen/codegen.go +// The purpose of this is serialization of rulesets from JSON or YAML into functional options suitable +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" +) + +type ResponseModifierFactory func(params ...string) proxychain.ResponseModification + +var rsmModMap map[string]ResponseModifierFactory + +func init() { + rsmModMap = make(map[string]ResponseModifierFactory) + + rsmModMap["APIContent"] = func(_ ...string) proxychain.ResponseModification { + return tx.APIContent() + } + + rsmModMap["BlockElementRemoval"] = func(params ...string) proxychain.ResponseModification { + return tx.BlockElementRemoval(params[0]) + } + + rsmModMap["BlockThirdPartyScripts"] = func(_ ...string) proxychain.ResponseModification { + return tx.BlockThirdPartyScripts() + } + + rsmModMap["BypassCORS"] = func(_ ...string) proxychain.ResponseModification { + return tx.BypassCORS() + } + + rsmModMap["BypassContentSecurityPolicy"] = func(_ ...string) proxychain.ResponseModification { + return tx.BypassContentSecurityPolicy() + } + + rsmModMap["SetContentSecurityPolicy"] = func(params ...string) proxychain.ResponseModification { + return tx.SetContentSecurityPolicy(params[0]) + } + + rsmModMap["DeleteLocalStorageData"] = func(_ ...string) proxychain.ResponseModification { + return tx.DeleteLocalStorageData() + } + + rsmModMap["DeleteSessionStorageData"] = func(_ ...string) proxychain.ResponseModification { + return tx.DeleteSessionStorageData() + } + + rsmModMap["ForwardResponseHeaders"] = func(_ ...string) proxychain.ResponseModification { + return tx.ForwardResponseHeaders() + } + + rsmModMap["GenerateReadableOutline"] = func(_ ...string) proxychain.ResponseModification { + return tx.GenerateReadableOutline() + } + + rsmModMap["InjectScriptBeforeDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptBeforeDOMContentLoaded(params[0]) + } + + rsmModMap["InjectScriptAfterDOMContentLoaded"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptAfterDOMContentLoaded(params[0]) + } + + rsmModMap["InjectScriptAfterDOMIdle"] = func(params ...string) proxychain.ResponseModification { + return tx.InjectScriptAfterDOMIdle(params[0]) + } + + rsmModMap["DeleteIncomingCookies"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteIncomingCookies(params[0]) + } + + rsmModMap["DeleteIncomingCookiesExcept"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteIncomingCookiesExcept(params[0]) + } + + rsmModMap["SetIncomingCookies"] = func(params ...string) proxychain.ResponseModification { + return tx.SetIncomingCookies(params[0]) + } + + rsmModMap["SetIncomingCookie"] = func(params ...string) proxychain.ResponseModification { + return tx.SetIncomingCookie(params[0], params[1]) + } + + rsmModMap["ModifyIncomingScriptsWithRegex"] = func(params ...string) proxychain.ResponseModification { + return tx.ModifyIncomingScriptsWithRegex(params[0], params[1]) + } + + rsmModMap["SetResponseHeader"] = func(params ...string) proxychain.ResponseModification { + return tx.SetResponseHeader(params[0], params[1]) + } + + rsmModMap["DeleteResponseHeader"] = func(params ...string) proxychain.ResponseModification { + return tx.DeleteResponseHeader(params[0]) + } + + rsmModMap["PatchDynamicResourceURLs"] = func(_ ...string) proxychain.ResponseModification { + return tx.PatchDynamicResourceURLs() + } + + rsmModMap["PatchTrackerScripts"] = func(_ ...string) proxychain.ResponseModification { + return tx.PatchTrackerScripts() + } + + rsmModMap["RewriteHTMLResourceURLs"] = func(_ ...string) proxychain.ResponseModification { + return tx.RewriteHTMLResourceURLs() + } + +} \ No newline at end of file diff --git a/proxychain/ruleset/rule_test.go b/proxychain/ruleset/rule_test.go new file mode 100644 index 00000000..08bde6c5 --- /dev/null +++ b/proxychain/ruleset/rule_test.go @@ -0,0 +1,137 @@ +package ruleset_v2 + +import ( + "encoding/json" + "fmt" + "testing" + + "gopkg.in/yaml.v3" +) + +// unmarshalRule is a helper function to unmarshal a Rule from a JSON string. +func unmarshalRule(t *testing.T, ruleJSON string) *Rule { + rule := &Rule{} + err := json.Unmarshal([]byte(ruleJSON), rule) + if err != nil { + t.Fatalf("expected no error in Unmarshal, got '%s'", err) + } + return rule +} + +func TestRuleUnmarshalJSON(t *testing.T) { + ruleJSON := `{ + "domains": ["example.com", "www.example.com"], + "responsemodifications": [{"name": "APIContent", "params": []}, {"name": "SetContentSecurityPolicy", "params": ["foobar"]}, {"name": "SetIncomingCookie", "params": ["authorization-bearer", "hunter2"]}], + "requestmodifications": [{"name": "ForwardRequestHeaders", "params": []}] + }` + + rule := unmarshalRule(t, ruleJSON) + + if len(rule.Domains) != 2 { + t.Errorf("expected number of domains to be 2") + } + if !(rule.Domains[0] == "example.com" || rule.Domains[1] == "example.com") { + t.Errorf("expected domain to be example.com") + } + if len(rule.ResponseModifications) != 3 { + t.Errorf("expected number of ResponseModifications to be 3, got %d", len(rule.ResponseModifications)) + } + if len(rule.RequestModifications) != 1 { + t.Errorf("expected number of RequestModifications to be 1, got %d", len(rule.RequestModifications)) + } +} + +func TestRuleMarshalJSON(t *testing.T) { + ruleJSON := `{ + "domains": ["example.com", "www.example.com"], + "responsemodifications": [{"name": "APIContent", "params": []}, {"name": "SetContentSecurityPolicy", "params": ["foobar"]}, {"name": "SetIncomingCookie", "params": ["authorization-bearer", "hunter2"]}], + "requestmodifications": [{"name": "ForwardRequestHeaders", "params": []}] + }` + + rule := unmarshalRule(t, ruleJSON) + + jsonRule, err := json.Marshal(rule) + if err != nil { + t.Errorf("expected no error marshalling rule to json, got '%s'", err.Error()) + } + fmt.Println(string(jsonRule)) +} + +// =============================================== + +// unmarshalYAMLRule is a helper function to unmarshal a Rule from a YAML string. +func unmarshalYAMLRule(t *testing.T, ruleYAML string) *Rule { + rule := &Rule{} + err := yaml.Unmarshal([]byte(ruleYAML), rule) + if err != nil { + t.Fatalf("expected no error in Unmarshal, got '%s'", err) + } + return rule +} + +func TestRuleUnmarshalYAML(t *testing.T) { + ruleYAML := ` +domains: +- example.com +- www.example.com +responsemodifications: +- name: APIContent + params: [] +- name: SetContentSecurityPolicy + params: + - foobar +- name: SetIncomingCookie + params: + - authorization-bearer + - hunter2 +requestmodifications: +- name: ForwardRequestHeaders + params: [] +` + + rule := unmarshalYAMLRule(t, ruleYAML) + + if len(rule.Domains) != 2 { + t.Errorf("expected number of domains to be 2") + } + if !(rule.Domains[0] == "example.com" || rule.Domains[1] == "example.com") { + t.Errorf("expected domain to be example.com") + } + if len(rule.ResponseModifications) != 3 { + t.Errorf("expected number of ResponseModifications to be 3, got %d", len(rule.ResponseModifications)) + } + if len(rule.RequestModifications) != 1 { + t.Errorf("expected number of RequestModifications to be 1, got %d", len(rule.RequestModifications)) + } +} + +func TestRuleMarshalYAML(t *testing.T) { + ruleYAML := ` +domains: +- example.com +- www.example.com +responsemodifications: +- name: APIContent + params: [] +- name: SetContentSecurityPolicy + params: + - foobar +- name: SetIncomingCookie + params: + - authorization-bearer + - hunter2 +requestmodifications: +- name: ForwardRequestHeaders + params: [] +` + + rule := unmarshalYAMLRule(t, ruleYAML) + + yamlRule, err := yaml.Marshal(rule) + if err != nil { + t.Errorf("expected no error marshalling rule to yaml, got '%s'", err.Error()) + } + if yamlRule == nil { + t.Errorf("expected marshalling rule to yaml to not be nil") + } +} diff --git a/proxychain/ruleset/ruleset.go b/proxychain/ruleset/ruleset.go new file mode 100644 index 00000000..eb4d4edd --- /dev/null +++ b/proxychain/ruleset/ruleset.go @@ -0,0 +1,382 @@ +package ruleset_v2 + +import ( + //"bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "encoding/json" + "gopkg.in/yaml.v3" +) + +type IRuleset interface { + YAML() (string, error) + JSON() (string, error) + HasRule(url *url.URL) bool + GetRule(url *url.URL) (rule *Rule, exists bool) +} + +type Ruleset struct { + Rules []Rule `json:"rules" yaml:"rules"` + _rulemap map[string]*Rule // internal map for fast lookups; points at a rule in the Rules slice +} + +func (rs *Ruleset) UnmarshalYAML(unmarshal func(interface{}) error) error { + type AuxRuleset struct { + Rules []Rule `yaml:"rules"` + } + yamlRuleset := &AuxRuleset{} + + // First, try to unmarshal as AuxRuleset + err := unmarshal(yamlRuleset) + if err != nil { + // If that fails, try to unmarshal directly into a slice of Rules + var directRules []Rule + if err := unmarshal(&directRules); err != nil { + return err + } + yamlRuleset.Rules = directRules + } + + rs._rulemap = make(map[string]*Rule) + rs.Rules = yamlRuleset.Rules + + // create a map of pointers to rules loaded above based on domain string keys + // this way we don't have two copies of the rule in ruleset + for i, rule := range rs.Rules { + rulePtr := &rs.Rules[i] + for _, domain := range rule.Domains { + rs._rulemap[domain] = rulePtr + if !strings.HasPrefix(domain, "www.") { + rs._rulemap["www."+domain] = rulePtr + } + } + } + + return nil +} + +// MarshalYAML implements the yaml.Marshaler interface. +// It customizes the marshaling of a Ruleset object into YAML +func (rs *Ruleset) MarshalYAML() (interface{}, error) { + + type AuxRule struct { + Domains []string `yaml:"domains"` + RequestModifications []_rqm `yaml:"requestmodifications"` + ResponseModifications []_rsm `yaml:"responsemodifications"` + } + + type Aux struct { + Rules []AuxRule `yaml:"rules"` + } + + aux := Aux{} + + for _, rule := range rs.Rules { + auxRule := AuxRule{ + Domains: rule.Domains, + RequestModifications: rule._rqms, + ResponseModifications: rule._rsms, + } + aux.Rules = append(aux.Rules, auxRule) + } + return aux, nil + + /* + var b bytes.Buffer + y := yaml.NewEncoder(&b) + y.SetIndent(2) + err := y.Encode(&aux) + + return b.String(), err + */ +} + +// ========================================================== + +func (rs *Ruleset) UnmarshalJSON(data []byte) error { + type AuxRuleset struct { + Rules []Rule `json:"rules"` + } + ar := &AuxRuleset{} + + if err := json.Unmarshal(data, ar); err != nil { + return err + } + + rs._rulemap = make(map[string]*Rule) + rs.Rules = ar.Rules + + for i, rule := range rs.Rules { + rulePtr := &rs.Rules[i] + for _, domain := range rule.Domains { + rs._rulemap[domain] = rulePtr + if !strings.HasPrefix(domain, "www.") { + rs._rulemap["www."+domain] = rulePtr + } + } + } + + return nil +} + +func (rs *Ruleset) MarshalJSON() ([]byte, error) { + type AuxRule struct { + Domains []string `json:"domains"` + RequestModifications []_rqm `json:"requestmodifications"` + ResponseModifications []_rsm `json:"responsemodifications"` + } + + type Aux struct { + Rules []AuxRule `json:"rules"` + } + + aux := Aux{} + for _, rule := range rs.Rules { + auxRule := AuxRule{ + Domains: rule.Domains, + RequestModifications: rule._rqms, + ResponseModifications: rule._rsms, + } + aux.Rules = append(aux.Rules, auxRule) + } + + return json.Marshal(aux) +} + +// =========================================================== + +func (rs Ruleset) GetRule(url *url.URL) (rule *Rule, exists bool) { + rule, exists = rs._rulemap[url.Hostname()] + return rule, exists +} + +func (rs Ruleset) HasRule(url *url.URL) bool { + _, exists := rs.GetRule(url) + return exists +} + +// NewRuleset loads a new RuleSet from a path +func NewRuleset(path string) (Ruleset, error) { + rs := Ruleset{ + _rulemap: map[string]*Rule{}, + Rules: []Rule{}, + } + + switch { + case strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://"): + err := rs.loadRulesFromRemoteFile(path) + return rs, err + default: + err := rs.loadRulesFromLocalDir(path) + return rs, err + } +} + +// NewRulesetFromEnv creates a new RuleSet based on the RULESET environment variable. +// It logs a warning and returns an empty RuleSet if the RULESET environment variable is not set. +// If the RULESET is set but the rules cannot be loaded, it panics. +func NewRulesetFromEnv() Ruleset { + rulesPath, ok := os.LookupEnv("RULESET") + if !ok { + log.Printf("WARN: No ruleset specified. Set the `RULESET` environment variable to load one for a better success rate.") + return Ruleset{} + } + + ruleSet, err := NewRuleset(rulesPath) + if err != nil { + log.Println(err) + } + + return ruleSet +} + +// loadRulesFromLocalDir loads rules from a local directory specified by the path. +// It walks through the directory, loading rules from YAML files. +// Returns an error if the directory cannot be accessed +// If there is an issue loading any file, it will be skipped +func (rs *Ruleset) loadRulesFromLocalDir(path string) error { + _, err := os.Stat(path) + if err != nil { + return fmt.Errorf("loadRulesFromLocalDir: invalid path - %s", err) + } + + err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + isYAML := filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" + if !isYAML { + return nil + } + fmt.Printf("loadRulesFromLocalDir :: loading rule: %s\n", path) + + tmpRs := Ruleset{_rulemap: make(map[string]*Rule)} + err = tmpRs.loadRulesFromLocalFile(path) + if err != nil { + log.Printf("WARN: failed to load directory ruleset '%s': %s, skipping", path, err) + return nil + } + rs.Rules = append(rs.Rules, tmpRs.Rules...) + + //log.Printf("INFO: loaded ruleset %s\n", path) + + return nil + }) + + // create a map of pointers to rules loaded above based on domain string keys + // this way we don't have two copies of the rule in ruleset + if rs._rulemap == nil { + rs._rulemap = make(map[string]*Rule) + } + for i, rule := range rs.Rules { + rulePtr := &rs.Rules[i] + for _, domain := range rule.Domains { + rs._rulemap[domain] = rulePtr + if !strings.HasPrefix(domain, "www.") { + rs._rulemap["www."+domain] = rulePtr + } + } + } + + if err != nil { + return err + } + + return nil +} + +// loadRulesFromLocalFile loads rules from a local YAML file specified by the path. +// Returns an error if the file cannot be read or if there's a syntax error in the YAML. +func (rs *Ruleset) loadRulesFromLocalFile(path string) error { + file, err := os.ReadFile(path) + if err != nil { + e := fmt.Errorf("failed to read rules from local file: '%s'", path) + return errors.Join(e, err) + } + fmt.Printf("loadRulesFromLocalFile :: %s\n", path) + + isJSON := strings.HasSuffix(path, ".json") + if isJSON { + err = json.Unmarshal(file, rs) + } else { + err = yaml.Unmarshal(file, rs) + } + + if err != nil { + e := fmt.Errorf("failed to load rules from local file, possible syntax error in '%s' - %s", path, err) + debugPrintRule(string(file), e) + return e + } + + return nil +} + +// loadRulesFromRemoteFile loads rules from a remote URL. +// It supports plain and gzip compressed content. +// Returns an error if there's an issue accessing the URL or if there's a syntax error in the YAML. +func (rs *Ruleset) loadRulesFromRemoteFile(rulesURL string) error { + + resp, err := http.Get(rulesURL) + if err != nil { + return fmt.Errorf("failed to load rules from remote url '%s' - %s", rulesURL, err) + } + + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return fmt.Errorf("failed to load rules from remote url (%s) on '%s' - %s", resp.Status, rulesURL, err) + } + + var reader io.Reader + + // in case remote server did not set content-encoding gzip header + isGzip := strings.HasSuffix(rulesURL, ".gz") || strings.HasSuffix(rulesURL, ".gzip") || resp.Header.Get("content-encoding") == "gzip" + if isGzip { + reader, err = gzip.NewReader(resp.Body) + + if err != nil { + return fmt.Errorf("failed to create gzip reader for URL '%s' with status code '%s': %w", rulesURL, resp.Status, err) + } + } else { + reader = resp.Body + } + + isJSON := strings.HasSuffix(rulesURL, ".json") || resp.Header.Get("content-type") == "application/json" + if isJSON { + err = json.NewDecoder(reader).Decode(&rs) + } else { + err = yaml.NewDecoder(reader).Decode(&rs) + } + + if err != nil { + return fmt.Errorf("failed to load rules from remote url '%s' with status code '%s' and possible syntax error - %s", rulesURL, resp.Status, err) + } + + return nil +} + +// ================= utility methods ========================== + +// YAML returns the ruleset as a Yaml string +func (rs Ruleset) YAML() (string, error) { + yml, err := yaml.Marshal(&rs) + if err != nil { + return "", err + } + return string(yml), nil +} + +// JSON returns the ruleset as a JSON string +func (rs Ruleset) JSON() (string, error) { + jsn, err := json.Marshal(&rs) + if err != nil { + return "", err + } + return string(jsn), nil +} + +// Domains extracts and returns a slice of all domains present in the RuleSet. +func (rs *Ruleset) Domains() []string { + var domains []string + for _, rule := range rs.Rules { + domains = append(domains, rule.Domains...) + } + return domains +} + +// DomainCount returns the count of unique domains present in the RuleSet. +func (rs *Ruleset) DomainCount() int { + return len(rs.Domains()) +} + +// Count returns the total number of rules in the RuleSet. +func (rs *Ruleset) Count() int { + return len(rs.Rules) +} + +// PrintStats logs the number of rules and domains loaded in the RuleSet. +func (rs *Ruleset) PrintStats() { + log.Printf("INFO: Loaded %d rules for %d domains\n", rs.Count(), rs.DomainCount()) +} + +// debugPrintRule is a utility function for printing a rule and associated error for debugging purposes. +func debugPrintRule(rule string, err error) { + fmt.Println("------------------------------ BEGIN DEBUG RULESET -----------------------------") + fmt.Printf("%s\n", err.Error()) + fmt.Println("--------------------------------------------------------------------------------") + fmt.Println(rule) + fmt.Println("------------------------------ END DEBUG RULESET -------------------------------") +} diff --git a/proxychain/ruleset/ruleset_test.go b/proxychain/ruleset/ruleset_test.go new file mode 100644 index 00000000..454edd0d --- /dev/null +++ b/proxychain/ruleset/ruleset_test.go @@ -0,0 +1,214 @@ +package ruleset_v2 + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + //"strings" + "testing" + "time" + + "github.com/gofiber/fiber/v2" + "github.com/stretchr/testify/assert" + //"gopkg.in/yaml.v3" +) + +var ( + validYAML = `rules: + - domains: + - example.com + - www.example.com + responsemodifications: + - name: APIContent + params: [] + - name: SetContentSecurityPolicy + params: + - foobar + - name: SetIncomingCookie + params: + - authorization-bearer + - hunter2 + requestmodifications: + - name: ForwardRequestHeaders + params: [] +` + + invalidYAML = ` +rules: + domains: + - example.com + - www.example.com + responsemodifications: + - name: APIContent + - name: SetContentSecurityPolicy + - name: INVALIDSetIncomingCookie + params: + - authorization-bearer + - hunter2 + requestmodifications: + - name: ForwardRequestHeaders + params: [] +` +) + +func TestYAMLUnmarshal(t *testing.T) { + rs, err := loadRuleFromString(validYAML) + fmt.Println(validYAML) + assert.NoError(t, err, "expected no error loading valid yml") + yml, err := rs.YAML() + assert.NoError(t, err, "expected no error marshalling ruleset") + + rs2, err := loadRuleFromString(yml) + assert.NoError(t, err, "expected no error loading yaml marshalled -> unmarshalled -> marshalled ruleset") + assert.Equal(t, 1, rs2.Count(), "expected one rule to be returned after marshalled -> unmarshalled -> marshalled ruleset") +} + +func TestJSONUnmarshal(t *testing.T) { + rs, err := loadRuleFromString(validYAML) + assert.NoError(t, err, "expected no error loading valid yml") + j, err := json.Marshal(&rs) + assert.NoError(t, err, "expected no error marshalling ruleset to json") + + fmt.Println(string(j)) + + rs2, err := loadRuleFromString(string(j)) + assert.NoError(t, err, "expected no error loading JSON marshalled -> unmarshalled -> marshalled ruleset") + + assert.Equal(t, 1, rs2.Count(), "expected one rule to be returned after JSON marshalled -> unmarshalled -> marshalled ruleset") +} + +func TestLoadRulesFromRemoteFile(t *testing.T) { + app := fiber.New() + defer app.Shutdown() + + app.Get("/valid-config.yml", func(c *fiber.Ctx) error { + c.SendString(validYAML) + return nil + }) + + app.Get("/invalid-config.yml", func(c *fiber.Ctx) error { + c.SendString(invalidYAML) + return nil + }) + + // Start the server in a goroutine + go func() { + if err := app.Listen("127.0.0.1:9999"); err != nil { + t.Errorf("Server failed to start: %s", err.Error()) + } + }() + + // Wait for the server to start + time.Sleep(time.Second * 1) + + rs, err := NewRuleset("http://127.0.0.1:9999/valid-config.yml") + if err != nil { + t.Errorf("failed to load plaintext ruleset from http server: %s", err.Error()) + } + + u, _ := url.Parse("http://example.com") + r, exists := rs.GetRule(u) + assert.True(t, exists, "expected example.com rule to be present") + assert.Equal(t, r.Domains[0], "example.com") + + u, _ = url.Parse("http://www.www.foobar.com") + _, exists = rs.GetRule(u) + assert.False(t, exists, "expected www.www.foobar.com rule to NOT be present") + + u, _ = url.Parse("http://example.com") + r, exists = rs.GetRule(u) + assert.Equal(t, r.Domains[0], "example.com") + + os.Setenv("RULESET", "http://127.0.0.1:9999/valid-config.yml") + + rs = NewRulesetFromEnv() + r, exists = rs.GetRule(u) + assert.True(t, exists, "expected example.com rule to be present from env") + if !assert.Equal(t, r.Domains[0], "example.com") { + t.Error("expected no errors loading ruleset from url using environment variable, but got one") + } +} + +func loadRuleFromString(yamlOrJSON string) (Ruleset, error) { + // Create a temporary file and load it + var tmpFile *os.File + if strings.HasPrefix(yamlOrJSON, "{") { + tmpFile, _ = os.CreateTemp("", "ruleset*.json") + } else { + tmpFile, _ = os.CreateTemp("", "ruleset*.yaml") + } + + defer os.Remove(tmpFile.Name()) + + tmpFile.WriteString(yamlOrJSON) + + rs := Ruleset{ + _rulemap: map[string]*Rule{}, + Rules: []Rule{}, + } + err := rs.loadRulesFromLocalFile(tmpFile.Name()) + + return rs, err +} + +// TestLoadRulesFromLocalFile tests the loading of rules from a local YAML file. +func TestLoadRulesFromLocalFile(t *testing.T) { + _, err := loadRuleFromString(validYAML) + if err != nil { + t.Errorf("Failed to load rules from valid YAML: %s", err) + } + + _, err = loadRuleFromString(invalidYAML) + if err == nil { + t.Errorf("Expected an error when loading invalid YAML, but got none") + } +} + +// TestLoadRulesFromLocalDir tests the loading of rules from a local nested directory full of yaml rulesets +func TestLoadRulesFromLocalDir(t *testing.T) { + // Create a temporary directory + baseDir, err := os.MkdirTemp("", "ruleset_test") + if err != nil { + t.Fatalf("Failed to create temporary directory: %s", err) + } + + defer os.RemoveAll(baseDir) + + // Create a nested subdirectory + nestedDir := filepath.Join(baseDir, "nested") + err = os.Mkdir(nestedDir, 0o755) + + if err != nil { + t.Fatalf("Failed to create nested directory: %s", err) + } + + // Create a nested subdirectory + nestedTwiceDir := filepath.Join(nestedDir, "nestedTwice") + err = os.Mkdir(nestedTwiceDir, 0o755) + if err != nil { + t.Fatalf("Failed to create twice-nested directory: %s", err) + } + + testCases := []string{"test.yaml", "test2.yaml", "test-3.yaml", "test 4.yaml", "1987.test.yaml.yml", "foobar.example.com.yaml", "foobar.com.yml"} + for _, fileName := range testCases { + filePath := filepath.Join(nestedDir, "2x-"+fileName) + os.WriteFile(filePath, []byte(validYAML), 0o644) + + filePath = filepath.Join(nestedDir, fileName) + os.WriteFile(filePath, []byte(validYAML), 0o644) + + filePath = filepath.Join(baseDir, "base-"+fileName) + os.WriteFile(filePath, []byte(validYAML), 0o644) + } + + rs := Ruleset{} + fmt.Println(baseDir) + err = rs.loadRulesFromLocalDir(baseDir) + + assert.NoError(t, err) + assert.Equal(t, len(testCases)*3, rs.Count()) +} diff --git a/proxychain/ruleset/todo.md b/proxychain/ruleset/todo.md new file mode 100644 index 00000000..6d94d89f --- /dev/null +++ b/proxychain/ruleset/todo.md @@ -0,0 +1 @@ +ruleset loading rule tests are failing; maybe concurrency issue with assigning to nil map? diff --git a/ruleset_v2.yaml b/ruleset_v2.yaml new file mode 100644 index 00000000..9fb33902 --- /dev/null +++ b/ruleset_v2.yaml @@ -0,0 +1,36 @@ +rules: + - domains: + - example.com + - www.example.com + responsemodifications: + - name: APIContent + params: [] + - name: SetContentSecurityPolicy + params: + - foobar + - name: SetIncomingCookie + params: + - authorization-bearer + - hunter2 + requestmodifications: + - name: ForwardRequestHeaders + params: [] + + - domains: + - quantamagzine.org + responsemodifications: + - name: BlockElementRemoval + params: + - "#postContent" + + - domains: + - techcrunch.com + responsemodifications: + - name: ModifyIncomingScriptsWithRegex + params: + - "window\\.location" + - | + {origin: "techcrunch.com"} + - name: BlockElementRemoval + params: + - ".article-content" diff --git a/rulesets_v2/BE_Belgium/demorgen-be.yaml b/rulesets_v2/BE_Belgium/demorgen-be.yaml new file mode 100644 index 00000000..76087ff8 --- /dev/null +++ b/rulesets_v2/BE_Belgium/demorgen-be.yaml @@ -0,0 +1,23 @@ +- domains: + - demorgen.be + + requestmodifiers: + - name: MasqueradeAsGoogleBot + + - name: SpoofReferrer + params: ["https://news.google.com"] + + - name: SetOutgoingCookie + params: ["isBot", "true"] + + - name: SetOutgoingCookie + params: ["authId", "1"] + + responsemodifiers: + - name: BypassContentSecurityPolicy + - name: InjectScriptAfterDOMContentLoaded + params: + - | + let paywall = document.querySelectorAll('script[src*="advertising-cdn.dpgmedia.cloud"], div[data-temptation-position="ARTICLE_BOTTOM"]'); + paywall.forEach(el => { el.remove(); }); + document.querySelector('div[data-advert-placeholder-collapses]').remove(); diff --git a/rulesets_v2/BE_Belgium/dpg-media.yaml b/rulesets_v2/BE_Belgium/dpg-media.yaml new file mode 100644 index 00000000..e680d928 --- /dev/null +++ b/rulesets_v2/BE_Belgium/dpg-media.yaml @@ -0,0 +1,44 @@ +- domains: + - myprivacy.dpgmedia.be + - myprivacy.dpgmedia.nl + + requestmodifiers: + - name: SpoofReferrer + params: ["https://news.google.com"] + + - name: SetOutgoingCookie + params: ["isBot", "true"] + + - name: SetOutgoingCookie + params: ["authId", "1"] + + - name: SpoofXForwardedFor + params: ["none"] + + +- domains: + - demorgen.be + + requestmodifiers: + - name: MasqueradeAsGoogleBot + + - name: SpoofReferrer + params: ["https://news.google.com"] + + - name: SetOutgoingCookie + params: ["isBot", "true"] + + - name: SetOutgoingCookie + params: ["authId", "1"] + + - name: SpoofXForwardedFor + params: ["none"] + + responsemodifiers: + - name: BypassContentSecurityPolicy + - name: InjectScriptAfterDOMContentLoaded + params: + - | + let paywall = document.querySelectorAll('script[src*="advertising-cdn.dpgmedia.cloud"], div[data-temptation-position="ARTICLE_BOTTOM"]'); + paywall.forEach(el => { el.remove(); }); + document.querySelector('div[data-advert-placeholder-collapses]').remove(); diff --git a/rulesets_v2/CA_Canada/_multi-metroland-media-group.yaml b/rulesets_v2/CA_Canada/_multi-metroland-media-group.yaml new file mode 100644 index 00000000..977c07a6 --- /dev/null +++ b/rulesets_v2/CA_Canada/_multi-metroland-media-group.yaml @@ -0,0 +1,34 @@ +rules: + - domains: + - thestar.com + - niagarafallsreview.ca + - stcatharinesstandard.ca + - thepeterboroughexaminer.com + - therecord.com + - thespec.com + - wellandtribune.ca + responsemodifications: + - name: DeleteLocalStorageData + - name: DeleteSessionStorageData + - name: InjectScriptAfterDOMContentLoaded + params: + - | + const paywall = document.querySelectorAll('div.subscriber-offers'); + paywall.forEach(el => { el.remove(); }); + const subscriber_only = document.querySelectorAll('div.subscriber-only'); + for (const elem of subscriber_only) { + if (elem.classList.contains('encrypted-content') && dompurify_loaded) { + const parser = new DOMParser(); + const doc = parser.parseFromString('
' + DOMPurify.sanitize(unscramble(elem.innerText)) + '
', 'text/html'); + const content_new = doc.querySelector('div'); + elem.parentNode.replaceChild(content_new, elem); + } + elem.removeAttribute('style'); + elem.removeAttribute('class'); + } + const banners = document.querySelectorAll('div.subscription-required, div.redacted-overlay, div.subscriber-hide, div.tnt-ads-container'); + banners.forEach(el => { el.remove(); }); + const ads = document.querySelectorAll('div.tnt-ads-container, div[class*="adLabelWrapper"]'); + ads.forEach(el => { el.remove(); }); + const recommendations = document.querySelectorAll('div[id^="tncms-region-article"]'); + recommendations.forEach(el => { el.remove(); }); diff --git a/styles/input.css b/styles/input.css index bd6213e1..172164e6 100644 --- a/styles/input.css +++ b/styles/input.css @@ -1,3 +1,79 @@ @tailwind base; @tailwind components; -@tailwind utilities; \ No newline at end of file +@tailwind utilities; + +@layer base { + a { + @apply text-slate-600 dark:text-slate-400 hover:text-blue-500 dark:hover:text-blue-500 underline underline-offset-4 transition-colors duration-300; + } + h1 { + @apply scroll-m-20 text-4xl font-extrabold tracking-tight lg:text-5xl text-slate-900 dark:text-slate-200; + } + h2 { + @apply scroll-m-20 border-b pb-2 text-3xl font-semibold tracking-tight first:mt-0 text-slate-900 dark:text-slate-200; + } + h3 { + @apply scroll-m-20 text-2xl font-semibold tracking-tight text-slate-900 dark:text-slate-200; + } + h4, + h5, + h6 { + @apply scroll-m-20 text-xl font-semibold tracking-tight text-slate-900 dark:text-slate-200; + } + p { + @apply leading-7 [&:not(:first-child)]:mt-6 text-slate-900 dark:text-slate-200; + } + kbd, + pre, + code { + @apply relative whitespace-break-spaces break-words rounded bg-slate-200 dark:bg-slate-800 py-[0.2rem] font-mono text-sm font-semibold; + } + blockquote { + @apply mt-6 border-l-2 pl-6 italic; + } + ul { + @apply my-6 ml-6 list-disc [&>li]:mt-2 text-slate-900 dark:text-slate-200; + } + ol { + @apply my-6 ml-6 list-decimal [&>li]:mt-2 text-slate-900 dark:text-slate-200; + } + dl { + @apply my-6 text-slate-900 dark:text-slate-200 font-bold [&>dd]:font-normal [&>dd]:ml-6 [&>dt]:mt-3; + } + li { + @apply text-slate-900 dark:text-slate-200; + } + table { + @apply w-full caption-bottom text-sm; + } + thead { + @apply [&_tr]:border-b; + } + tbody { + @apply [&_tr:last-child]:border-0; + } + tfoot { + @apply border-t border-slate-400 dark:border-slate-700 bg-slate-700/50 dark:bg-slate-200/50 font-medium [&>tr]:last:border-b-0; + } + tr { + @apply border-b border-slate-400 dark:border-slate-700 transition-colors hover:bg-slate-200/50 dark:hover:bg-slate-700/50 data-[state=selected]:bg-slate-700 dark:data-[state=selected]:bg-slate-200; + } + th { + @apply h-12 px-4 text-left align-middle font-medium text-slate-600 dark:text-slate-200 [&:has([role=checkbox])]:pr-0; + } + td { + @apply p-4 align-middle [&:has([role=checkbox])]:pr-0; + } + caption { + @apply mt-4 text-sm text-slate-600 dark:text-slate-200; + } + img { + @apply h-auto w-auto object-cover max-w-full mx-auto rounded-md shadow-md dark:shadow-slate-700; + } + figcaption { + @apply mt-2 text-sm text-slate-600 dark:text-slate-400; + } + hr { + @apply shrink-0 bg-slate-200 dark:bg-slate-700 h-[1px] w-full; + } +} diff --git a/tailwind.config.js b/tailwind.config.js index fea75a72..1e404352 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -1,9 +1,13 @@ /** @type {import('tailwindcss').Config} */ module.exports = { - content: ["./handlers/**/*.html"], - theme: { - extend: {}, - }, - plugins: [], - } - \ No newline at end of file + content: [ + "./cmd/**/*.{html,js}", + "./handlers/**/*.{html,js}", + "./proxychain/**/*.{html,js}", + ], + darkMode: "class", + theme: { + extend: {}, + }, + plugins: [], +}; diff --git a/tests/package-lock.json b/tests/package-lock.json new file mode 100644 index 00000000..dc73f9a1 --- /dev/null +++ b/tests/package-lock.json @@ -0,0 +1,91 @@ +{ + "name": "tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tests", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.40.0", + "@types/node": "^20.10.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.40.0.tgz", + "integrity": "sha512-PdW+kn4eV99iP5gxWNSDQCbhMaDVej+RXL5xr6t04nbKLCBwYtA046t7ofoczHOm8u6c+45hpDKQVZqtqwkeQg==", + "dev": true, + "dependencies": { + "playwright": "1.40.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@types/node": { + "version": "20.10.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.0.tgz", + "integrity": "sha512-D0WfRmU9TQ8I9PFx9Yc+EBHw+vSpIub4IDvQivcp26PtPrdMGAq5SDcpXEo/epqa/DXotVpekHiLNTg3iaKXBQ==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.40.0.tgz", + "integrity": "sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw==", + "dev": true, + "dependencies": { + "playwright-core": "1.40.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.40.0.tgz", + "integrity": "sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/tests/package.json b/tests/package.json new file mode 100644 index 00000000..51b2a6aa --- /dev/null +++ b/tests/package.json @@ -0,0 +1,14 @@ +{ + "name": "tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.40.0", + "@types/node": "^20.10.0" + } +} diff --git a/tests/playwright.config.ts b/tests/playwright.config.ts new file mode 100644 index 00000000..7f975a6f --- /dev/null +++ b/tests/playwright.config.ts @@ -0,0 +1,77 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./tests", + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + /* + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + */ + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/tests/run_test.sh b/tests/run_test.sh new file mode 100644 index 00000000..04758590 --- /dev/null +++ b/tests/run_test.sh @@ -0,0 +1,2 @@ +npx playwright test +npx playwright show-report diff --git a/tests/tests/www-wellandtribune-ca.spec.ts b/tests/tests/www-wellandtribune-ca.spec.ts new file mode 100644 index 00000000..f83ee1f4 --- /dev/null +++ b/tests/tests/www-wellandtribune-ca.spec.ts @@ -0,0 +1,18 @@ +import { expect, test } from "@playwright/test"; + +const paywallText = "This article is exclusive to subscribers."; +const articleURL = + "https://www.wellandtribune.ca/news/niagara-region/niagara-transit-commission-rejects-council-request-to-reduce-its-budget-increase/article_e9fb424c-8df5-58ae-a6c3-3648e2a9df66.html"; + +const ladderURL = "http://localhost:8080"; +let domain = (new URL(articleURL)).host; + +test(`${domain} has paywall by default`, async ({ page }) => { + await page.goto(articleURL); + await expect(page.getByText(paywallText)).toBeVisible(); +}); + +test(`${domain} + Ladder doesn't have paywall`, async ({ page }) => { + await page.goto(`${ladderURL}/${articleURL}`); + await expect(page.getByText(paywallText)).toBeVisible(); +});