diff --git a/.air.toml b/.air.toml index fa8d9aa1..b7dbf81e 100644 --- a/.air.toml +++ b/.air.toml @@ -7,21 +7,21 @@ tmp_dir = "tmp" bin = "./tmp/main" cmd = "go build -o ./tmp/main ./cmd" delay = 1000 - exclude_dir = ["assets", "tmp", "vendor", "testdata"] - exclude_file = [] + exclude_dir = ["assets", "tmp", "vendor", "testdata",] + exclude_file = ["proxychain/ruleset/rule_resmod_types.gen.go", "proxychain/ruleset/rule_reqmod_types.gen.go", "handlers/api_modifiers_structdef.gen.go"] exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false - full_bin = "RULESET=./ruleset.yaml ./tmp/main" + full_bin = "./tmp/main --ruleset ./rulesets_v2" include_dir = [] - include_ext = ["go", "tpl", "tmpl", "yaml", "html"] + include_ext = ["go", "tpl", "tmpl", "yaml", "html", "js"] include_file = [] kill_delay = "0s" log = "build-errors.log" poll = false poll_interval = 0 post_cmd = [] - pre_cmd = ["echo 'dev' > handlers/VERSION"] + pre_cmd = ["git submodule update --init --recursive; git rev-parse --short HEAD > handlers/VERSION; git rev-parse --short HEAD > cmd/VERSION; cd proxychain/codegen && go run codegen.go && cd ../../handlers/api_modifiers_codegen && go run api_modifiers_codegen.go"] rerun = false rerun_delay = 500 send_interrupt = false diff --git a/.github/workflows/build-css.yaml b/.github/workflows/build-css.yaml index 47a33659..5ad392dd 100644 --- a/.github/workflows/build-css.yaml +++ b/.github/workflows/build-css.yaml @@ -3,7 +3,10 @@ name: Build Tailwind CSS on: push: paths: + - "handlers/error_page.html" - "handlers/form.html" + - "handlers/playground.html" + - "proxychain/responsemodifiers/vendor/generate_readable_outline.html" workflow_dispatch: jobs: @@ -27,16 +30,17 @@ jobs: name: Build Tailwind CSS run: pnpm build - - name: Commit generated stylesheet + name: Commit generated stylesheet for handlers/styles.css run: | - if git diff --quiet cmd/styles.css; then + if git diff --quiet handlers/styles.css; then echo "No changes to commit." exit 0 else echo "Changes detected, committing..." git config --global user.name "Github action" git config --global user.email "username@users.noreply.github.com" - git add cmd + git add handlers + git add proxychain/responsemodifiers/vendor/ git commit -m "Generated stylesheet" git push - fi \ No newline at end of file + fi diff --git a/.gitignore b/.gitignore index 212e5021..f943b087 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # dev binary ladder +tmp/main +tmp VERSION -output.css \ No newline at end of file +output.css +.aider* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..1c1ef8c7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "proxychain/responsemodifiers/vendor/ddg-tracker-surrogates"] + path = proxychain/responsemodifiers/vendor/ddg-tracker-surrogates + url = https://github.com/duckduckgo/tracker-surrogates +[submodule "proxychain/requestmodifiers/vendor/ua-parser-js"] + path = proxychain/requestmodifiers/vendor/ua-parser-js + url = https://github.com/faisalman/ua-parser-js.git diff --git a/Dockerfile b/Dockerfile index 07005717..c8f9ad4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY . . RUN go mod download -RUN CGO_ENABLED=0 GOOS=linux go build -o ladder cmd/main.go +RUN make build FROM debian:12-slim as release @@ -18,8 +18,4 @@ RUN chmod +x /app/ladder RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/* -#EXPOSE 8080 - -#ENTRYPOINT ["/usr/bin/dumb-init", "--"] - CMD ["sh", "-c", "/app/ladder"] \ No newline at end of file diff --git a/Makefile b/Makefile index 98f30974..b2c7a3d0 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,11 @@ +build: + cd proxychain/codegen && go run codegen.go + cd handlers/api_modifiers_codegen && go run api_modifiers_codegen.go + git submodule update --init --recursive + git rev-parse --short HEAD > handlers/VERSION + git rev-parse --short HEAD > cmd/VERSION + go build -o ladder -ldflags="-s -w" cmd/main.go + lint: gofumpt -l -w . golangci-lint run -c .golangci-lint.yaml --fix @@ -7,4 +15,7 @@ lint: install-linters: go install mvdan.cc/gofumpt@latest - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 \ No newline at end of file + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 + +run: + go run ./cmd/. diff --git a/README.md b/README.md index be1b96fd..d07fe0a9 100644 --- a/README.md +++ b/README.md @@ -91,18 +91,28 @@ Or create a bookmark with the following URL: ```javascript javascript:window.location.href="http://localhost:8080/"+location.href ``` +### Outline +```bash +curl -X GET "http://localhost:8080/outline/https://www.example.com" +``` ### API ```bash -curl -X GET "http://localhost:8080/api/https://www.example.com" +curl -X GET "http://localhost:8080/api/content/https://www.example.com" ``` ### RAW -http://localhost:8080/raw/https://www.example.com +http://localhost:8080/api/raw/https://www.example.com ### Running Ruleset -http://localhost:8080/ruleset +http://localhost:8080/api/ruleset + +### Running Rule +http://localhost:8080/api/ruleset/https://example.com + +### List available modifiers +http://localhost:8080/api/modifiers ## Configuration @@ -189,7 +199,10 @@ There is a basic ruleset available in a separate repository [ruleset.yaml](https To run a development server at http://localhost:8080: ```bash -echo "dev" > handlers/VERSION +git clone git@github.com-ladddder:everywall/ladder.git +git submodule update --init --recursive +echo "dev " > handlers/VERSION +echo "dev " > cmd/VERSION RULESET="./ruleset.yaml" go run cmd/main.go ``` diff --git a/cmd/main.go b/cmd/main.go index d28787d1..08aa78ae 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,26 +1,23 @@ package main import ( - "embed" + _ "embed" "fmt" "log" "os" - "strings" - "ladder/handlers" - "ladder/handlers/cli" + "github.com/everywall/ladder/handlers" + "github.com/everywall/ladder/internal/cli" + + "github.com/everywall/ladder/proxychain/requestmodifiers/bot" + ruleset_v2 "github.com/everywall/ladder/proxychain/ruleset" "github.com/akamensky/argparse" "github.com/gofiber/fiber/v2" - "github.com/gofiber/fiber/v2/middleware/basicauth" - "github.com/gofiber/fiber/v2/middleware/favicon" ) -//go:embed favicon.ico -var faviconData string - -//go:embed styles.css -var cssData embed.FS +//go:embed VERSION +var version string func main() { parser := argparse.NewParser("ladder", "Every Wall needs a Ladder") @@ -41,6 +38,23 @@ func main() { Help: "This will spawn multiple processes listening", }) + verbose := parser.Flag("v", "verbose", &argparse.Options{ + Required: false, + Help: "Adds verbose logging", + }) + + randomGoogleBot := parser.Flag("", "random-googlebot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Googlebot IPs, and use a random one for each masqueraded request", + }) + + randomBingBot := parser.Flag("", "random-bingbot", &argparse.Options{ + Required: false, + Help: "Update the list of trusted Bingbot IPs, and use a random one for each masqueraded request", + }) + + // TODO: add version flag that reads from handers/VERSION + ruleset := parser.String("r", "ruleset", &argparse.Options{ Required: false, Help: "File, Directory or URL to a ruleset.yaml. Overrides RULESET environment variable.", @@ -51,14 +65,9 @@ func main() { Help: "Compiles a directory of yaml files into a single ruleset.yaml. Requires --ruleset arg.", }) - mergeRulesetsGzip := parser.Flag("", "merge-rulesets-gzip", &argparse.Options{ - Required: false, - Help: "Compiles a directory of yaml files into a single ruleset.gz Requires --ruleset arg.", - }) - mergeRulesetsOutput := parser.String("", "merge-rulesets-output", &argparse.Options{ Required: false, - Help: "Specify output file for --merge-rulesets and --merge-rulesets-gzip. Requires --ruleset and --merge-rulesets args.", + Help: "Specify output file for --merge-rulesets. Requires --ruleset and --merge-rulesets args.", }) err := parser.Parse(os.Args) @@ -66,20 +75,36 @@ func main() { fmt.Print(parser.Usage(err)) } + if *randomGoogleBot { + err := bot.GoogleBot.UpdatePool("https://developers.google.com/static/search/apis/ipranges/googlebot.json") + if err != nil { + fmt.Println("error while retrieving list of Googlebot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Googlebot identity") + } + } + + if *randomBingBot { + err := bot.BingBot.UpdatePool("https://www.bing.com/toolbox/bingbot.json") + if err != nil { + fmt.Println("error while retrieving list of Bingbot IPs: " + err.Error()) + fmt.Println("defaulting to known trusted Bingbot identity") + } + } + // utility cli flag to compile ruleset directory into single ruleset.yaml - if *mergeRulesets || *mergeRulesetsGzip { + if *mergeRulesets { output := os.Stdout if *mergeRulesetsOutput != "" { output, err = os.Create(*mergeRulesetsOutput) - + if err != nil { fmt.Println(err) os.Exit(1) } } - err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, *mergeRulesetsGzip, output) + err = cli.HandleRulesetMerge(*ruleset, *mergeRulesets, output) if err != nil { fmt.Println(err) os.Exit(1) @@ -91,28 +116,30 @@ func main() { *prefork = true } + var rs ruleset_v2.IRuleset + + switch { + case *ruleset != "": + rs, err = ruleset_v2.NewRuleset(*ruleset) + if err != nil { + fmt.Printf("ERROR: failed to load ruleset from %s\n", *ruleset) + } + case os.Getenv("RULESET") != "": + rs = ruleset_v2.NewRulesetFromEnv() + } + app := fiber.New( fiber.Config{ - Prefork: *prefork, - GETOnly: true, + Prefork: *prefork, + GETOnly: false, + ReadBufferSize: 4096 * 4, // increase max header size + DisableStartupMessage: true, }, ) - userpass := os.Getenv("USERPASS") - if userpass != "" { - userpass := strings.Split(userpass, ":") - - app.Use(basicauth.New(basicauth.Config{ - Users: map[string]string{ - userpass[0]: userpass[1], - }, - })) - } - - app.Use(favicon.New(favicon.Config{ - Data: []byte(faviconData), - URL: "/favicon.ico", - })) + app.Use(handlers.Auth()) + app.Use(handlers.Favicon()) + app.Use(handlers.RenderErrorPage()) if os.Getenv("NOLOGS") != "true" { app.Use(func(c *fiber.Ctx) error { @@ -122,23 +149,28 @@ func main() { }) } + proxyOpts := &handlers.ProxyOptions{ + Verbose: *verbose, + Ruleset: rs, + } + app.Get("/", handlers.Form) - app.Get("/styles.css", func(c *fiber.Ctx) error { - cssData, err := cssData.ReadFile("styles.css") - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString("Internal Server Error") - } + app.Get("styles.css", handlers.Styles) + app.Get("script.js", handlers.Script) + app.Get("playground-script.js", handlers.Script) - c.Set("Content-Type", "text/css") + app.All("api/raw/*", handlers.NewRawProxySiteHandler(proxyOpts)) - return c.Send(cssData) - }) + app.Get("api/modifiers", handlers.NewAPIModifersListHandler(proxyOpts)) + app.Get("api/ruleset/*", handlers.NewRulesetSiteHandler(proxyOpts)) + app.Get("api/content/*", handlers.NewAPIContentHandler("api/outline/*", proxyOpts)) + + app.Get("outline/*", handlers.NewOutlineHandler("outline/*", proxyOpts)) + app.All("playground/*", handlers.PlaygroundHandler("playground/*", proxyOpts)) - app.Get("ruleset", handlers.Ruleset) - app.Get("raw/*", handlers.Raw) - app.Get("api/*", handlers.Api) - app.Get("/*", handlers.ProxySite(*ruleset)) + app.All("/*", handlers.NewProxySiteHandler(proxyOpts)) + fmt.Println(cli.StartupMessage(version, *port, *ruleset)) log.Fatal(app.Listen(":" + *port)) } diff --git a/cmd/styles.css b/cmd/styles.css deleted file mode 100644 index 33803d81..00000000 --- a/cmd/styles.css +++ /dev/null @@ -1 +0,0 @@ -*,::after,::before{box-sizing:border-box;border-width:0;border-style:solid;border-color:#e5e7eb}::after,::before{--tw-content:''}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-feature-settings:inherit;font-variation-settings:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0;padding:0}legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::after,::before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgb(59 130 246 / 0.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.inset-y-0{top:0;bottom:0}.right-0{right:0}.mx-4{margin-left:1rem;margin-right:1rem}.mx-auto{margin-left:auto;margin-right:auto}.mt-10{margin-top:2.5rem}.block{display:block}.grid{display:grid}.hidden{display:none}.w-full{width:100%}.max-w-3xl{max-width:48rem}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.items-center{align-items:center}.gap-4{gap:1rem}.rounded-md{border-radius:.375rem}.bg-white{--tw-bg-opacity:1;background-color:rgb(255 255 255 / var(--tw-bg-opacity))}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-1\.5{padding-top:.375rem;padding-bottom:.375rem}.pl-2{padding-left:.5rem}.pr-2{padding-right:.5rem}.pr-3{padding-right:.75rem}.pt-10{padding-top:2.5rem}.text-center{text-align:center}.text-3xl{font-size:1.875rem;line-height:2.25rem}.text-sm{font-size:.875rem;line-height:1.25rem}.font-extrabold{font-weight:800}.leading-6{line-height:1.5rem}.tracking-tight{letter-spacing:-.025em}.text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.text-slate-500{--tw-text-opacity:1;color:rgb(100 116 139 / var(--tw-text-opacity))}.text-slate-600{--tw-text-opacity:1;color:rgb(71 85 105 / var(--tw-text-opacity))}.text-slate-900{--tw-text-opacity:1;color:rgb(15 23 42 / var(--tw-text-opacity))}.underline-offset-2{text-underline-offset:2px}.antialiased{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.shadow-sm{--tw-shadow:0 1px 2px 0 rgb(0 0 0 / 0.05);--tw-shadow-colored:0 1px 2px 0 var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.ring-1{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.ring-slate-900\/10{--tw-ring-color:rgb(15 23 42 / 0.1)}.transition-colors{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(0.4,0,0.2,1);transition-duration:150ms}.duration-300{transition-duration:.3s}.hover\:text-blue-500:hover{--tw-text-opacity:1;color:rgb(59 130 246 / var(--tw-text-opacity))}.hover\:text-slate-400:hover{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.hover\:underline:hover{text-decoration-line:underline}.hover\:ring-slate-300:hover{--tw-ring-opacity:1;--tw-ring-color:rgb(203 213 225 / var(--tw-ring-opacity))}@media (prefers-color-scheme:dark){.dark\:bg-slate-800{--tw-bg-opacity:1;background-color:rgb(30 41 59 / var(--tw-bg-opacity))}.dark\:bg-slate-900{--tw-bg-opacity:1;background-color:rgb(15 23 42 / var(--tw-bg-opacity))}.dark\:text-slate-200{--tw-text-opacity:1;color:rgb(226 232 240 / var(--tw-text-opacity))}.dark\:text-slate-400{--tw-text-opacity:1;color:rgb(148 163 184 / var(--tw-text-opacity))}.dark\:hover\:bg-slate-700:hover{--tw-bg-opacity:1;background-color:rgb(51 65 85 / var(--tw-bg-opacity))}.hover\:dark\:text-slate-300:hover{--tw-text-opacity:1;color:rgb(203 213 225 / var(--tw-text-opacity))}}@media (min-width:640px){.sm\:text-4xl{font-size:2.25rem;line-height:2.5rem}} diff --git a/docker-compose.yaml b/docker-compose.yaml index 6325d095..f45e99ae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,7 @@ services: ladder: image: ghcr.io/everywall/ladder:latest container_name: ladder - #build: . + build: . #restart: always #command: sh -c ./ladder environment: diff --git a/go.mod b/go.mod index 5d6ca5dd..15fb7f35 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,59 @@ -module ladder +module github.com/everywall/ladder go 1.21.1 require ( - github.com/PuerkitoBio/goquery v1.8.1 github.com/akamensky/argparse v1.4.0 - github.com/gofiber/fiber/v2 v2.50.0 + github.com/bogdanfinn/fhttp v0.5.24 + github.com/bogdanfinn/tls-client v1.6.1 + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c + github.com/gofiber/fiber/v2 v2.51.0 + github.com/markusmobius/go-trafilatura v1.5.1 github.com/stretchr/testify v1.8.4 + golang.org/x/net v0.19.0 + golang.org/x/term v0.15.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/abadojack/whatlanggo v1.0.1 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/bogdanfinn/utls v1.5.16 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elliotchance/pie/v2 v2.8.0 // indirect + github.com/forPelevin/gomoji v1.1.8 // indirect + github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 // indirect + github.com/gofiber/template v1.8.2 // indirect + github.com/gofiber/template/html/v2 v2.0.5 + github.com/gofiber/utils v1.1.0 // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/google/uuid v1.4.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/hablullah/go-hijri v1.0.2 // indirect + github.com/hablullah/go-juliandays v1.0.0 // indirect + github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/magefile/mage v1.15.0 // indirect + github.com/markusmobius/go-dateparser v1.2.1 // indirect + github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 // indirect + github.com/markusmobius/go-htmldate v1.2.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/rs/zerolog v1.31.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect + github.com/tetratelabs/wazero v1.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.50.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.14.0 + github.com/wasilibs/go-re2 v1.4.1 // indirect + github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + ) diff --git a/go.sum b/go.sum index f171d696..df0254a9 100644 --- a/go.sum +++ b/go.sum @@ -1,87 +1,158 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= +github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/bogdanfinn/fhttp v0.5.24 h1:OlyBKjvJp6a3TotN3wuj4mQHHRbfK7QUMrzCPOZGhRc= +github.com/bogdanfinn/fhttp v0.5.24/go.mod h1:brqi5woc5eSCVHdKYBV8aZLbO7HGqpwyDLeXW+fT18I= +github.com/bogdanfinn/tls-client v1.6.1 h1:GTIqQssFoIvLaDf4btoYRzDhUzudLqYD4axvfUCXl3I= +github.com/bogdanfinn/tls-client v1.6.1/go.mod h1:FtwQ3DndVZ0xAOO704v4iNAgbHOcEc5kPk9tjICTNQ0= +github.com/bogdanfinn/utls v1.5.16 h1:NhhWkegEcYETBMj9nvgO4lwvc6NcLH+znrXzO3gnw4M= +github.com/bogdanfinn/utls v1.5.16/go.mod h1:mHeRCi69cUiEyVBkKONB1cAbLjRcZnlJbGzttmiuK4o= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= -github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/elliotchance/pie/v2 v2.8.0 h1://QS43W8sEha8XV/fjngO5iMudN3XARJV5cpBayAcVY= +github.com/elliotchance/pie/v2 v2.8.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= +github.com/forPelevin/gomoji v1.1.8 h1:JElzDdt0TyiUlecy6PfITDL6eGvIaxqYH1V52zrd0qQ= +github.com/forPelevin/gomoji v1.1.8/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789 h1:G6wSuUyCoLB9jrUokipsmFuRi8aJozt3phw/g9Sl4Xs= +github.com/go-shiori/go-readability v0.0.0-20231029095239-6b97d5aba789/go.mod h1:2DpZlTJO/ycxp/vsc/C11oUyveStOgIXB88SYV1lncI= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofiber/fiber/v2 v2.51.0 h1:JNACcZy5e2tGApWB2QrRpenTWn0fq0hkFm6k0C86gKQ= +github.com/gofiber/fiber/v2 v2.51.0/go.mod h1:xaQRZQJGqnKOQnbQw+ltvku3/h8QxvNi8o6JiJ7Ll0U= +github.com/gofiber/template v1.8.2 h1:PIv9s/7Uq6m+Fm2MDNd20pAFFKt5wWs7ZBd8iV9pWwk= +github.com/gofiber/template v1.8.2/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8= +github.com/gofiber/template/html/v2 v2.0.5 h1:BKLJ6Qr940NjntbGmpO3zVa4nFNGDCi/IfUiDB9OC20= +github.com/gofiber/template/html/v2 v2.0.5/go.mod h1:RCF14eLeQDCSUPp0IGc2wbSSDv6yt+V54XB/+Unz+LM= +github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM= +github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k= +github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ= +github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0= +github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/markusmobius/go-dateparser v1.2.1 h1:mYRRdu3TzpAeE6fSl2Gn3arfxEtoTRvFOKlumlVsUtg= +github.com/markusmobius/go-dateparser v1.2.1/go.mod h1:5xYsZ1h7iB3sE1BSu8bkjYpbFST7EU1/AFxcyO3mgYg= +github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3 h1:D83RvMz1lQ0ilKlJt6DWc65+Q77CXGRFmfihR0bfQvc= +github.com/markusmobius/go-domdistiller v0.0.0-20230515154422-71af71939ff3/go.mod h1:n1AYw0wiJDT3YXnIsElJPiDR63YGXT2yv3uq0CboGmU= +github.com/markusmobius/go-htmldate v1.2.2 h1:tp1IxhefCYpEoL9CM1LiU6l+2YayTpuTjkkdnik6hXE= +github.com/markusmobius/go-htmldate v1.2.2/go.mod h1:26VRz16sCosuiv42MNRW9iPBGnGLo+q/Z6TWitt8uzs= +github.com/markusmobius/go-trafilatura v1.5.1 h1:EXhZY2AVRyepUlLZHeuZUme3v7Ms9G8lDOLl4u+Jp5M= +github.com/markusmobius/go-trafilatura v1.5.1/go.mod h1:FhuBBPZ9ph4ufpGBKAkuq5oQwEhg0KKnIOUlv5h7EHg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= +github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0= +github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= -github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/wasilibs/go-re2 v1.4.1 h1:E5+9O1M8UoGeqLB2A9omeoaWImqpuYDs9cKwvTJq/Oo= +github.com/wasilibs/go-re2 v1.4.1/go.mod h1:ynB8eCwd9JsqUnsk8WlPDk6cEeme8BguZmnqOSURE4Y= +github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= +github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/api.go b/handlers/api.go index 190c8fe8..96ff82a3 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -2,7 +2,6 @@ package handlers import ( _ "embed" - "log" "github.com/gofiber/fiber/v2" ) @@ -12,48 +11,5 @@ import ( var version string func Api(c *fiber.Ctx) error { - // Get the url from the URL - urlQuery := c.Params("*") - - queries := c.Queries() - body, req, resp, err := fetchSite(urlQuery, queries) - if err != nil { - log.Println("ERROR:", err) - c.SendStatus(500) - return c.SendString(err.Error()) - } - - response := Response{ - Version: version, - Body: body, - } - - response.Request.Headers = make([]any, 0, len(req.Header)) - for k, v := range req.Header { - response.Request.Headers = append(response.Request.Headers, map[string]string{ - "key": k, - "value": v[0], - }) - } - - response.Response.Headers = make([]any, 0, len(resp.Header)) - for k, v := range resp.Header { - response.Response.Headers = append(response.Response.Headers, map[string]string{ - "key": k, - "value": v[0], - }) - } - - return c.JSON(response) -} - -type Response struct { - Version string `json:"version"` - Body string `json:"body"` - Request struct { - Headers []interface{} `json:"headers"` - } `json:"request"` - Response struct { - Headers []interface{} `json:"headers"` - } `json:"response"` + return nil } diff --git a/handlers/api.test.go b/handlers/api.test.go deleted file mode 100644 index 710a0b29..00000000 --- a/handlers/api.test.go +++ /dev/null @@ -1,44 +0,0 @@ -// BEGIN: 7d5e1f7c7d5e -package handlers - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/gofiber/fiber/v2" - "github.com/stretchr/testify/assert" -) - -func TestApi(t *testing.T) { - app := fiber.New() - app.Get("/api/*", Api) - - tests := []struct { - name string - url string - expectedStatus int - }{ - { - name: "valid url", - url: "https://www.google.com", - expectedStatus: http.StatusOK, - }, - { - name: "invalid url", - url: "invalid-url", - expectedStatus: http.StatusBadRequest, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/api/"+tt.url, nil) - resp, err := app.Test(req) - assert.NoError(t, err) - assert.Equal(t, tt.expectedStatus, resp.StatusCode) - }) - } -} - -// END: 7d5e1f7c7d5e diff --git a/handlers/api_content.go b/handlers/api_content.go new file mode 100644 index 00000000..306cbade --- /dev/null +++ b/handlers/api_content.go @@ -0,0 +1,45 @@ +package handlers + +import ( + rx "github.com/everywall/ladder/proxychain/requestmodifiers" + tx "github.com/everywall/ladder/proxychain/responsemodifiers" + + "github.com/everywall/ladder/proxychain" + + "github.com/gofiber/fiber/v2" +) + +func NewAPIContentHandler(path string, opts *ProxyOptions) fiber.Handler { + // TODO: implement ruleset logic + /* + var rs ruleset.RuleSet + if opts.RulesetPath != "" { + r, err := ruleset.NewRuleset(opts.RulesetPath) + if err != nil { + panic(err) + } + rs = r + } + */ + + return func(c *fiber.Ctx) error { + proxychain := proxychain. + NewProxyChain(). + WithAPIPath(path). + SetDebugLogging(opts.Verbose). + SetRequestModifications( + rx.MasqueradeAsGoogleBot(), + rx.ForwardRequestHeaders(), + rx.SpoofReferrerFromGoogleSearch(), + ). + AddResponseModifications( + tx.DeleteIncomingCookies(), + tx.RewriteHTMLResourceURLs(), + tx.APIContent(), + ). + SetFiberCtx(c). + Execute() + + return proxychain + } +} diff --git a/handlers/api_modifiers.go b/handlers/api_modifiers.go new file mode 100644 index 00000000..cb30df50 --- /dev/null +++ b/handlers/api_modifiers.go @@ -0,0 +1,29 @@ +package handlers + +import ( + "encoding/json" + + "github.com/everywall/ladder/proxychain/responsemodifiers/api" + "github.com/gofiber/fiber/v2" +) + +func NewAPIModifersListHandler(opts *ProxyOptions) fiber.Handler { + payload := ModifiersAPIResponse{ + Success: true, + Result: AllMods, + } + body, err := json.MarshalIndent(payload, "", " ") + if err != nil { + panic(err) + } + + return func(c *fiber.Ctx) error { + c.Set("content-type", "application/json") + if err != nil { + c.SendStatus(500) + return c.SendStream(api.CreateAPIErrReader(err)) + } + + return c.Send(body) + } +} diff --git a/handlers/api_modifiers_codegen/api_modifiers_codegen.go b/handlers/api_modifiers_codegen/api_modifiers_codegen.go new file mode 100644 index 00000000..c3c3a443 --- /dev/null +++ b/handlers/api_modifiers_codegen/api_modifiers_codegen.go @@ -0,0 +1,196 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "io/fs" + "os/exec" + + //"io/fs" + "os" + "path/filepath" + "strings" + //"strings" +) + +func genModStruct(fn *ast.FuncDecl, githubEditLink string, filename string) string { + params := []string{} + for _, fd := range fn.Type.Params.List { + p := fmt.Sprintf(` {Name: "%s", Type: "%+v"},`, fd.Names[0], fd.Type) + params = append(params, p) + } + + block := fmt.Sprintf(`{ + Name: "%s", + Description: "%s", + CodeEditLink: "%s%s", + Params: []Param{ +%s + }, +},`, + fn.Name.String(), + strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(fn.Doc.Text()), "\n", " "), `"`, `\"`), + githubEditLink, filename, + strings.Join(params, "\n"), + ) + + return block +} + +func modCodeGen(dir string, githubEditLink string) (code string, err error) { + fset := token.NewFileSet() + + files, err := os.ReadDir(dir) + if err != nil { + panic(err) + } + + modStructs := []string{} + for _, file := range files { + if !shouldGenCodeFor(file) { + continue + } + + // Parse each Go file + node, err := parser.ParseFile(fset, filepath.Join(dir, file.Name()), nil, parser.ParseComments) + if err != nil { + return "", err + } + + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if ok && fn.Recv == nil && fn.Name.IsExported() { + modStructs = append(modStructs, genModStruct(fn, githubEditLink, file.Name())) + } + return true + }) + + } + + code = strings.Join(modStructs, "\n") + return code, nil +} + +func shouldGenCodeFor(file fs.DirEntry) bool { + if file.IsDir() { + return false + } + if filepath.Ext(file.Name()) != ".go" { + return false + } + if strings.HasSuffix(file.Name(), "_test.go") { + return false + } + return true +} + +func getGitRemoteURL(remoteName string) (string, error) { + cmd := exec.Command("git", "remote", "get-url", remoteName) + output, err := cmd.Output() + if err != nil { + return "", err + } + url := strings.TrimSpace(string(output)) + + // Convert SSH format to HTTPS format + if strings.HasPrefix(url, "git@") { + url = strings.Replace(url, ":", "/", 1) + url = strings.Replace(url, "git@", "https://", 1) + url = strings.TrimSuffix(url, ".git") + } + + return url, nil +} + +func getCurrentGitBranch() (string, error) { + cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD") + output, err := cmd.Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)), nil +} + +func main() { + gitURL, err := getGitRemoteURL("origin") + if err != nil { + fmt.Println("Error getting Git remote URL:", err) + return + } + + branchName, err := getCurrentGitBranch() + if err != nil { + fmt.Println("Error getting current Git branch:", err) + return + } + + githubEditLink := fmt.Sprintf("%s/edit/%s/proxychain/requestmodifiers/", gitURL, branchName) + rqmCode, err := modCodeGen("../../proxychain/requestmodifiers/", githubEditLink) + if err != nil { + panic(err) + } + + githubEditLink = fmt.Sprintf("%s/edit/%s/proxychain/responsemodifiers/", gitURL, branchName) + rsmCode, err := modCodeGen("../../proxychain/responsemodifiers/", githubEditLink) + if err != nil { + panic(err) + } + + code := fmt.Sprintf(` +package handlers +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go +// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions. +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain/responsemodifiers/api" +) + +type ModifiersAPIResponse struct { + Success bool ||json:"success"|| + Error api.ErrorDetails ||json:"error"|| + Result Modifiers ||json:"result"|| +} + +type Modifiers struct { + RequestModifiers []Modifier ||json:"requestmodifiers"|| + ResponseModifiers []Modifier ||json:"responsemodifiers"|| +} + +type Modifier struct { + Name string ||json:"name"|| + Description string ||json:"description"|| + CodeEditLink string ||json:"code_edit_link"|| + Params []Param ||json:"params"|| +} + +type Param struct { + Name string ||json:"name"|| + Type string ||json:"type"|| +} + +var AllMods Modifiers = Modifiers{ + RequestModifiers: []Modifier{ +%s + }, + ResponseModifiers: []Modifier{ +%s + }, +} +`, rqmCode, rsmCode) + code = strings.ReplaceAll(code, "||", "`") + + //fmt.Println(code) + + fq, err := os.Create("../api_modifiers_structdef.gen.go") + if err != nil { + panic(err) + } + _, err = io.WriteString(fq, code) + if err != nil { + panic(err) + } +} diff --git a/handlers/api_modifiers_structdef.gen.go b/handlers/api_modifiers_structdef.gen.go new file mode 100644 index 00000000..bc4caffb --- /dev/null +++ b/handlers/api_modifiers_structdef.gen.go @@ -0,0 +1,567 @@ + +package handlers +// DO NOT EDIT THIS FILE. It is automatically generated by ladder/handlers/api_modifiers_codegen/api_modifiers_codegen.go +// The purpose of this is to produce an API reponse listing all the available modifier, their parameters and usage instructions. +// for use in proxychains. + +import ( + "github.com/everywall/ladder/proxychain/responsemodifiers/api" +) + +type ModifiersAPIResponse struct { + Success bool `json:"success"` + Error api.ErrorDetails `json:"error"` + Result Modifiers `json:"result"` +} + +type Modifiers struct { + RequestModifiers []Modifier `json:"requestmodifiers"` + ResponseModifiers []Modifier `json:"responsemodifiers"` +} + +type Modifier struct { + Name string `json:"name"` + Description string `json:"description"` + CodeEditLink string `json:"code_edit_link"` + Params []Param `json:"params"` +} + +type Param struct { + Name string `json:"name"` + Type string `json:"type"` +} + +var AllMods Modifiers = Modifiers{ + RequestModifiers: []Modifier{ +{ + Name: "AddCacheBusterQuery", + Description: "AddCacheBusterQuery modifies query params to add a random parameter key In order to get the upstream network stack to serve a fresh copy of the page.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/add_cache_buster_query.go", + Params: []Param{ + + }, +}, +{ + Name: "ForwardRequestHeaders", + Description: "ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/forward_request_headers.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsGoogleBot", + Description: "MasqueradeAsGoogleBot modifies user agent and x-forwarded for to appear to be a Google Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsBingBot", + Description: "MasqueradeAsBingBot modifies user agent and x-forwarded for to appear to be a Bing Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsWaybackMachineBot", + Description: "MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for to appear to be a archive.org (wayback machine) Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsFacebookBot", + Description: "MasqueradeAsFacebookBot modifies user agent and x-forwarded for to appear to be a Facebook Bot (link previews?)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsYandexBot", + Description: "MasqueradeAsYandexBot modifies user agent and x-forwarded for to appear to be a Yandex Spider Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsBaiduBot", + Description: "MasqueradeAsBaiduBot modifies user agent and x-forwarded for to appear to be a Baidu Spider Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsDuckDuckBot", + Description: "MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for to appear to be a DuckDuckGo Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "MasqueradeAsYahooBot", + Description: "MasqueradeAsYahooBot modifies user agent and x-forwarded for to appear to be a Yahoo Bot", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/masquerade_as_trusted_bot.go", + Params: []Param{ + + }, +}, +{ + Name: "ModifyDomainWithRegex", + Description: "", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_domain_with_regex.go", + Params: []Param{ + {Name: "matchRegex", Type: "string"}, + {Name: "replacement", Type: "string"}, + }, +}, +{ + Name: "SetOutgoingCookie", + Description: "SetOutgoingCookie modifes a specific cookie name by modifying the request cookie headers going to the upstream server. If the cookie name does not already exist, it is created.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "SetOutgoingCookies", + Description: "SetOutgoingCookies modifies a client request's cookie header to a raw Cookie string, overwriting existing cookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "cookies", Type: "string"}, + }, +}, +{ + Name: "DeleteOutgoingCookie", + Description: "DeleteOutgoingCookie modifies the http request's cookies header to delete a specific request cookie going to the upstream server. If the cookie does not exist, it does not do anything.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + }, +}, +{ + Name: "DeleteOutgoingCookies", + Description: "DeleteOutgoingCookies removes the cookie header entirely, preventing any cookies from reaching the upstream server.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + + }, +}, +{ + Name: "DeleteOutgoingCookiesExcept", + Description: "DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client to the upstream proxy server. Cookies whose names are in the whitelist are not removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_outgoing_cookies.go", + Params: []Param{ + {Name: "whitelist", Type: "&{Ellipsis:12476 Elt:string}"}, + }, +}, +{ + Name: "ModifyPathWithRegex", + Description: "", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_path_with_regex.go", + Params: []Param{ + {Name: "matchRegex", Type: "string"}, + {Name: "replacement", Type: "string"}, + }, +}, +{ + Name: "ModifyQueryParams", + Description: "ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. If the query param key doesn't exist, it is created.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_query_params.go", + Params: []Param{ + {Name: "key", Type: "string"}, + {Name: "value", Type: "string"}, + }, +}, +{ + Name: "SetRequestHeader", + Description: "SetRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "DeleteRequestHeader", + Description: "DeleteRequestHeader modifies a specific outgoing header This is the header that the upstream server will see.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/modify_request_headers.go", + Params: []Param{ + {Name: "name", Type: "string"}, + }, +}, +{ + Name: "RequestArchiveIs", + Description: "RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_archive_is.go", + Params: []Param{ + + }, +}, +{ + Name: "RequestGoogleCache", + Description: "RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_google_cache.go", + Params: []Param{ + + }, +}, +{ + Name: "RequestWaybackMachine", + Description: "RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/request_wayback_machine.go", + Params: []Param{ + + }, +}, +{ + Name: "ResolveWithGoogleDoH", + Description: "ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL using Google's DNS over HTTPs service", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/resolve_with_google_doh.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofOrigin", + Description: "SpoofOrigin modifies the origin header if the upstream server returns a Vary header it means you might get a different response if you change this", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go", + Params: []Param{ + {Name: "url", Type: "string"}, + }, +}, +{ + Name: "HideOrigin", + Description: "HideOrigin modifies the origin header so that it is the original origin, not the proxy", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_origin.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrer", + Description: "SpoofReferrer modifies the referrer header. It is useful if the page can be accessed from a search engine or social media site, but not by browsing the website itself. if url is \"\", then the referrer header is removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go", + Params: []Param{ + {Name: "url", Type: "string"}, + }, +}, +{ + Name: "HideReferrer", + Description: "HideReferrer modifies the referrer header so that it is the original referrer, not the proxy", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromBaiduSearch", + Description: "SpoofReferrerFromBaiduSearch modifies the referrer header pretending to be from a BaiduSearch", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_baidu_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromBingSearch", + Description: "SpoofReferrerFromBingSearch modifies the referrer header pretending to be from a bing search site", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_bing_search.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromGoogleSearch", + Description: "SpoofReferrerFromGoogleSearch modifies the referrer header pretending to be from a google search site", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_google_search.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromLinkedInPost", + Description: "SpoofReferrerFromLinkedInPost modifies the referrer header pretending to be from a linkedin post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_linkedin_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromNaverSearch", + Description: "SpoofReferrerFromNaverSearch modifies the referrer header pretending to be from a Naver search (popular in South Korea)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_naver_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromPinterestPost", + Description: "SpoofReferrerFromPinterestPost modifies the referrer header pretending to be from a pinterest post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_pinterest_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromQQPost", + Description: "SpoofReferrerFromQQPost modifies the referrer header pretending to be from a QQ post (popular social media in China)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_qq_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromRedditPost", + Description: "SpoofReferrerFromRedditPost modifies the referrer header pretending to be from a reddit post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_reddit_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromTumblrPost", + Description: "SpoofReferrerFromTumblrPost modifies the referrer header pretending to be from a tumblr post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_tumblr_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromTwitterPost", + Description: "SpoofReferrerFromTwitterPost modifies the referrer header pretending to be from a twitter post", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_twitter_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromVkontaktePost", + Description: "SpoofReferrerFromVkontaktePost modifies the referrer header pretending to be from a vkontakte post (popular in Russia)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_vkontake_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofReferrerFromWeiboPost", + Description: "SpoofReferrerFromWeiboPost modifies the referrer header pretending to be from a Weibo post (popular in China)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_referrer_from_weibo_post.go", + Params: []Param{ + + }, +}, +{ + Name: "SpoofUserAgent", + Description: "SpoofUserAgent modifies the user agent", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_user_agent.go", + Params: []Param{ + {Name: "ua", Type: "string"}, + }, +}, +{ + Name: "SpoofXForwardedFor", + Description: "SpoofXForwardedFor modifies the X-Forwarded-For header in some cases, a forward proxy may interpret this as the source IP", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/requestmodifiers/spoof_x_forwarded_for.go", + Params: []Param{ + {Name: "ip", Type: "string"}, + }, +}, + }, + ResponseModifiers: []Modifier{ +{ + Name: "APIContent", + Description: "APIContent creates an JSON representation of the article and returns it as an API response.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/api_content.go", + Params: []Param{ + + }, +}, +{ + Name: "BlockElementRemoval", + Description: "BlockElementRemoval prevents paywall javascript from removing a particular element by detecting the removal, then immediately reinserting it. This is useful when a page will return a \"fake\" 404, after flashing the content briefly. If the /outline/ API works, but the regular API doesn't, try this modifier.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/block_element_removal.go", + Params: []Param{ + {Name: "cssSelector", Type: "string"}, + }, +}, +{ + Name: "BlockThirdPartyScripts", + Description: "BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/block_third_party_scripts.go", + Params: []Param{ + + }, +}, +{ + Name: "BypassCORS", + Description: "BypassCORS modifies response headers to prevent the browser from enforcing any CORS restrictions. This should run at the end of the chain.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_cors.go", + Params: []Param{ + + }, +}, +{ + Name: "BypassContentSecurityPolicy", + Description: "BypassContentSecurityPolicy modifies response headers to prevent the browser from enforcing any CSP restrictions. This should run at the end of the chain.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go", + Params: []Param{ + + }, +}, +{ + Name: "SetContentSecurityPolicy", + Description: "SetContentSecurityPolicy modifies response headers to a specific CSP", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/bypass_csp.go", + Params: []Param{ + {Name: "csp", Type: "string"}, + }, +}, +{ + Name: "DeleteLocalStorageData", + Description: "DeleteLocalStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteSessionStorageData and DeleteIncomingCookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/delete_localstorage_data.go", + Params: []Param{ + + }, +}, +{ + Name: "DeleteSessionStorageData", + Description: "DeleteSessionStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteLocalStorageData and DeleteIncomingCookies", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/delete_sessionstorage_data.go", + Params: []Param{ + + }, +}, +{ + Name: "ForwardResponseHeaders", + Description: "ForwardResponseHeaders forwards the response headers from the upstream server to the client", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/forward_response_headers.go", + Params: []Param{ + + }, +}, +{ + Name: "GenerateReadableOutline", + Description: "GenerateReadableOutline creates an reader-friendly distilled representation of the article. This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/generate_readable_outline.go", + Params: []Param{ + + }, +}, +{ + Name: "InjectScriptBeforeDOMContentLoaded", + Description: "InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "InjectScriptAfterDOMContentLoaded", + Description: "InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "InjectScriptAfterDOMIdle", + Description: "InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded)", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/inject_script.go", + Params: []Param{ + {Name: "js", Type: "string"}, + }, +}, +{ + Name: "DeleteIncomingCookies", + Description: "DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server back down to the client.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "_", Type: "&{Ellipsis:18780 Elt:string}"}, + }, +}, +{ + Name: "DeleteIncomingCookiesExcept", + Description: "DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server to the client. Cookies whose names are in the whitelist are not removed.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "whitelist", Type: "&{Ellipsis:19325 Elt:string}"}, + }, +}, +{ + Name: "SetIncomingCookies", + Description: "SetIncomingCookies adds a raw cookie string being sent from the proxy server down to the client", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "cookies", Type: "string"}, + }, +}, +{ + Name: "SetIncomingCookie", + Description: "SetIncomingCookie modifies a specific cookie in the response from the proxy server to the client.", + CodeEditLink: "https://github.com/everywall/ladder/edit/origin/proxy_v2/proxychain/responsemodifiers/modify_incoming_cookies.go", + Params: []Param{ + {Name: "name", Type: "string"}, + {Name: "val", Type: "string"}, + }, +}, +{ + Name: "ModifyIncomingScriptsWithRegex", + Description: "ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline + +
++{{.Status}}+{{.Message}}+Type: {{.Type}}+Cause: {{.Cause}}+