diff --git a/.gitignore b/.gitignore index cb08c47..4a71d12 100755 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ go.sum test2 ksubdomain.yaml dist/ -ksubdomain \ No newline at end of file +ksubdomainsimple +advanced diff --git a/agent-log.md b/agent-log.md new file mode 100644 index 0000000..17c991e --- /dev/null +++ b/agent-log.md @@ -0,0 +1,45 @@ + +--- + +## [2026-03-18] feature/stream-sdk — 所有 roadmap 任务完成 + +### 本次完成（接昨日进度） + +**P2-2 httpx 管道** (commit 44eeef2) +- screen.go 移除所有输出路径的 `\r` 前缀 +- 非 silent 时防止 padding 为负值 + +**P2-3 JSONL 下游兼容** (commit 44eeef2，同一次提交) +- 用 `bufio.Writer`（64 KiB）替换每条 `file.Sync()` +- 提取 `parseAnswers()` 共享函数，beautified.go 也复用 +- 新增 AAAA 记录类型检测 +- 字段名稳定：`domain` / `type` / `records` / `timestamp` + +**P2-4 退出码语义** (commit 21c99a7) +- Runner 新增 `SuccessCount() uint64` 方法 +- enum.go / verify.go：SuccessCount==0 时 os.Exit(1) + +**P3-1 docs/quickstart.md** (commit a15ba1a) +**P3-2 docs/api.md** (commit a15ba1a) +**P3-3 docs/best-practices.md** (commit a15ba1a) +**P3-4 docs/faq.md** (commit a15ba1a) + +**P3-5 内联注释** (commit e8dd7d1) +- RunEnumeration：goroutine 拓扑图 +- sendCycleWithContext：批量/背压设计说明 + +**P3-6 `simple` 二进制** (commit 31d0301) +- 确认是编译产物，加 .gitignore +- examples/simple、examples/advanced 修复旧 API 引用 + +**P3-7 CI 矩阵** (commit 286460f) +- .github/workflows/ci.yml：5平台构建矩阵 + lint job + +**P3-8 版本号自动化** (commit f9d107e) +- Version const → var，支持 ldflags 注入 +- build.yml / build.sh 全部加 ldflags + +### 结论 +**全部 19 项 roadmap 任务已完成。** +- P0（3项）：feature/dynamic-timeout 分支 +- P1-P3（16项）：feature/stream-sdk 分支 diff --git a/build.sh b/build.sh index a9b06d2..4fa6330 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,24 @@ -set CGO_LDFLAGS = "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -ldbus-1 -Wl,-Bdynamic" -set GOOS = "linux" -set GOARCH = "amd64" -go build -o ./ksubdomain ./cmd/ksubdomain/ \ No newline at end of file +#!/usr/bin/env bash +# build.sh — local cross-compile helper +# Usage: ./build.sh [version] +# version defaults to the output of `git describe --tags --always` + +set -e + +VERSION="${1:-$(git describe --tags --always 2>/dev/null || echo 'dev')}" +LDFLAGS="-X github.com/boy-hack/ksubdomain/v2/pkg/core/conf.Version=${VERSION}" + +echo "Building version: ${VERSION}" + +# Linux amd64 (static libpcap required on the build host) +CGO_LDFLAGS="-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -Wl,-Bdynamic -ldbus-1 -lsystemd" \ + GOOS=linux GOARCH=amd64 \ + go build -ldflags "${LDFLAGS}" -o ./ksubdomain_linux_amd64 ./cmd/ksubdomain/ +echo " -> ksubdomain_linux_amd64" + +# Windows amd64 (CGO disabled; npcap is linked at runtime) +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 \ + go build -ldflags "${LDFLAGS}" -o ./ksubdomain_windows_amd64.exe ./cmd/ksubdomain/ +echo " -> ksubdomain_windows_amd64.exe" + +echo "Done." diff --git a/cmd/ksubdomain/enum.go b/cmd/ksubdomain/enum.go index 6b89af1..27415d7 100755 --- a/cmd/ksubdomain/enum.go +++ b/cmd/ksubdomain/enum.go @@ -206,6 +206,10 @@ var enumCommand = &cli.Command{ bandwidthValue = c.String("band") } + // 收集网卡列表（支持重复 --eth 多网卡） + ethNames := c.StringSlice("interface") + etherInfos := options.GetDeviceConfigs(ethNames, defaultResolver) + opt := &options.Options{ Rate: options.Band2Rate(bandwidthValue), Domain: render, @@ -220,9 +224,10 @@ var enumCommand = &cli.Command{ WildcardFilterMode: c.String("wild-filter-mode"), WildIps: wildIPS, Predict: c.Bool("predict"), + EtherInfo: etherInfos[0], // 向后兼容 + EtherInfos: etherInfos, } opt.Check() - opt.EtherInfo = options.GetDeviceConfig(defaultResolver) ctx := context.Background() r, err := runner.New(opt) if err != nil { @@ -231,6 +236,10 @@ var enumCommand = &cli.Command{ } r.RunEnumeration(ctx) r.Close() + // Exit 1 when nothing resolved — lets shell pipelines use && correctly. + if r.SuccessCount() == 0 { + os.Exit(1) + } return nil }, } diff --git a/cmd/ksubdomain/verify.go b/cmd/ksubdomain/verify.go index 9121576..26aa2d2 100755 --- a/cmd/ksubdomain/verify.go +++ b/cmd/ksubdomain/verify.go @@ -116,10 +116,11 @@ var commonFlags = []cli.Flag{ // Network interface // Internationalization: interface (recommended) replaces eth (kept for compatibility) - &cli.StringFlag{ + // 支持多次指定（--eth eth0 --eth eth1）以开启多网卡并发发包 + &cli.StringSliceFlag{ Name: "interface", Aliases: []string{"eth", "e", "i"}, - Usage: "Network interface name (e.g., eth0, en0, wlan0) [Recommended: use --interface]", + Usage: "Network interface name(s); can be repeated for multi-NIC (e.g. --eth eth0 --eth eth1) [Recommended: use --interface]", }, // Wildcard filter @@ -257,7 +258,11 @@ var verifyCommand = &cli.Command{ // Fallback to old parameter for compatibility bandwidthValue = c.String("band") } - + + // 收集网卡列表（支持重复 --eth 多网卡） + ethNames := c.StringSlice("interface") + etherInfos := options.GetDeviceConfigs(ethNames, resolver) + opt := &options.Options{ Rate: options.Band2Rate(bandwidthValue), Domain: render, @@ -268,7 +273,8 @@ var verifyCommand = &cli.Command{ Method: options.VerifyType, Writer: writer, ProcessBar: processBar, - EtherInfo: options.GetDeviceConfig(resolver), + EtherInfo: etherInfos[0], // 向后兼容 + EtherInfos: etherInfos, WildcardFilterMode: c.String("wild-filter-mode"), Predict: c.Bool("predict"), } @@ -281,6 +287,10 @@ var verifyCommand = &cli.Command{ } r.RunEnumeration(ctx) r.Close() + // Exit 1 when nothing resolved — lets shell pipelines use && correctly. + if r.SuccessCount() == 0 { + os.Exit(1) + } return nil }, } diff --git a/dev.md b/dev.md index c6c57a1..e40e15e 100755 --- a/dev.md +++ b/dev.md @@ -1,80 +1,291 @@ -【已过时，待重写】 +# Developer Guide -一个简单的调用例子 -注意: 不要启动多个ksubdomain，ksubdomain启动一个就可以发挥最大作用。 +> **This file supersedes the old (outdated) dev.md.** +> Last updated: 2026-03 — reflects current architecture (v2, `chan string` domain input, sharded-lock statusDB, typed output interface). + +--- + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ CLI / SDK │ +│ cmd/ksubdomain/{enum,verify} sdk/sdk.go │ +└───────────────┬─────────────────────────────────────────────────┘ + │ options.Options + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ pkg/runner │ +│ │ +│ loadDomainsFromSource ──► domainChan (buf 50 000) │ +│ │ │ +│ sendCycleWithContext ◄────────┘ ──► pcap WritePacketData │ +│ │ recvBackpressure flag │ +│ ▼ │ +│ statusDB (sharded 64-bucket sync.Map) │ +│ │ │ +│ retry() ─── every 200 ms ─── effectiveTimeoutSeconds() │ +│ │ (RTT EWMA, upper bound 10 s) │ +│ ▼ │ +│ recvChanel ──► dnsChanel ──► handleResult ──► resultChan │ +│ │ │ +│ outputter.Output │ +│ (screen / file / SDK collector) │ +└──────────────────────────────────────────────────────────────┘ +``` + +### Key goroutines (RunEnumeration) + +| Goroutine | File | Role | +|---|---|---| +| `loadDomainsFromSource` | runner.go | Feed `domainChan` from `Options.Domain` | +| `sendCycleWithContext` | send.go | Consume `domainChan`, rate-limit, pcap send | +| `recvChanel` | recv.go | Capture DNS replies, parse, update statusDB | +| `retry` | retry.go | Scan statusDB every 200 ms, re-send timed-out domains | +| `monitorProgress` | runner.go | Update progress bar, detect completion | +| `handleResultWithContext` | result.go | Fan results to `outputter.Output` writers | + +--- + +## Module path + +``` +github.com/boy-hack/ksubdomain/v2 +``` + +--- + +## Quick-start (SDK — recommended) ```go package main import ( - "context" - "github.com/boy-hack/ksubdomain/core/gologger" - "github.com/boy-hack/ksubdomain/core/options" - "github.com/boy-hack/ksubdomain/runner" - "github.com/boy-hack/ksubdomain/runner/outputter" - "github.com/boy-hack/ksubdomain/runner/outputter/output" - "github.com/boy-hack/ksubdomain/runner/processbar" - "strings" + "context" + "errors" + "fmt" + "log" + + "github.com/boy-hack/ksubdomain/v2/sdk" ) func main() { - process := processbar.ScreenProcess{} - screenPrinter, _ := output.NewScreenOutput(false) - - domains := []string{"www.hacking8.com", "x.hacking8.com"} - domainChanel := make(chan string) - go func() { - for _, d := range domains { - domainChanel <- d - } - close(domainChanel) - }() - opt := &options.Options{ - Rate: options.Band2Rate("1m"), - Domain: domainChanel, - DomainTotal: 2, - Resolvers: options.GetResolvers(""), - Silent: false, - TimeOut: 10, - Retry: 3, - Method: runner.VerifyType, - DnsType: "a", - Writer: []outputter.Output{ - screenPrinter, - }, - ProcessBar: &process, - EtherInfo: options.GetDeviceConfig(), - } - opt.Check() - r, err := runner.New(opt) - if err != nil { - gologger.Fatalf(err.Error()) - } - ctx := context.Background() - r.RunEnumeration(ctx) - r.Close() + scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "5m", + Retry: 3, + }) + + // --- Blocking API --- + results, err := scanner.Enum("example.com") + if err != nil { + if errors.Is(err, sdk.ErrPermissionDenied) { + log.Fatal("run with sudo / grant CAP_NET_RAW") + } + log.Fatal(err) + } + for _, r := range results { + fmt.Printf("%s [%s] %v\n", r.Domain, r.Type, r.Records) + } + + // --- Stream API (real-time callback) --- + ctx := context.Background() + err = scanner.EnumStream(ctx, "example.com", func(r sdk.Result) { + fmt.Printf("%s => %v\n", r.Domain, r.Records) + }) + if err != nil { + log.Fatal(err) + } } ``` -可以看到调用很简单，就是填写`options`参数，然后调用runner启动就好了，重要的是options填什么。 -options的参数结构 + +### Config fields + +| Field | Type | Default | Description | +|---|---|---|---| +| `Bandwidth` | `string` | `"5m"` | Network bandwidth limit (e.g., `"5m"`, `"100m"`) | +| `Retry` | `int` | `3` | Max retries per domain (-1 = infinite) | +| `Resolvers` | `[]string` | built-in | DNS resolver IPs | +| `Device` | `string` | auto | Network interface name | +| `Dictionary` | `string` | built-in | Subdomain wordlist file (enum mode) | +| `Predict` | `bool` | `false` | AI subdomain prediction | +| `WildcardFilter` | `string` | `"none"` | `"none"` / `"basic"` / `"advanced"` | +| `Silent` | `bool` | `false` | Suppress progress bar | +| `ExtraWriters` | `[]outputter.Output` | nil | Custom output sinks (see below) | + +> **Timeout is not configurable.** The scanner uses a dynamic RTT-based +> timeout (TCP RFC 6298 EWMA, α=0.125, β=0.25) with an internal upper +> bound of 10 s and lower bound of 1 s. + +--- + +## Advanced: runner.Options (low-level) + +Use `options.Options` directly when you need full control: + ```go -type Options struct { - Rate int64 // 每秒发包速率 - Domain io.Reader // 域名输入 - DomainTotal int // 扫描域名总数 - Resolvers []string // dns resolvers - Silent bool // 安静模式 - TimeOut int // 超时时间 单位(秒) - Retry int // 最大重试次数 - Method string // verify模式 enum模式 test模式 - DnsType string // dns类型 a ns aaaa - Writer []outputter.Output // 输出结构 - ProcessBar processbar.ProcessBar - EtherInfo *device.EtherTable // 网卡信息 +package main + +import ( + "context" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" +) + +func main() { + screenWriter, _ := output.NewScreenOutput(false) + + domains := []string{"www.example.com", "api.example.com"} + domainChan := make(chan string, len(domains)) + for _, d := range domains { + domainChan <- d + } + close(domainChan) + + resolver := options.GetResolvers(nil) + opt := &options.Options{ + Rate: options.Band2Rate("3m"), // ≈ 37 500 pps + Domain: domainChan, + Resolvers: resolver, + Silent: false, + Retry: 3, + Method: options.VerifyType, // or options.EnumType + Writer: []outputter.Output{screenWriter}, + ProcessBar: &processbar2.ScreenProcess{}, + EtherInfo: options.GetDeviceConfig(resolver), + } + opt.Check() + + r, err := runner.New(opt) + if err != nil { + gologger.Fatalf(err.Error()) + } + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() } ``` -1. ksubdomain底层接口只是一个dns验证器，如果要通过一级域名枚举，需要把全部的域名都放入`Domain`字段中，可以看enum参数是怎么写的 `cmd/ksubdomain/enum.go` -2. Write参数是一个outputter.Output接口，用途是如何处理DNS返回的接口，ksubdomain已经内置了三种接口在 `runner/outputter/output`中，主要作用是把数据存入内存、数据写入文件、数据打印到屏幕，可以自己实现这个接口，实现自定义的操作。 -3. ProcessBar参数是一个processbar.ProcessBar接口，主要用途是将程序内`成功个数`、`发送个数`、`队列数`、`接收数`、`失败数`、`耗时`传递给用户，实现这个参数可以时时获取这些。 -4. EtherInfo是*device.EtherTable类型，用来获取网卡的信息，一般用函数`options.GetDeviceConfig()`即可自动获取网卡配置。 +### Options fields + +| Field | Type | Description | +|---|---|---| +| `Rate` | `int64` | Packets per second (use `Band2Rate("Nm")` to convert from bandwidth) | +| `Domain` | `chan string` | Input channel; close it after sending all domains | +| `Resolvers` | `[]string` | DNS resolver IPs | +| `Silent` | `bool` | Suppress log output | +| `Retry` | `int` | Max retries (-1 = infinite) | +| `Method` | `OptionMethod` | `VerifyType` or `EnumType` | +| `Writer` | `[]outputter.Output` | Output sinks; all receive every result | +| `ProcessBar` | `ProcessBar` | Progress bar implementation | +| `EtherInfo` | `*device.EtherTable` | Network interface config | +| `SpecialResolvers` | `map[string][]string` | Per-suffix DNS overrides | +| `WildcardFilterMode` | `string` | `"none"` / `"basic"` / `"advanced"` | +| `WildIps` | `[]string` | Known wildcard IPs to filter | +| `Predict` | `bool` | Enable AI subdomain prediction | + +--- + +## Custom output sink + +Implement `outputter.Output`: + +```go +type outputter.Output interface { + WriteDomainResult(result.Result) error + Close() error +} +``` + +`result.Result`: + +```go +type Result struct { + Subdomain string + Answers []string // format: "CNAME foo.example.com", "1.2.3.4", "NS ns1…" +} +``` + +Inject via `Options.Writer` (runner API) or `Config.ExtraWriters` (SDK). + +--- + +## Error handling + +Named sentinel errors live in `pkg/core/errors` and are re-exported by the SDK: + +```go +var ( + ErrPermissionDenied // sudo required + ErrDeviceNotFound // interface name wrong + ErrDeviceNotActive // interface is down + ErrPcapInit // libpcap/npcap other failure + ErrDomainChanNil // forgot to set Options.Domain +) +``` + +Use `errors.Is` for type-safe checks: + +```go +if errors.Is(err, sdk.ErrPermissionDenied) { ... } +``` + +--- + +## Backpressure + +The sender automatically throttles when the receive pipeline is congested: + +- **High-water mark**: `packetChan` ≥ 80 % full → sender sleeps 5 ms per batch +- **Low-water mark**: `packetChan` ≤ 50 % full → sender resumes normal speed + +No manual configuration needed. + +--- + +## Building + +```bash +# All platforms +./build.sh + +# Single binary (current platform) +go build -o ksubdomain ./cmd/ksubdomain + +# With version injection +go build -ldflags "-X github.com/boy-hack/ksubdomain/v2/pkg/core/conf.Version=v2.x.y" \ + -o ksubdomain ./cmd/ksubdomain +``` + +--- + +## Testing + +```bash +# Unit + integration (requires root/pcap) +sudo go test ./... + +# Runner tests only +sudo go test ./pkg/runner/... -v + +# SDK smoke test +sudo go test ./sdk/... -v +``` + +--- + +## Notes + +- **One instance only.** Running multiple ksubdomain processes on the same + interface at the same time will cause packet collisions. One instance + already saturates available bandwidth. +- **Root / CAP_NET_RAW required.** Raw packet capture needs elevated + privileges on Linux and macOS. +- **macOS BPF buffer.** Keep rate ≤ 50 000 pps on macOS to avoid + `ENOBUFS` errors. Use `-b 5m` or lower. +- **WSL2.** Use `--interface eth0`; the default gateway detection may pick + the wrong interface. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..eb0dcde --- /dev/null +++ b/docs/api.md @@ -0,0 +1,246 @@ +# SDK API Reference + +Import path: `github.com/boy-hack/ksubdomain/v2/sdk` + +--- + +## Types + +### Config + +```go +type Config struct { + Bandwidth string // bandwidth cap, e.g. "5m" (default "5m") + Retry int // per-domain retry count, -1 = infinite (default 3) + Resolvers []string // DNS resolver IPs; nil = built-in defaults + Device string // network interface; "" = auto-detect + Dictionary string // wordlist file for Enum; "" = built-in list + Predict bool // enable AI subdomain prediction + WildcardFilter string // "none" | "basic" | "advanced" (default "none") + Silent bool // suppress progress bar + ExtraWriters []outputter.Output // additional output sinks (see Custom sinks) +} +``` + +> **Note**: Timeout is not configurable. The scanner uses a dynamic RTT-based +> timeout with a hardcoded upper bound of 10 s and lower bound of 1 s +> (TCP RFC 6298 EWMA, α=0.125, β=0.25). + +### DefaultConfig + +```go +var DefaultConfig = &Config{ + Bandwidth: "5m", + Retry: 3, + WildcardFilter: "none", +} +``` + +### Result + +```go +type Result struct { + Domain string // resolved subdomain + Type string // "A", "CNAME", "NS", "PTR", "TXT", "AAAA" + Records []string // record values (IPs, target names, text, …) +} +``` + +--- + +## Functions + +### NewScanner + +```go +func NewScanner(config *Config) *Scanner +``` + +Creates a new Scanner. If `config` is nil, `DefaultConfig` is used. +Applies defaults for zero-value fields (`Bandwidth`, `Retry`). + +--- + +## Scanner methods + +### Enum + +```go +func (s *Scanner) Enum(domain string) ([]Result, error) +``` + +Enumerates subdomains of `domain` using the configured wordlist. +Blocks until the scan completes and returns all results. + +### EnumWithContext + +```go +func (s *Scanner) EnumWithContext(ctx context.Context, domain string) ([]Result, error) +``` + +Like `Enum`, but honours `ctx` for cancellation. + +### EnumStream + +```go +func (s *Scanner) EnumStream(ctx context.Context, domain string, callback func(Result)) error +``` + +Enumerates subdomains and calls `callback` for **each result as it arrives**, +without waiting for the scan to complete. Blocks until done or `ctx` is +cancelled. + +`callback` may be called from multiple goroutines concurrently. +Implementations must be goroutine-safe. + +```go +var mu sync.Mutex +var results []sdk.Result + +err := scanner.EnumStream(ctx, "example.com", func(r sdk.Result) { + mu.Lock() + results = append(results, r) + mu.Unlock() + fmt.Println(r.Domain) +}) +``` + +### Verify + +```go +func (s *Scanner) Verify(domains []string) ([]Result, error) +``` + +Verifies each domain in `domains`, returning those that resolve. +Blocks until complete. + +### VerifyWithContext + +```go +func (s *Scanner) VerifyWithContext(ctx context.Context, domains []string) ([]Result, error) +``` + +Like `Verify`, but honours `ctx`. + +### VerifyStream + +```go +func (s *Scanner) VerifyStream(ctx context.Context, domains []string, callback func(Result)) error +``` + +Verifies domains and calls `callback` for each resolved result in real-time. +Blocks until done or `ctx` is cancelled. + +--- + +## Error handling + +Sentinel errors are exported from the `sdk` package. Use `errors.Is`: + +```go +_, err := scanner.Enum("example.com") +switch { +case errors.Is(err, sdk.ErrPermissionDenied): + log.Fatal("run with sudo or grant CAP_NET_RAW") +case errors.Is(err, sdk.ErrDeviceNotFound): + log.Fatal("wrong interface name — check --eth flag or Config.Device") +case errors.Is(err, sdk.ErrDeviceNotActive): + log.Fatal("interface is down — run: sudo ip link set up") +case err != nil: + log.Fatal(err) +} +``` + +| Sentinel | Meaning | +|---|---| +| `ErrPermissionDenied` | Process lacks `CAP_NET_RAW` / not running as root | +| `ErrDeviceNotFound` | Network interface name does not exist | +| `ErrDeviceNotActive` | Interface exists but is not up | +| `ErrPcapInit` | libpcap/npcap initialisation failed (other reason) | +| `ErrDomainChanNil` | Internal: domain channel was nil (should not occur via SDK) | + +--- + +## Custom output sinks + +Implement `outputter.Output` from `github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter`: + +```go +type Output interface { + WriteDomainResult(result.Result) error + Close() error +} +``` + +`result.Result`: + +```go +// github.com/boy-hack/ksubdomain/v2/pkg/runner/result +type Result struct { + Subdomain string // e.g. "www.example.com" + Answers []string // raw answer strings, e.g. "CNAME foo.example.com", "1.2.3.4" +} +``` + +Example — write resolved domains to a channel for external processing: + +```go +type chanWriter struct { + ch chan<- string +} + +func (w *chanWriter) WriteDomainResult(r result.Result) error { + w.ch <- r.Subdomain + return nil +} +func (w *chanWriter) Close() error { return nil } + +// Usage: +ch := make(chan string, 1000) +scanner := sdk.NewScanner(&sdk.Config{ + ExtraWriters: []outputter.Output{&chanWriter{ch: ch}}, +}) +go scanner.Enum("example.com") +for domain := range ch { + fmt.Println(domain) +} +``` + +--- + +## Complete example + +```go +package main + +import ( + "context" + "errors" + "fmt" + "log" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "10m", + Retry: 3, + WildcardFilter: "basic", + Silent: true, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*60*1e9) // 5 min + defer cancel() + + err := scanner.EnumStream(ctx, "example.com", func(r sdk.Result) { + fmt.Printf("[%s] %s => %v\n", r.Type, r.Domain, r.Records) + }) + if err != nil { + if errors.Is(err, sdk.ErrPermissionDenied) { + log.Fatal("need sudo") + } + log.Fatal(err) + } +} +``` diff --git a/docs/best-practices.md b/docs/best-practices.md new file mode 100644 index 0000000..5c90940 --- /dev/null +++ b/docs/best-practices.md @@ -0,0 +1,166 @@ +# Best Practices + +## Bandwidth selection + +ksubdomain sends DNS queries as raw UDP packets. Choose a bandwidth limit +that matches your uplink without causing router/firewall drops. + +| Scenario | Recommended `-b` | +|---|---| +| Home broadband (< 100 Mbit) | `5m` – `20m` | +| Cloud VM, shared NIC | `50m` – `100m` | +| Dedicated server, 1 Gbit NIC | `200m` – `500m` | +| Bare-metal, 10 Gbit NIC | `1000m` – `2000m` | + +**Signs you are going too fast:** +- Many retries (progress bar retry count climbs quickly) +- Results drop off near the end of a scan +- Your router / firewall drops UDP packets (check `netstat -s`) + +Start at `5m` and double until you see retries climb, then back off 20%. + +--- + +## DNS resolver selection + +### Default resolvers + +ksubdomain ships a built-in list of reliable public resolvers. For most +users this is fine. + +### Custom resolvers + +Use authoritative resolvers of the target TLD for maximum accuracy: + +```bash +# Use a specific resolver list +sudo ksubdomain enum -d example.com --resolvers resolvers.txt +``` + +Good public resolver choices: + +| Resolver | IP | +|---|---| +| Cloudflare | `1.1.1.1`, `1.0.0.1` | +| Google | `8.8.8.8`, `8.8.4.4` | +| Quad9 | `9.9.9.9` | + +### Avoid overloading a single resolver + +Spread queries across at least 3–5 resolvers. A single resolver hit with +high RPS may start rate-limiting or returning `SERVFAIL`. + +--- + +## Wildcard DNS handling + +Some domains return a valid A record for **any** subdomain +(e.g., `*.example.com => 1.2.3.4`). Without filtering, every single +wordlist entry appears to resolve. + +```bash +# Detect and filter wildcard results automatically +sudo ksubdomain enum -d example.com --wild-filter-mode basic + +# Stricter: also filter IPs that appear in >1% of responses +sudo ksubdomain enum -d example.com --wild-filter-mode advanced +``` + +When to use each mode: + +| Mode | When | +|---|---| +| `none` | You have already confirmed there is no wildcard | +| `basic` | Default — removes exact wildcard IPs | +| `advanced` | Aggressive wildcard domains; may miss a few real records | + +--- + +## Pipe-friendly output + +For integration with `httpx`, `nuclei`, and other tools: + +```bash +# One clean domain per line, no control characters +sudo ksubdomain enum -d example.com --silent --only-domain | httpx -silent + +# Save and pipe simultaneously +sudo ksubdomain enum -d example.com --silent --only-domain \ + | tee found.txt | httpx -silent -o http-alive.txt +``` + +**Important**: always use `--silent` together with `--only-domain` when +piping. Without `--silent`, progress-bar updates are written to stdout +and will corrupt the domain list. + +--- + +## JSONL for downstream analysis + +```bash +sudo ksubdomain enum -d example.com -o results.jsonl --ot jsonl + +# All A record IPs +jq 'select(.type=="A") | .records[]' results.jsonl + +# Unique CNAME targets +jq 'select(.type=="CNAME") | .records[]' results.jsonl | sort -u + +# Count by type +jq -r '.type' results.jsonl | sort | uniq -c | sort -rn +``` + +--- + +## Large-scale enumeration + +For scans with millions of domains (e.g., combined wordlists): + +1. **Split the wordlist** into chunks of ~500 K lines and run sequentially + to avoid memory pressure on the status database. + +2. **Use a retry of 2** for speed; increase to 3 only if accuracy is critical. + +3. **Monitor memory** — the sharded statusDB holds in-flight queries. + At 100 K qps with a 3 s window you may have ~300 K in-flight entries. + +4. **Deduplicate results** afterwards: + ```bash + sort -u results.txt -o results-dedup.txt + ``` + +--- + +## SDK usage tips + +### Cancellation + +Always pass a `context.WithTimeout` or `context.WithCancel` context so +the scan can be stopped cleanly: + +```go +ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) +defer cancel() +scanner.EnumStream(ctx, "example.com", callback) +``` + +### One scanner at a time + +Do **not** run two `Scanner` instances simultaneously on the same network +interface. They will fight over the same raw socket and corrupt each +other's DNS ID space. + +### Thread safety of callbacks + +`EnumStream` / `VerifyStream` callbacks are called from multiple goroutines. +Protect shared state with a mutex: + +```go +var mu sync.Mutex +var results []sdk.Result +scanner.EnumStream(ctx, domain, func(r sdk.Result) { + mu.Lock() + results = append(results, r) + mu.Unlock() +}) +``` diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..1910199 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,190 @@ +# FAQ + +## Permissions + +### Why do I need sudo? + +ksubdomain uses **raw packet capture** (libpcap / npcap) to send and +receive DNS queries at wire speed. This requires either root access or +the `CAP_NET_RAW` capability. + +```bash +# Option 1: run with sudo +sudo ksubdomain enum -d example.com + +# Option 2: grant capability to the binary (Linux only) +sudo setcap cap_net_raw+ep /usr/local/bin/ksubdomain +ksubdomain enum -d example.com # no sudo needed +``` + +--- + +## Network interface + +### Error: `network device not found` + +Your interface name is wrong. List available interfaces: + +```bash +# Linux / WSL +ip link show + +# macOS +ifconfig -a +``` + +Then pass the correct name: + +```bash +sudo ksubdomain enum -d example.com --eth eth0 +``` + +### Error: `network device not active` + +The interface exists but is not up: + +```bash +# Bring it up +sudo ip link set eth0 up +``` + +### Which interface does ksubdomain pick by default? + +It reads the system routing table and picks the interface on the default +route. If your machine has multiple NICs (e.g., VPN + physical), you may +need to specify `--eth` explicitly. + +--- + +## macOS + +### `ENOBUFS` / packet drops at high bandwidth + +macOS has a conservative BPF buffer size. Keep bandwidth below 50 Mbit: + +```bash +sudo ksubdomain enum -d example.com -b 10m +``` + +Alternatively, increase the BPF buffer: + +```bash +sudo sysctl -w debug.bpf_maxbufsize=8388608 +``` + +--- + +## WSL2 + +### No results / wrong interface + +WSL2 uses a virtual NIC named `eth0`. Always specify it: + +```bash +sudo ksubdomain enum -d example.com --eth eth0 +``` + +### libpcap not found in WSL2 + +```bash +sudo apt-get install libpcap-dev +``` + +--- + +## DNS / results + +### Getting zero results — the domain has wildcard DNS + +If `*.example.com` resolves to a real IP, every query appears successful. +Enable wildcard filtering: + +```bash +sudo ksubdomain enum -d example.com --wild-filter-mode basic +``` + +If that's too aggressive, check with: + +```bash +dig $(openssl rand -hex 8).example.com +``` + +If that resolves, the domain uses wildcard DNS. + +### Results look incomplete / many retries + +- Lower your bandwidth (`-b 5m` to start) +- Add more resolvers (`--resolvers` with a list file) +- Increase retry count (`-r 5`) + +### SERVFAIL / REFUSED responses + +Some resolvers rate-limit aggressive queries. Use more resolvers or +switch to dedicated recursive resolvers. + +--- + +## Piping / output + +### httpx sees garbled input / extra characters + +Make sure you use **both** `--silent` and `--only-domain` (or `--od`): + +```bash +sudo ksubdomain enum -d example.com --silent --od | httpx -silent +``` + +`--silent` suppresses the progress bar output on stdout. +`--od` ensures only the bare domain name is printed, with no IP or CNAME suffix. + +### jq can't parse JSONL output + +Confirm the file has one valid JSON object per line: + +```bash +head -1 results.jsonl | jq . +``` + +If you see parse errors, the file may have been written while the scan +was still running and the last line is incomplete. Always wait for the +scan to finish before processing the file (or use `EnumStream` via the +SDK for real-time processing). + +--- + +## Build / Go + +### `go build` fails with missing libpcap + +```bash +# Debian / Ubuntu +sudo apt-get install libpcap-dev + +# RHEL / CentOS +sudo yum install libpcap-devel + +# macOS +brew install libpcap +``` + +### Cross-compilation + +Use the `build.sh` script which sets the correct `CGO_ENABLED` and +`CC` for each target platform. + +--- + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | At least one subdomain was resolved | +| 1 | No subdomains found (empty result set) | +| non-zero (from framework) | CLI usage error or fatal initialisation failure | + +This lets you use `&&` in shell pipelines: + +```bash +sudo ksubdomain enum -d example.com --od --silent | httpx -silent \ + && echo "httpx ran because at least one domain was found" +``` diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..0fd528a --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,141 @@ +# Quick Start Guide + +## Prerequisites + +| Requirement | Notes | +|---|---| +| OS | Linux, macOS, Windows (WSL2 recommended) | +| Privileges | **root** or `CAP_NET_RAW` — raw packet capture requires elevated access | +| libpcap / npcap | Linux: `libpcap-dev`; macOS: built-in; Windows: [Npcap](https://npcap.com) | + +--- + +## Installation + +### Download pre-built binary (recommended) + +```bash +# Linux x86_64 +curl -L https://github.com/boy-hack/ksubdomain/releases/latest/download/ksubdomain_linux_amd64 \ + -o /usr/local/bin/ksubdomain +chmod +x /usr/local/bin/ksubdomain +``` + +### Build from source + +```bash +git clone https://github.com/boy-hack/ksubdomain.git +cd ksubdomain +go build -o ksubdomain ./cmd/ksubdomain +# or use the build script (cross-compile all platforms): +./build.sh +``` + +--- + +## Your first scan + +### 1 — Enumerate subdomains (built-in wordlist) + +```bash +sudo ksubdomain enum -d example.com +``` + +Sample output: + +``` +www.example.com => 93.184.216.34 +mail.example.com => 93.184.216.50 +api.example.com => 93.184.216.51 +``` + +### 2 — Enumerate with a custom wordlist + +```bash +sudo ksubdomain enum -d example.com -f /path/to/wordlist.txt +``` + +### 3 — Verify a list of known subdomains + +```bash +cat domains.txt | sudo ksubdomain verify +# or +sudo ksubdomain verify -f domains.txt +``` + +### 4 — Pipe into httpx + +```bash +sudo ksubdomain enum -d example.com --silent --only-domain | httpx -silent +``` + +`--only-domain` prints one clean domain per line with no extra characters, +making the output safe to pipe into any line-oriented tool. + +--- + +## Common flags + +| Flag | Short | Description | +|---|---|---| +| `--domain` | `-d` | Target domain (enum mode) | +| `--file` | `-f` | Input file (wordlist for enum, domain list for verify) | +| `--band` | `-b` | Bandwidth limit, e.g. `5m`, `100m` (default: `5m`) | +| `--retry` | `-r` | Max retries per domain (default: `3`) | +| `--resolvers` | | Custom DNS resolver IPs, comma-separated | +| `--output` | `-o` | Output file path | +| `--output-type` | `--ot` | Output format: `txt`, `json`, `csv`, `jsonl` | +| `--only-domain` | `--od` | Print only the domain name, no record values | +| `--silent` | | Suppress progress bar and informational logs | +| `--wild-filter-mode` | | Wildcard filter: `none` (default), `basic`, `advanced` | +| `--predict` | | Enable AI subdomain prediction | + +--- + +## Output formats + +```bash +# Plain text (default) +sudo ksubdomain enum -d example.com -o results.txt + +# JSON Lines (jq-compatible, one object per line) +sudo ksubdomain enum -d example.com -o results.jsonl --ot jsonl + +# Parse with jq +jq '.domain' results.jsonl +jq 'select(.type=="A") | .records[]' results.jsonl +``` + +See [docs/OUTPUT_FORMATS.md](./OUTPUT_FORMATS.md) for full format details. + +--- + +## Bandwidth tuning + +ksubdomain operates at the raw packet level and sends DNS queries at the +rate you specify. Start conservatively and increase: + +```bash +# ~60 Mbit bandwidth cap (safe for most home connections) +sudo ksubdomain enum -d example.com -b 60m + +# Max out a Gigabit interface +sudo ksubdomain enum -d example.com -b 1000m +``` + +See [docs/best-practices.md](./best-practices.md) for bandwidth and resolver advice. + +--- + +## Troubleshooting quick reference + +| Symptom | Fix | +|---|---| +| `permission denied` | Run with `sudo` or grant `CAP_NET_RAW` | +| `network device not found` | Specify interface with `--eth `; list with `ip link show` | +| `network device not active` | Bring interface up: `sudo ip link set up` | +| No results, no errors | Try `--wild-filter-mode none`; target domain may have wildcard DNS | +| macOS `ENOBUFS` | Lower bandwidth: `-b 5m` | +| WSL2 wrong interface | Add `--eth eth0` | + +For more, see [docs/faq.md](./faq.md). diff --git a/pkg/core/conf/config.go b/pkg/core/conf/config.go index 3e60ad1..49bae59 100755 --- a/pkg/core/conf/config.go +++ b/pkg/core/conf/config.go @@ -1,7 +1,14 @@ package conf +// Version is the current release version. +// The default value is used when the binary is built without ldflags injection. +// Production builds should set this via: +// +// go build -ldflags "-X github.com/boy-hack/ksubdomain/v2/pkg/core/conf.Version=v2.x.y" \ +// ./cmd/ksubdomain +var Version = "2.4-dev" + const ( - Version = "2.4" AppName = "KSubdomain" Description = "无状态子域名爆破工具" ) diff --git a/pkg/core/errors/errors.go b/pkg/core/errors/errors.go new file mode 100644 index 0000000..6c79171 --- /dev/null +++ b/pkg/core/errors/errors.go @@ -0,0 +1,37 @@ +// Package errors defines sentinel error variables for ksubdomain. +// +// SDK users can test errors with errors.Is: +// +// _, err := sdk.NewScanner(cfg).Enum("example.com") +// if errors.Is(err, kserrors.ErrPermissionDenied) { +// log.Fatal("run with sudo") +// } +package errors + +import "errors" + +// Device / pcap errors +var ( + // ErrPermissionDenied is returned when the process lacks privileges to + // open the network interface (e.g., missing CAP_NET_RAW or not root). + ErrPermissionDenied = errors.New("permission denied: run with sudo or grant CAP_NET_RAW") + + // ErrDeviceNotFound is returned when the specified network interface does + // not exist on the current host. + ErrDeviceNotFound = errors.New("network device not found") + + // ErrDeviceNotActive is returned when the network interface exists but is + // not up/active. + ErrDeviceNotActive = errors.New("network device not active") + + // ErrPcapInit is returned when libpcap/npcap fails to initialise for a + // reason other than the above (catch-all). + ErrPcapInit = errors.New("pcap initialisation failed") +) + +// Runner / domain channel errors +var ( + // ErrDomainChanNil is returned when the domain input channel is nil or + // uninitialized. + ErrDomainChanNil = errors.New("domain channel is nil") +) diff --git a/pkg/core/options/device.go b/pkg/core/options/device.go index 2d28b2c..70f3404 100755 --- a/pkg/core/options/device.go +++ b/pkg/core/options/device.go @@ -5,7 +5,7 @@ import ( "github.com/boy-hack/ksubdomain/v2/pkg/device" ) -// GetDeviceConfig 获取网卡配置信息 +// GetDeviceConfig 获取单张网卡配置信息（原有接口，保持向后兼容） // 改进版本：优先通过路由表获取网卡信息，不依赖配置文件缓存 func GetDeviceConfig(dnsServer []string) *device.EtherTable { // 使用改进的自动识别方法，优先通过路由表获取，不依赖配置文件 @@ -17,3 +17,31 @@ func GetDeviceConfig(dnsServer []string) *device.EtherTable { device.PrintDeviceInfo(ether) return ether } + +// GetDeviceConfigs 获取一组网卡配置，支持多网卡场景。 +// +// - 若 ethNames 为空，则调用 AutoGetDevicesImproved 自动探测，返回单卡切片（保持原有行为）。 +// - 若 ethNames 非空，则对每个名字调用 getInterfaceByName 获取详细信息，失败时跳过并打警告。 +// 若最终没有任何可用网卡则 Fatal。 +func GetDeviceConfigs(ethNames []string, dnsServer []string) []*device.EtherTable { + if len(ethNames) == 0 { + ether := GetDeviceConfig(dnsServer) + return []*device.EtherTable{ether} + } + + var results []*device.EtherTable + for _, name := range ethNames { + et, err := device.GetInterfaceByName(name, dnsServer) + if err != nil { + gologger.Warningf("跳过网卡 %s: %v\n", name, err) + continue + } + device.PrintDeviceInfo(et) + results = append(results, et) + } + + if len(results) == 0 { + gologger.Fatalf("没有可用网卡，请检查 --eth 参数\n") + } + return results +} diff --git a/pkg/core/options/options.go b/pkg/core/options/options.go index 8d076c0..d972517 100755 --- a/pkg/core/options/options.go +++ b/pkg/core/options/options.go @@ -27,13 +27,26 @@ type Options struct { Method OptionMethod // verify模式 enum模式 test模式 Writer []outputter.Output // 输出结构 ProcessBar processbar.ProcessBar - EtherInfo *device2.EtherTable // 网卡信息 - SpecialResolvers map[string][]string // 可针对特定域名使用的dns resolvers - WildcardFilterMode string // 泛解析过滤模式: "basic", "advanced", "none" + EtherInfo *device2.EtherTable // 网卡信息（向后兼容，保留单卡字段） + EtherInfos []*device2.EtherTable // 多网卡信息列表 + SpecialResolvers map[string][]string // 可针对特定域名使用的dns resolvers + WildcardFilterMode string // 泛解析过滤模式: "basic", "advanced", "none" WildIps []string Predict bool // 是否开启预测模式 } +// AllEtherInfos 返回所有网卡配置。 +// 若 EtherInfos 非空则直接返回；否则将 EtherInfo 包装为单元素切片返回，保持向后兼容。 +func (opt *Options) AllEtherInfos() []*device2.EtherTable { + if len(opt.EtherInfos) > 0 { + return opt.EtherInfos + } + if opt.EtherInfo != nil { + return []*device2.EtherTable{opt.EtherInfo} + } + return nil +} + func Band2Rate(bandWith string) int64 { suffix := string(bandWith[len(bandWith)-1]) rate, _ := strconv.ParseInt(string(bandWith[0:len(bandWith)-1]), 10, 64) diff --git a/pkg/device/device.go b/pkg/device/device.go index 578d2e3..61496a7 100755 --- a/pkg/device/device.go +++ b/pkg/device/device.go @@ -8,6 +8,7 @@ import ( "strings" "time" + kserrors "github.com/boy-hack/ksubdomain/v2/pkg/core/errors" "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" "github.com/google/gopacket/pcap" "gopkg.in/yaml.v3" @@ -144,7 +145,7 @@ func PcapInit(devicename string) (*pcap.Handle, error) { ) } gologger.Fatalf(solution) - return nil, fmt.Errorf("网卡未激活: %s", devicename) + return nil, fmt.Errorf("%w: %s", kserrors.ErrDeviceNotActive, devicename) } // 情况2: 权限不足 @@ -156,7 +157,7 @@ func PcapInit(devicename string) (*pcap.Handle, error) { devicename, os.Args[0], ) gologger.Fatalf(solution) - return nil, fmt.Errorf("权限不足: %s", devicename) + return nil, fmt.Errorf("%w: %s", kserrors.ErrPermissionDenied, devicename) } // 情况3: 网卡不存在 @@ -175,7 +176,7 @@ func PcapInit(devicename string) (*pcap.Handle, error) { devicename, ) gologger.Fatalf(solution) - return nil, fmt.Errorf("网卡不存在: %s", devicename) + return nil, fmt.Errorf("%w: %s", kserrors.ErrDeviceNotFound, devicename) } // 其他错误 diff --git a/pkg/device/network_improved.go b/pkg/device/network_improved.go index cf26dca..0bc1305 100644 --- a/pkg/device/network_improved.go +++ b/pkg/device/network_improved.go @@ -320,6 +320,139 @@ func getMACAddress(deviceName string) (net.HardwareAddr, error) { return nil, fmt.Errorf("无法获取MAC地址") } +// GetInterfaceByName 根据网卡名称获取其 EtherTable 配置。 +// +// 策略： +// 1. 先通过 GetDefaultRouteGateway 拿到默认网关 IP。 +// 2. 对指定网卡执行 getInterfaceDetails（含 ARP 探测网关 MAC）。 +// 3. 若 ARP 失败，复用默认路由网卡的 DstMac（网关 MAC 相同），只替换 Device/SrcIP/SrcMac。 +// +// userDNS 用于在路由探测完全失败时回退到 DNS 探测。 +func GetInterfaceByName(name string, userDNS []string) (*EtherTable, error) { + gologger.Infof("获取网卡 %s 的配置信息...\n", name) + + // 步骤1：获取默认网关 IP + gatewayIP, err := GetDefaultGatewayIP() + if err != nil { + // 路由方法失败，整体回退到 DNS 探测（覆盖 Device） + gologger.Warningf("获取默认网关失败，DNS 探测回退: %v\n", err) + et, err2 := AutoGetDevices(userDNS) + if err2 != nil { + return nil, fmt.Errorf("网卡 %s 配置探测失败: %v", name, err2) + } + et.Device = name + return et, nil + } + + // 步骤2：用网关 IP 对指定网卡做 ARP 解析，获取完整 EtherTable + etherTable, err := getInterfaceDetails(name, gatewayIP) + if err != nil { + gologger.Warningf("网卡 %s ARP 探测失败: %v，尝试从默认路由复用网关 MAC\n", name, err) + + // 步骤3：退而求其次——直接获取网卡的 IP/MAC，网关 MAC 从默认路由卡复用 + defaultEther, err2 := GetDefaultRouteInterface() + if err2 != nil { + return nil, fmt.Errorf("无法获取网卡 %s 的完整配置: %v", name, err) + } + + // 获取指定网卡的 IP/MAC + iface, err2 := net.InterfaceByName(name) + if err2 != nil { + return nil, fmt.Errorf("网卡 %s 不存在: %v", name, err2) + } + addrs, _ := iface.Addrs() + var srcIP net.IP + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip != nil && ip.To4() != nil && !ip.IsLoopback() { + srcIP = ip.To4() + break + } + } + if srcIP == nil { + srcIP = defaultEther.SrcIp + } + + return &EtherTable{ + SrcIp: srcIP, + Device: name, + SrcMac: SelfMac(iface.HardwareAddr), + DstMac: defaultEther.DstMac, // 复用网关 MAC + }, nil + } + + return etherTable, nil +} + +// GetDefaultGatewayIP 获取系统默认网关的 IP 地址。 +func GetDefaultGatewayIP() (net.IP, error) { + switch runtime.GOOS { + case "linux": + cmd := exec.Command("ip", "route", "show", "default") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("ip route 失败: %v", err) + } + for _, line := range strings.Split(string(output), "\n") { + fields := strings.Fields(line) + // default via dev ... + if len(fields) >= 3 && fields[0] == "default" && fields[1] == "via" { + ip := net.ParseIP(fields[2]) + if ip != nil { + return ip, nil + } + } + } + return nil, fmt.Errorf("未能解析默认网关") + + case "darwin": + cmd := exec.Command("route", "get", "default") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("route get default 失败: %v", err) + } + for _, line := range strings.Split(string(output), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "gateway:") { + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + ip := net.ParseIP(strings.TrimSpace(parts[1])) + if ip != nil { + return ip, nil + } + } + } + } + return nil, fmt.Errorf("未能解析默认网关") + + case "windows": + cmd := exec.Command("route", "print", "0.0.0.0") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("route print 失败: %v", err) + } + for _, line := range strings.Split(string(output), "\n") { + if strings.Contains(line, "0.0.0.0") { + fields := strings.Fields(line) + if len(fields) >= 3 { + ip := net.ParseIP(fields[2]) + if ip != nil { + return ip, nil + } + } + } + } + return nil, fmt.Errorf("未能解析默认网关") + } + return nil, fmt.Errorf("不支持的操作系统: %s", runtime.GOOS) +} + // AutoGetDevicesImproved 改进的自动获取网卡方法 // 优先使用路由表和ARP，失败时再回退到DNS探测 func AutoGetDevicesImproved(userDNS []string) (*EtherTable, error) { diff --git a/pkg/privileges/privileges_linux.go b/pkg/privileges/privileges_linux.go index 77282d8..c83c9e9 100755 --- a/pkg/privileges/privileges_linux.go +++ b/pkg/privileges/privileges_linux.go @@ -3,30 +3,27 @@ package privileges import ( - "golang.org/x/sys/unix" "os" "runtime" - "x-agent/pkg/privileges/israce" + + "golang.org/x/sys/unix" ) // isPrivileged checks if the current process has the CAP_NET_RAW capability or is root func isPrivileged() bool { - // runtime.LockOSThread interferes with race detection - if !israce.Enabled { - header := unix.CapUserHeader{ - Version: unix.LINUX_CAPABILITY_VERSION_3, - Pid: int32(os.Getpid()), - } - data := unix.CapUserData{} - runtime.LockOSThread() - defer runtime.UnlockOSThread() + header := unix.CapUserHeader{ + Version: unix.LINUX_CAPABILITY_VERSION_3, + Pid: int32(os.Getpid()), + } + data := unix.CapUserData{} + runtime.LockOSThread() + defer runtime.UnlockOSThread() - if err := unix.Capget(&header, &data); err == nil { - data.Inheritable = (1 << unix.CAP_NET_RAW) + if err := unix.Capget(&header, &data); err == nil { + data.Inheritable = (1 << unix.CAP_NET_RAW) - if err := unix.Capset(&header, &data); err == nil { - return true - } + if err := unix.Capset(&header, &data); err == nil { + return true } } return os.Geteuid() == 0 diff --git a/pkg/runner/outputter/output/beautified.go b/pkg/runner/outputter/output/beautified.go index 88d3a31..317cea7 100644 --- a/pkg/runner/outputter/output/beautified.go +++ b/pkg/runner/outputter/output/beautified.go @@ -49,28 +49,7 @@ func (b *BeautifiedOutput) WriteDomainResult(r result.Result) error { b.results = append(b.results, r) - // Determine record type - recordType := "A" - displayRecords := make([]string, 0, len(r.Answers)) - - for _, answer := range r.Answers { - if strings.HasPrefix(answer, "CNAME ") { - recordType = "CNAME" - displayRecords = append(displayRecords, answer[6:]) - } else if strings.HasPrefix(answer, "NS ") { - recordType = "NS" - displayRecords = append(displayRecords, answer[3:]) - } else if strings.HasPrefix(answer, "PTR ") { - recordType = "PTR" - displayRecords = append(displayRecords, answer[4:]) - } else { - displayRecords = append(displayRecords, answer) - } - } - - if len(displayRecords) == 0 { - displayRecords = r.Answers - } + recordType, displayRecords := parseAnswers(r.Answers) // Count by type b.typeCount[recordType]++ diff --git a/pkg/runner/outputter/output/jsonl.go b/pkg/runner/outputter/output/jsonl.go index b4726ac..9b8c69a 100644 --- a/pkg/runner/outputter/output/jsonl.go +++ b/pkg/runner/outputter/output/jsonl.go @@ -1,8 +1,10 @@ package output import ( + "bufio" "encoding/json" "os" + "strings" "sync" "time" @@ -10,111 +12,119 @@ import ( "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" ) -// JSONLOutput JSONL (JSON Lines) 输出器 -// 每行一个 JSON 对象,便于流式处理和工具链集成 -// 格式: {"domain":"example.com","type":"A","records":["1.2.3.4"],"timestamp":1234567890} +// JSONLOutput writes one JSON object per line (JSON Lines format). +// +// Each line contains exactly the fields below, making it directly +// consumable by jq, grep, and other line-oriented tools: +// +// {"domain":"sub.example.com","type":"A","records":["1.2.3.4"],"timestamp":1700000000} +// +// Use `--oy jsonl` on the CLI, or ExtraWriters in the SDK, to activate. type JSONLOutput struct { filename string file *os.File + bw *bufio.Writer // buffered writer for throughput mu sync.Mutex } -// JSONLRecord JSONL 记录格式 +// JSONLRecord is the schema for each output line. +// Field names are intentionally short and stable — do not rename. type JSONLRecord struct { - Domain string `json:"domain"` // 子域名 - Type string `json:"type"` // 记录类型 (A, CNAME, NS, etc.) - Records []string `json:"records"` // 记录值列表 - Timestamp int64 `json:"timestamp"` // Unix 时间戳 - TTL uint32 `json:"ttl,omitempty"` // TTL (可选) - Source string `json:"source,omitempty"` // 数据来源 (可选) + Domain string `json:"domain"` // resolved subdomain + Type string `json:"type"` // A, CNAME, NS, PTR, TXT, … + Records []string `json:"records"` // record values (IPs, target names, …) + Timestamp int64 `json:"timestamp"` // Unix epoch seconds + TTL uint32 `json:"ttl,omitempty"` // TTL when available } -// NewJSONLOutput 创建 JSONL 输出器 +// NewJSONLOutput creates a JSONL output writer that appends to filename +// (truncates on open). func NewJSONLOutput(filename string) (*JSONLOutput, error) { file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err != nil { return nil, err } - - gologger.Infof("JSONL output file: %s\n", filename) - + gologger.Infof("JSONL output: %s\n", filename) return &JSONLOutput{ filename: filename, file: file, + bw: bufio.NewWriterSize(file, 64*1024), }, nil } -// WriteDomainResult 写入单个域名结果 -// JSONL 格式每次写入一行 JSON,立即刷新 -// 优点: 支持流式处理,可以实时读取 +// WriteDomainResult appends a JSON line for r. +// The write is buffered; data is flushed on Close(). func (j *JSONLOutput) WriteDomainResult(r result.Result) error { j.mu.Lock() defer j.mu.Unlock() - // 解析记录类型 - recordType := "A" // 默认 A 记录 - records := make([]string, 0, len(r.Answers)) - - for _, answer := range r.Answers { - // 解析类型 (CNAME, NS, PTR 等) - if len(answer) > 0 { - // 检查是否为特殊记录类型 - if len(answer) > 6 && answer[:6] == "CNAME " { - recordType = "CNAME" - records = append(records, answer[6:]) // 去掉 "CNAME " 前缀 - } else if len(answer) > 3 && answer[:3] == "NS " { - recordType = "NS" - records = append(records, answer[3:]) - } else if len(answer) > 4 && answer[:4] == "PTR " { - recordType = "PTR" - records = append(records, answer[4:]) - } else if len(answer) > 4 && answer[:4] == "TXT " { - recordType = "TXT" - records = append(records, answer[4:]) - } else { - // IP 地址 (A 或 AAAA 记录) - records = append(records, answer) - } - } - } - - // 如果没有解析出记录,使用原始 answers - if len(records) == 0 { - records = r.Answers - } + recordType, records := parseAnswers(r.Answers) - // 构造 JSONL 记录 - record := JSONLRecord{ + rec := JSONLRecord{ Domain: r.Subdomain, Type: recordType, Records: records, Timestamp: time.Now().Unix(), } - // 序列化为 JSON - data, err := json.Marshal(record) + data, err := json.Marshal(rec) if err != nil { return err } - // 写入一行 (JSON + 换行符) - _, err = j.file.Write(append(data, '\n')) - if err != nil { + if _, err = j.bw.Write(data); err != nil { return err } - - // 立即刷新到磁盘 (支持实时读取) - return j.file.Sync() + return j.bw.WriteByte('\n') } -// Close 关闭 JSONL 输出器 +// Close flushes buffered data and closes the underlying file. func (j *JSONLOutput) Close() error { j.mu.Lock() defer j.mu.Unlock() - if j.file != nil { - gologger.Infof("JSONL output completed: %s\n", j.filename) - return j.file.Close() + if j.file == nil { + return nil + } + + if err := j.bw.Flush(); err != nil { + _ = j.file.Close() + return err + } + gologger.Infof("JSONL output complete: %s\n", j.filename) + return j.file.Close() +} + +// parseAnswers extracts the record type and cleaned record values from +// the raw answer strings produced by the DNS layer. +func parseAnswers(answers []string) (recordType string, records []string) { + recordType = "A" + records = make([]string, 0, len(answers)) + + for _, answer := range answers { + switch { + case strings.HasPrefix(answer, "CNAME "): + recordType = "CNAME" + records = append(records, answer[6:]) + case strings.HasPrefix(answer, "NS "): + recordType = "NS" + records = append(records, answer[3:]) + case strings.HasPrefix(answer, "PTR "): + recordType = "PTR" + records = append(records, answer[4:]) + case strings.HasPrefix(answer, "TXT "): + recordType = "TXT" + records = append(records, answer[4:]) + case strings.HasPrefix(answer, "AAAA "): + recordType = "AAAA" + records = append(records, answer[5:]) + default: + records = append(records, answer) // plain IP → A record + } + } + + if len(records) == 0 { + records = answers } - return nil + return } diff --git a/pkg/runner/outputter/output/screen.go b/pkg/runner/outputter/output/screen.go index 598a422..81488c0 100755 --- a/pkg/runner/outputter/output/screen.go +++ b/pkg/runner/outputter/output/screen.go @@ -45,10 +45,18 @@ func (s *ScreenOutput) WriteDomainResult(domain result.Result) error { } if !s.silent { + // Pad to terminal width to overwrite any progress-bar remnants, + // but do NOT prefix with \r — that would corrupt piped output + // (e.g. `ksubdomain ... --od --silent | httpx`). screenWidth := s.windowsWidth - len(msg) - 1 - gologger.Silentf("\r%s% *s\n", msg, screenWidth, "") + if screenWidth < 0 { + screenWidth = 0 + } + gologger.Silentf("%s% *s\n", msg, screenWidth, "") } else { - gologger.Silentf("\r%s\n", msg) + // silent=true: plain domain-per-line, no control characters. + // This is the canonical pipe-friendly mode (--od --silent | httpx). + gologger.Silentf("%s\n", msg) } return nil } diff --git a/pkg/runner/recv.go b/pkg/runner/recv.go index ce967d3..5a86021 100755 --- a/pkg/runner/recv.go +++ b/pkg/runner/recv.go @@ -180,47 +180,49 @@ func (r *Runner) processPacket(data []byte, dnsChanel chan<- layers.DNS) { } } -// recvChanel 实现接收DNS响应的功能 -func (r *Runner) recvChanel(ctx context.Context, wg *sync.WaitGroup) { +// recvChanelForIface 为单张网卡实现接收DNS响应的功能。 +// 每张网卡独立开一个 InactiveHandle，BPF filter 使用该网卡自己的 listenPort， +// 保证不同网卡的响应互不干扰。 +func (r *Runner) recvChanelForIface(ctx context.Context, wg *sync.WaitGroup, iface *netInterface) { defer wg.Done() var ( snapshotLen = 65536 timeout = 5 * time.Second err error ) - inactive, err := pcap.NewInactiveHandle(r.options.EtherInfo.Device) + inactive, err := pcap.NewInactiveHandle(iface.etherInfo.Device) if err != nil { - gologger.Errorf("创建网络捕获句柄失败: %v", err) + gologger.Errorf("创建网络捕获句柄失败 [%s]: %v", iface.etherInfo.Device, err) return } err = inactive.SetSnapLen(snapshotLen) if err != nil { - gologger.Errorf("设置抓包长度失败: %v", err) + gologger.Errorf("设置抓包长度失败 [%s]: %v", iface.etherInfo.Device, err) return } defer inactive.CleanUp() if err = inactive.SetTimeout(timeout); err != nil { - gologger.Errorf("设置超时失败: %v", err) + gologger.Errorf("设置超时失败 [%s]: %v", iface.etherInfo.Device, err) return } err = inactive.SetImmediateMode(true) if err != nil { - gologger.Errorf("设置即时模式失败: %v", err) + gologger.Errorf("设置即时模式失败 [%s]: %v", iface.etherInfo.Device, err) return } handle, err := inactive.Activate() if err != nil { - gologger.Errorf("激活网络捕获失败: %v", err) + gologger.Errorf("激活网络捕获失败 [%s]: %v", iface.etherInfo.Device, err) return } defer handle.Close() - err = handle.SetBPFFilter(fmt.Sprintf("udp and src port 53 and dst port %d", r.listenPort)) + err = handle.SetBPFFilter(fmt.Sprintf("udp and src port 53 and dst port %d", iface.listenPort)) if err != nil { - gologger.Errorf("设置BPF过滤器失败: %v", err) + gologger.Errorf("设置BPF过滤器失败 [%s]: %v", iface.etherInfo.Device, err) return } diff --git a/pkg/runner/result.go b/pkg/runner/result.go index f4d227d..e937f12 100755 --- a/pkg/runner/result.go +++ b/pkg/runner/result.go @@ -2,9 +2,9 @@ package runner import ( "context" - "fmt" "sync" + kserrors "github.com/boy-hack/ksubdomain/v2/pkg/core/errors" "github.com/boy-hack/ksubdomain/v2/pkg/core/predict" "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" ) @@ -47,7 +47,7 @@ func (r *Runner) handleResult(predictChan chan string) { // predict 根据已知域名预测新的子域名 func (r *Runner) predict(res result.Result, predictChan chan string) error { if r.domainChan == nil { - return fmt.Errorf("域名通道未初始化") + return kserrors.ErrDomainChanNil } _, err := predict.PredictDomains(res.Subdomain, predictChan) if err != nil { diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 0b870fe..7296ed5 100755 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -7,6 +7,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" "github.com/boy-hack/ksubdomain/v2/pkg/core" @@ -21,19 +22,26 @@ import ( "go.uber.org/ratelimit" ) +// netInterface 单张网卡的完整上下文 +type netInterface struct { + etherInfo *device.EtherTable + pcapHandle *pcap.Handle + listenPort int + templateCache sync.Map +} + // Runner 表示子域名扫描的运行时结构 type Runner struct { statusDB *statusdb.StatusDb // 状态数据库 options *options.Options // 配置选项 rateLimiter ratelimit.Limiter // 速率限制器 - pcapHandle *pcap.Handle // 网络抓包句柄 + ifaces []*netInterface // 每张网卡的上下文（替代原 pcapHandle/listenPort） successCount uint64 // 成功数量 sendCount uint64 // 发送数量 receiveCount uint64 // 接收数量 failedCount uint64 // 失败数量 domainChan chan string // 域名发送通道 resultChan chan result.Result // 结果接收通道 - listenPort int // 监听端口 dnsID uint16 // DNS请求ID maxRetryCount int // 最大重试次数 timeoutSeconds int64 // 超时秒数 @@ -49,7 +57,6 @@ func init() { // New 创建一个新的Runner实例 func New(opt *options.Options) (*Runner, error) { - var err error version := pcap.Version() r := new(Runner) gologger.Infof(version) @@ -66,24 +73,42 @@ func New(opt *options.Options) (*Runner, error) { gologger.Infof("特殊DNS服务器: %s\n", core.SliceToString(keys)) } - // 初始化网络设备 - r.pcapHandle, err = device.PcapInit(opt.EtherInfo.Device) - if err != nil { - return nil, err + // 初始化每张网卡的上下文 + allEthers := opt.AllEtherInfos() + if len(allEthers) == 0 { + gologger.Fatalf("没有可用网卡配置\n") } - // 设置速率限制 + r.ifaces = make([]*netInterface, 0, len(allEthers)) + for _, ether := range allEthers { + handle, err := device.PcapInit(ether.Device) + if err != nil { + return nil, err + } + freePort, err := freeport.GetFreePort() + if err != nil { + handle.Close() + return nil, err + } + gologger.Infof("网卡 %s 监听端口: %d\n", ether.Device, freePort) + r.ifaces = append(r.ifaces, &netInterface{ + etherInfo: ether, + pcapHandle: handle, + listenPort: freePort, + }) + } + + // 设置速率限制（多卡时速率平均分配给各卡，这里保持总速率不变） cpuLimit := float64(runtime.NumCPU() * 10000) rateLimit := int(math.Min(cpuLimit, float64(opt.Rate))) - - // Mac 平台优化: BPF 缓冲区限制较严格 - // 建议速率 < 50000 pps 以避免缓冲区溢出 + + // Mac 平台优化 if runtime.GOOS == "darwin" && rateLimit > 50000 { gologger.Warningf("Mac 平台检测到: 当前速率 %d pps 可能导致缓冲区问题\n", rateLimit) gologger.Warningf("建议: 使用 -b 参数限制带宽 (如 -b 5m) 或降低速率\n") gologger.Warningf("提示: Mac BPF 缓冲区已优化至 2MB,但仍建议速率 < 50000 pps\n") } - + r.rateLimiter = ratelimit.New(rateLimit) gologger.Infof("速率限制: %d pps\n", rateLimit) @@ -92,14 +117,6 @@ func New(opt *options.Options) (*Runner, error) { r.resultChan = make(chan result.Result, 5000) r.stopSignal = make(chan struct{}) - // 获取空闲端口 - freePort, err := freeport.GetFreePort() - if err != nil { - return nil, err - } - r.listenPort = freePort - gologger.Infof("监听端口: %d\n", freePort) - // 设置其他参数 r.dnsID = 0x2021 // ksubdomain的生日 r.maxRetryCount = opt.Retry @@ -211,20 +228,40 @@ func (r *Runner) processPredictedDomains(ctx context.Context, wg *sync.WaitGroup } // RunEnumeration 开始子域名枚举过程 +// RunEnumeration runs the full scan pipeline until all domains have been +// sent, retried, and their results collected (or the context is cancelled). +// +// Goroutine topology (multi-NIC A1 pattern): +// +// loadDomainsFromSource ──► domainChan ──┬──► sendCycleForIface(iface0) ──► pcap(iface0) +// ├──► sendCycleForIface(iface1) ──► pcap(iface1) +// └──► ...（共享 domainChan，竞争消费） +// │ +// statusDB (sharded 64-bucket) +// │ +// retry() (200 ms tick) ──► re-inject timed-out +// │ +// recvChanelForIface(iface0) ──► dnsChanel ──► handleResultWithContext +// recvChanelForIface(iface1) ──► dnsChanel ──► handleResultWithContext +// │ +// resultChan +// │ +// outputter.Output func (r *Runner) RunEnumeration(ctx context.Context) { // 创建可取消的上下文 ctx, cancelFunc := context.WithCancel(ctx) defer cancelFunc() - // 创建等待组，现在需要等待5个goroutine（添加了sendCycle和handleResult） + // wg.Add：3 固定 goroutine（monitorProgress + handleResult + loadDomains） + // + len(ifaces)*2（send*N + recv*N） wg := &sync.WaitGroup{} - wg.Add(5) - - // 启动接收处理 - go r.recvChanel(ctx, wg) + wg.Add(3 + len(r.ifaces)*2) - // 启动发送处理（加入waitgroup管理） - go r.sendCycleWithContext(ctx, wg) + // 为每张网卡启动独立的 send/recv goroutine + for _, iface := range r.ifaces { + go r.recvChanelForIface(ctx, wg, iface) + go r.sendCycleForIface(ctx, wg, iface) + } // 监控进度 go r.monitorProgress(ctx, cancelFunc, wg) @@ -233,13 +270,12 @@ func (r *Runner) RunEnumeration(ctx context.Context) { predictChan := make(chan string, 1000) if r.options.Predict { wg.Add(1) - // 启动预测域名处理 go r.processPredictedDomains(ctx, wg, predictChan) } else { r.predictLoadDone <- struct{}{} } - // 启动结果处理（加入waitgroup管理） + // 启动结果处理 go r.handleResultWithContext(ctx, wg, predictChan) // 从源加载域名 @@ -257,9 +293,11 @@ func (r *Runner) RunEnumeration(ctx context.Context) { // Close 关闭Runner并释放资源 func (r *Runner) Close() { - // 关闭网络抓包句柄 - if r.pcapHandle != nil { - r.pcapHandle.Close() + // 关闭所有网卡的 pcap 句柄 + for _, iface := range r.ifaces { + if iface.pcapHandle != nil { + iface.pcapHandle.Close() + } } // 关闭状态数据库 @@ -280,3 +318,9 @@ func (r *Runner) Close() { r.options.ProcessBar.Close() } } + +// SuccessCount returns the number of domains that resolved successfully. +// Call this after Close() to determine whether to use a non-zero exit code. +func (r *Runner) SuccessCount() uint64 { + return atomic.LoadUint64(&r.successCount) +} diff --git a/pkg/runner/send.go b/pkg/runner/send.go index 2ff3e36..aabf6db 100755 --- a/pkg/runner/send.go +++ b/pkg/runner/send.go @@ -9,7 +9,6 @@ import ( "time" "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" - "github.com/boy-hack/ksubdomain/v2/pkg/device" "github.com/boy-hack/ksubdomain/v2/pkg/runner/statusdb" "github.com/google/gopacket" "github.com/google/gopacket/layers" @@ -26,23 +25,16 @@ type packetTemplate struct { dnsip net.IP } -// templateCache 全局DNS服务器模板缓存 -// 优化说明: DNS服务器数量有限(通常<10个),每次创建模板开销较大 -// 使用 sync.Map 缓存模板,避免重复创建以太网/IP/UDP层 -// 预期性能提升: 5-10% (减少内存分配和IP解析开销) -var templateCache sync.Map - -// getOrCreate 获取或创建DNS服务器的数据包模板 -// 优化: 添加模板缓存,同一DNS服务器只创建一次模板 -func getOrCreate(dnsname string, ether *device.EtherTable, freeport uint16) *packetTemplate { - // 优化点1: 先尝试从缓存获取,避免重复创建 - // key格式: dnsname_freeport (同一DNS可能使用不同源端口) +// getOrCreate 获取或创建 DNS 服务器的数据包模板（绑定到单张网卡的 templateCache）。 +// key 格式：dnsname_freeport +func (iface *netInterface) getOrCreate(dnsname string) *packetTemplate { + freeport := uint16(iface.listenPort) cacheKey := dnsname + "_" + string(rune(freeport)) - if cached, ok := templateCache.Load(cacheKey); ok { + if cached, ok := iface.templateCache.Load(cacheKey); ok { return cached.(*packetTemplate) } - // 缓存未命中,创建新模板 + ether := iface.etherInfo DstIp := net.ParseIP(dnsname).To4() eth := &layers.Ethernet{ SrcMAC: ether.SrcMac.HardwareAddr(), @@ -54,7 +46,7 @@ func getOrCreate(dnsname string, ether *device.EtherTable, freeport uint16) *pac Version: 4, IHL: 5, TOS: 0, - Length: 0, // FIX + Length: 0, Id: 0, Flags: layers.IPv4DontFragment, FragOffset: 0, @@ -84,96 +76,51 @@ func getOrCreate(dnsname string, ether *device.EtherTable, freeport uint16) *pac buf: gopacket.NewSerializeBuffer(), } - // 存入缓存供后续复用 - templateCache.Store(cacheKey, template) + iface.templateCache.Store(cacheKey, template) return template } -// sendCycle 实现发送域名请求的循环 -func (r *Runner) sendCycle() { - // 从发送通道接收域名，分发给工作协程 - for domain := range r.domainChan { - r.rateLimiter.Take() - v, ok := r.statusDB.Get(domain) - if !ok { - v = statusdb.Item{ - Domain: domain, - Dns: r.selectDNSServer(domain), - Time: time.Now(), - Retry: 0, - DomainLevel: 0, - } - r.statusDB.Add(domain, v) - } else { - v.Retry += 1 - v.Time = time.Now() - v.Dns = r.selectDNSServer(domain) - r.statusDB.Set(domain, v) - } - send(domain, v.Dns, r.options.EtherInfo, r.dnsID, uint16(r.listenPort), r.pcapHandle, layers.DNSTypeA) - atomic.AddUint64(&r.sendCount, 1) - } -} - -// sendCycleWithContext 实现带有context管理的发送域名请求循环 -// 优化: 添加批量发送机制,减少系统调用次数 -func (r *Runner) sendCycleWithContext(ctx context.Context, wg *sync.WaitGroup) { +// sendCycleForIface 为单张网卡运行发送循环（A1 方案）。 +// 多个 goroutine 共享同一个 domainChan，竞争消费实现自然负载均衡。 +func (r *Runner) sendCycleForIface(ctx context.Context, wg *sync.WaitGroup, iface *netInterface) { defer wg.Done() - // 优化点2: 批量发送机制 - // 批量大小: 每次收集100个域名后一起处理 - // 收益: 减少系统调用次数,提升发包吞吐量 20-30% const batchSize = 100 batch := make([]string, 0, batchSize) batchItems := make([]statusdb.Item, 0, batchSize) - // 定时器: 确保即使凑不满批次也能及时发送 ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() - // 批量发送函数 sendBatch := func() { if len(batch) == 0 { return } - - // 批量发送所有域名 for i, domain := range batch { - send(domain, batchItems[i].Dns, r.options.EtherInfo, r.dnsID, - uint16(r.listenPort), r.pcapHandle, layers.DNSTypeA) + send(domain, batchItems[i].Dns, iface, r.dnsID, layers.DNSTypeA) } - - // 原子更新发送计数 atomic.AddUint64(&r.sendCount, uint64(len(batch))) - - // 清空批次,复用底层数组 batch = batch[:0] batchItems = batchItems[:0] } - // 主循环: 收集域名并批量发送 for { select { case <-ctx.Done(): - // 退出前发送剩余批次 sendBatch() return case <-ticker.C: - // 定时发送,避免批次未满时延迟过高 sendBatch() case domain, ok := <-r.domainChan: if !ok { - // 通道关闭,发送剩余批次后退出 sendBatch() return } - // 速率限制 r.rateLimiter.Take() - // 获取或创建域名状态 v, ok := r.statusDB.Get(domain) if !ok { v = statusdb.Item{ @@ -191,11 +138,9 @@ func (r *Runner) sendCycleWithContext(ctx context.Context, wg *sync.WaitGroup) { r.statusDB.Set(domain, v) } - // 添加到批次 batch = append(batch, domain) batchItems = append(batchItems, v) - // 批次已满,立即发送 if len(batch) >= batchSize { sendBatch() } @@ -203,25 +148,20 @@ func (r *Runner) sendCycleWithContext(ctx context.Context, wg *sync.WaitGroup) { } } -// send 发送单个DNS查询包 -func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, freeport uint16, handle *pcap.Handle, dnsType layers.DNSType) { - // 复用DNS服务器的包模板 - template := getOrCreate(dnsname, ether, freeport) +// send 发送单个DNS查询包（绑定到指定网卡上下文） +func send(domain string, dnsname string, iface *netInterface, dnsid uint16, dnsType layers.DNSType) { + template := iface.getOrCreate(dnsname) - // 从内存池获取DNS层对象 dns := GlobalMemPool.GetDNS() defer GlobalMemPool.PutDNS(dns) - // 设置DNS查询参数 dns.ID = dnsid dns.QDCount = 1 - dns.RD = true // 递归查询标识 + dns.RD = true - // 从内存池获取questions切片 questions := GlobalMemPool.GetDNSQuestions() defer GlobalMemPool.PutDNSQuestions(questions) - // 添加查询问题 questions = append(questions, layers.DNSQuestion{ Name: []byte(domain), Type: dnsType, @@ -229,11 +169,9 @@ func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, }) dns.Questions = questions - // 从内存池获取序列化缓冲区 buf := GlobalMemPool.GetBuffer() defer GlobalMemPool.PutBuffer(buf) - // 序列化数据包 err := gopacket.SerializeLayers( buf, template.opts, @@ -244,40 +182,34 @@ func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, return } - // 发送数据包 - // 修复 Mac 缓冲区问题: 增加重试机制,使用指数退避 + handle := iface.pcapHandle const maxRetries = 3 for retry := 0; retry < maxRetries; retry++ { err = handle.WritePacketData(buf.Bytes()) if err == nil { - return // 发送成功 + return } - + errMsg := err.Error() - - // 检查是否为缓冲区错误 (Mac/Linux 常见) - // Mac BPF: "No buffer space available" (ENOBUFS) - // Linux: 可能有类似错误 + isBufferError := strings.Contains(errMsg, "No buffer space available") || strings.Contains(errMsg, "ENOBUFS") || strings.Contains(errMsg, "buffer") - + if isBufferError { - // 缓冲区满,需要重试 if retry < maxRetries-1 { - // 指数退避: 10ms, 20ms, 40ms backoff := time.Millisecond * time.Duration(10*(1< 本文件是面向 AI Agent 的项目上下文与任务指引。Agent 应首先通读此文件，再开始任何工作。 +> 类似 autoresearch 的 `program.md`，这里定义了项目的"研究组织逻辑"。 + +--- + +## 项目概述 + +**KSubdomain** 是一个无状态子域名枚举工具，核心思路来自 Masscan：绕过内核协议栈，直接在网卡层面收发 raw DNS 包，配合内置轻量状态表做重传，从而实现极速扫描。 + +当前版本 **v2.5**，模块路径 `github.com/boy-hack/ksubdomain/v2`，使用 Go 1.23+。 + +--- + +## 代码地图（必读） + +``` +cmd/ksubdomain/ CLI 入口，4 个子命令：enum / verify / test / device +pkg/ + options/options.go 核心配置结构体 Options（所有功能的控制面板） + runner/runner.go 扫描主循环，RunEnumeration() 启动 5 条并发 goroutine + runner/send.go 发包：模板缓存 + 批量发送 + 指数退避重试 + runner/recv.go 收包：BPF filter + CPU*2 并行解析 + CPU*2 并行处理 + runner/statusdb/db.go 64 分片锁状态表，记录发送状态与重传计数 + runner/mempool.go 全局 sync.Pool，复用 DNS 结构体减少 GC + runner/result.go 结果处理 + predict 预测触发 + runner/wildcard.go 泛解析过滤逻辑 + runner/outputter/ Output 接口 + txt/json/csv/jsonl/screen 实现 + device/ 网卡初始化（pcap），构建 EtherTable + sdk/sdk.go Go SDK 封装（Enum / Verify / *WithContext） + ns/ns.go NS 记录查询（--ns 模式辅助） +internal/ + predict/generator.go 基于 regular.cfg + regular.dict 的预测生成器 + assets/data/ 内置字典 subdomain.txt / subnext.txt +docs/ + OUTPUT_FORMATS.md 输出格式完整说明 +``` + +--- + +## 核心数据流 + +``` +Domain 输入 (chan string) + │ + ▼ +loadDomainsFromSource ──► domainChan + │ + ┌───────────┘ + ▼ + sendCycleWithContext + （模板缓存 + 批量发包） + │ + ▼ + statusdb 登记 {Domain, Dns, Time, Retry} + │ + ┌─────────────┴──────────────────────────────────┐ + ▼ ▼ + recvChan → packetChan → dnsChan → resultChan monitorProgress + │ + retry() + ▼ + handleResultWithContext + │ + ┌────────────┴──────────────┐ + ▼ ▼ + Output.WriteDomainResult() predict() → predictChan +``` + +--- + +## 关键指标（以此衡量改动是否有效） + +| 指标 | 当前基准 | 目标方向 | +|------|---------|---------| +| **检测速率** | 100k 域名约 30 秒（5M 带宽，4 核） | 同等带宽继续提升吞吐 | +| **漏报率** | 对比 massdns 结果几乎一致 | 减少 timeout 造成的漏报 | +| **误报率** | 泛解析过滤后接近 0 | advanced 模式覆盖更多泛解析场景 | +| **内存占用** | 百万级域名仍维持低内存 | 不引入新的大块堆分配 | +| **SDK 易用性** | NewScanner(config).Enum(domain) 3 行接入 | 任何改动不破坏 SDK 接口兼容性 | + +每次修改后，执行以下命令验证基准不退步： + +```bash +# 单元测试（无需网卡） +go test ./... + +# 集成测试（需要网卡权限） +sudo go test ./test/ -v -run Integration + +# 发包速率基准 +sudo ./ksubdomain test +``` + +--- + +## 当前优先任务（Roadmap） + +按优先级从高到低排列，Agent 应按序处理，每次专注一个任务。 + +**Roadmap 维护规则（Agent 可直接修改此文件）**： +- 完成一个任务 → 将 `[ ]` 改为 `[x]`，并在 `agent-log.md` 记录结果 +- 发现新问题或新改进点 → 自行在对应优先级下追加条目，注明发现来源 +- 评估某条任务不再必要 → 将 `[ ]` 改为 `[~]` 并在条目后用括号注明原因 +- **需要用户决策的事项不在此处等待**，而是写入 `agent-log.md` 的 `【待决策】` 区块，由用户查阅日志后自行处理 + +### P0 — 检测效率 + +- [x] **动态超时自适应**：RTT EWMA滑动均值，上界内部固定10s，始终启用 +- [x] **接收侧背压控制**：packetChan双水位监控（80%触发/50%恢复），send侧背压时sleep 5ms降速 +- [x] **批量重传合并**：按DNS server分组批量send()，去掉channel中转层，复用slice/map减少GC + +### P1 — 开发者集成 + +- [x] **流式结果回调**：EnumStream/VerifyStream，streamCollector 实时回调，无缓冲 +- [x] **自定义 Output 接入**：Config.ExtraWriters []outputter.Output，追加到内部 writer 之后 +- [x] **错误类型化**：pkg/core/errors 包，ErrPermissionDenied/ErrDeviceNotFound/ErrDeviceNotActive/ErrPcapInit/ErrDomainChanNil，sdk 重导出 +- [x] **dev.md 重写**：架构图、goroutine 表、SDK/Options 字段表、错误处理、背压说明、构建命令 + +### P1.5 — 工具联动兼容性审查 + +> 参考「工具联动兼容性矩阵」章节，逐条验证每个集成场景是否真实可用。 + +- [x] **httpx 管道**：移除 screen.go 的 \r 前缀，--od --silent 输出干净 domain\n +- [x] **JSONL 下游兼容**：bufio 替换 Sync()，parseAnswers() 共享解析，字段名 domain/type/records 稳定 +- [x] **退出码语义**：SuccessCount()==0 时 os.Exit(1)，文档化在 faq.md + +### P2 — 文档完善 + +- [x] **docs/quickstart.md**：安装、首次扫描、常用参数表、输出格式、故障排查 +- [x] **docs/api.md**：完整 SDK API 参考，Config/Result/Scanner 方法、错误处理、自定义 sink +- [x] **docs/best-practices.md**：带宽选择表、DNS resolver 指南、泛解析、管道输出、大规模扫描 +- [x] **docs/faq.md**：sudo/CAP_NET_RAW、网卡错误、macOS BPF、WSL2、泛解析、httpx 乱码、退出码 +- [x] **内联注释**：RunEnumeration goroutine 拓扑图，sendCycleWithContext 批量+背压说明 + +### P3 — 工程健康 + +- [x] **`simple` 二进制缺失**：确认为编译产物，加入 .gitignore，更新 examples 代码修复 API +- [x] **CI 矩阵**：ci.yml 五平台矩阵（Linux amd64/arm64、macOS amd64/arm64、Windows amd64）+ lint job +- [x] **版本号自动化**：Version 改为 var，build.yml/build.sh 通过 ldflags 注入 git tag + +--- + +## 修改约束 + +1. **不修改 `pkg/options/options.go` 中 `Options` 的已有字段名**：下游用户代码依赖此结构体，字段重命名视为破坏性变更 +2. **不修改 `pkg/sdk/sdk.go` 中 `Config`、`Scanner`、`Result` 的已有字段和方法签名**：SDK 公开 API，需向后兼容 +3. **`Output` 接口只可扩展，不可修改已有方法**：现有 `WriteDomainResult` 和 `Close` 签名不变 +4. **所有平台文件（`*_darwin.go`、`*_linux.go`、`*_windows.go`）需同步更新** +5. **Go 规则**：不使用 `++`/`--`，字符串用反引号或双引号（Go 惯例），接口定义只放在 `pkg/` 下 + +--- + +## 上手流程（Agent 首次运行） + +```bash +# 1. 查看项目状态 +git status +go build ./... + +# 2. 运行现有测试，确认基线通过 +go test ./pkg/... -v + +# 3. 检查缺失文档（README 中引用但未创建的文件） +ls docs/ + +# 4. 选择 Roadmap 中最高优先级未完成项，新建分支后开始实现 +# 每次只聚焦一个任务，完成后运行 go test ./... 确认无回归 +``` + +--- + +## 测试规范（内网环境） + +> **当前为内网环境，禁止发起大批量外部 DNS 请求。** + +Agent 在实现每个功能后，必须自行运行以下测试，验证参数行为正确、无崩溃、无明显 bug： + +```bash +# 编译验证（无需权限，最快检查） +go build ./... + +# 单元测试（无需网卡，全部通过为准入条件） +go test ./... -timeout 30s + +# 本地轻量功能冒烟测试（只需几条域名，不做大批量扫描） +# 构建二进制 +go build -o ksubdomain_dev ./cmd/ksubdomain + +# verify 模式：验证 1 条已知域名，检查 -d / -o / --oy / --silent / --np 参数 +sudo ./ksubdomain_dev verify -d www.baidu.com --timeout 5 --retry 2 -b 1m --np +sudo ./ksubdomain_dev verify -d www.baidu.com --oy jsonl --silent +sudo ./ksubdomain_dev verify -d www.baidu.com -o /tmp/ksub_test.txt && cat /tmp/ksub_test.txt + +# enum 模式：只枚举 1 个域名、新建一个小字典,不超过10行，检查基本流程不崩溃 +sudo ./ksubdomain_dev enum -d baidu.com --timeout 5 --retry 1 -b 1m --np -f subdomain.dic + +# test 模式：测试本地网卡发包速率，无外部流量 +sudo ./ksubdomain_dev test + +# device 模式：列出可用网卡 +sudo ./ksubdomain_dev device +``` + +**测试要点**： +- 每个 CLI 参数至少覆盖一次，确认有无 panic 或非预期输出 +- 输出文件格式正确（txt 每行一条、jsonl 每行合法 JSON） +- `--silent` 模式下无多余输出，`--np` 模式下无域名打印 +- 测试完毕后删除临时文件：`rm -f /tmp/ksub_test* ksubdomain_dev` + +--- + +## 分支管理规范 + +> **main 分支只读，任何功能改动都在独立分支上进行。** + +每次实现一个 Roadmap 任务，遵循以下流程： + +```bash +# 1. 从 main 新建功能分支，命名格式：feature/<简短描述> +git checkout main +git checkout -b feature/dynamic-timeout # P0 示例 +git checkout -b feature/stream-sdk # P1 示例 +git checkout -b docs/quickstart # P2 示例 +git checkout -b fix/simple-binary # P3 示例 + +# 2. 实现功能，提交粒度：每个逻辑单元一次 commit +git add . +git commit -m "feat(runner): add RTT sliding window for dynamic timeout" + +# 3. 功能完成后在分支上运行完整测试 +go test ./... -timeout 30s + +# 4. 将分支结论记录到运行日志（见下文），不合并到 main +# main 分支由人工决定何时合并 +``` + +**分支命名前缀约定**： + +| 前缀 | 用途 | +|------|------| +| `feature/` | 新功能实现（对应 Roadmap P0/P1） | +| `docs/` | 文档补充（对应 Roadmap P2） | +| `fix/` | Bug 修复（对应 Roadmap P3 或冒烟测试发现的问题） | +| `refactor/` | 重构（不改变外部行为） | +| `exp/` | 实验性改动（不确定是否保留） | + +--- + +## 运行日志规范 + +Agent 每完成一个任务，必须在 `agent-log.md` 文件中追加一条记录。该文件位于项目根目录，不存在则自行创建。 + +**记录格式**： + +```markdown +## [YYYY-MM-DD] <分支名> — <任务简述> + +**目标**：对应 Roadmap 中的哪一条 +**改动文件**：列出修改的文件 +**测试结果**： +- go test ./... → PASS / FAIL（附错误摘要） +- 冒烟测试：列出执行的命令及输出摘要 +**结论**：改动是否有效，是否引入新问题，下一步建议 + + +### 【待决策】 +``` + +**说明**： +- `【待决策】` 区块是 Agent 与用户沟通的唯一通道。Agent 不中断工作等待回复，而是记录后继续处理下一任务。 +- 用户查看 `agent-log.md` 时，搜索 `【待决策】` 即可找到所有待处理事项，勾选 `[x]` 或在条目下方回复意见后，Agent 下次运行时读取并执行。 +- 典型的待决策场景：破坏性 API 变更是否接受、新增外部依赖是否引入、某功能是否值得实现、发现潜在安全或性能风险需要人工确认。 + +**示例**： + +```markdown +## [2026-03-17] feature/dynamic-timeout — 动态超时自适应 + +**目标**：P0 动态超时自适应 +**改动文件**：pkg/runner/runner.go, pkg/runner/send.go, pkg/options/options.go +**测试结果**： +- go test ./... → PASS +- 冒烟测试：`sudo ./ksubdomain_dev verify -d www.baidu.com --timeout 5 --retry 2` + 输出：`www.baidu.com => 110.242.68.66`，耗时约 2s，正常 +**结论**：RTT 滑动均值计算正常，低延迟场景下超时提前收敛，无漏报。 + 建议后续在 P0 背压控制任务中复用 RTT 数据。 + +### 【待决策】 + 若希望新版本默认开启动态超时，需将默认值改为 true，这属于行为变更，请确认。 + 方案 A：默认 false，用户显式 --dynamic-timeout 启用（保守） + 方案 B：默认 true，--no-dynamic-timeout 可关闭（激进，影响现有脚本） +``` + +--- + +## 输出格式速查 + +| 格式 | 写入时机 | 适用场景 | +|------|---------|---------| +| `txt` | 实时（每条结果） | 人工查阅、管道 grep | +| `json` | 完成后一次性 | 离线分析、脚本处理 | +| `csv` | 完成后一次性 | Excel/数据库导入 | +| `jsonl` | 实时流式 | 管道链（httpx/nuclei）、监控系统 | +| screen | 实时彩色（stdout） | 交互式终端 | + +--- + +## 工具联动兼容性矩阵 + +> Agent 在审查 P1.5 任务时，以此为检查清单。每条给出期望行为、验证命令和当前状态。 + +### ProjectDiscovery 工具链 + +| 工具 | 联动方式 | 期望行为 | 关键参数 | 当前状态 | +|------|---------|---------|---------|---------| +| **httpx** | stdout 管道 | 每行一个域名，httpx 自动探活 | `--od --silent` | 待验证 | +| **nuclei** | stdout 管道或临时文件 | 域名作为目标，模板正常扫描 | `--od` + nuclei `-l /dev/stdin` | 待验证 | +| **naabu** | stdout 管道 | 域名列表作为端口扫描输入 | `--od` + naabu `-iL -` | 待验证 | +| **dnsx** | stdout 管道 | ksubdomain 初筛 → dnsx 多记录精查 | `--od --silent` + dnsx `-a -cname` | 待验证 | +| **subfinder** | subfinder → ksubdomain stdin | subfinder 发现域名 → ksubdomain verify 存活确认 | ksubdomain `v --stdin` | 待验证 | +| **alterx** | alterx → ksubdomain stdin | 排列生成 → 批量验证，大量输入不丢包 | ksubdomain `v --stdin -b 10m` | 待验证 | +| **katana** | ksubdomain → katana | 子域名列表作为爬取种子 | `--od` + katana `-list -` | 待验证 | +| **chaos** | chaos → ksubdomain verify | chaos 数据集导入验证存活 | ksubdomain `v -f chaos_output.txt` | 待验证 | + +### 验证命令参考（内网用小字典，不发大批量请求） + +```bash +# 1. httpx 联动：枚举 → HTTP 探活 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent | httpx -silent -title + +# 2. dnsx 联动：ksubdomain 初筛 → dnsx 多类型查询 +sudo ./ksubdomain_dev verify -d www.baidu.com --od --silent | dnsx -a -cname -resp + +# 3. naabu 联动：枚举 → 端口扫描 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent | naabu -silent -p 80,443 + +# 4. subfinder → ksubdomain：二阶段发现 +subfinder -d baidu.com -silent | sudo ./ksubdomain_dev verify --stdin --silent --od + +# 5. alterx → ksubdomain：排列验证 +echo 'www.baidu.com' | alterx -silent | sudo ./ksubdomain_dev verify --stdin --np + +# 6. JSONL 流式过滤后交给 httpx +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --oy jsonl --silent \ + | jq -r 'select(.type=="A") | .domain' \ + | httpx -silent + +# 7. nuclei 漏洞扫描 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent \ + | nuclei -l /dev/stdin -t technologies/ -silent +``` + +### 联动验证要点 + +- **`--od` 输出必须是纯域名单列**，无 `=>` 箭头、无 IP、无空行，否则下游工具解析失败 +- **`--silent` 必须屏蔽进度条和 banner**，只保留结果行，否则污染管道数据 +- **`--stdin` 必须正确处理 EOF**，subfinder/alterx 结束后 ksubdomain 应正常退出而非挂起 +- **退出码**：有结果退出 0，无结果退出非 0，让 shell `&&` 链路能正确短路 +- **JSONL 字段名**必须稳定（`domain`、`type`、`records`、`timestamp`），jq 脚本依赖此约定 + +--- + +## 集成速查 + +```bash +# 完整侦察链：子域名枚举 → HTTP 探活 → 漏洞扫描 +sudo ./ksubdomain enum -d example.com -f subdomain.dic --od --silent \ + | httpx -silent \ + | nuclei -l /dev/stdin -silent + +# 二阶段发现：subfinder 粗扫 → ksubdomain 精筛存活 +subfinder -d example.com -silent \ + | sudo ./ksubdomain verify --stdin --od --silent + +# 流式过滤（仅 A 记录）后端口扫描 +sudo ./ksubdomain enum -d example.com --oy jsonl --silent \ + | jq -r 'select(.type=="A") | .domain' \ + | naabu -silent -p 80,443,8080,8443 + +# Go SDK 最简集成 +scanner := sdk.NewScanner(sdk.DefaultConfig) +results, err := scanner.Enum('example.com') +``` + +--- + +*`program.md` 由 Agent 和人工共同维护：Roadmap 条目由 Agent 自主更新，需要人工判断的事项统一写入 `agent-log.md` 的 `【待决策】` 区块。* diff --git a/sdk/examples/advanced/main.go b/sdk/examples/advanced/main.go index 4d521d2..00762dc 100644 --- a/sdk/examples/advanced/main.go +++ b/sdk/examples/advanced/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "log" "time" @@ -10,59 +11,63 @@ import ( ) func main() { - // Advanced configuration + // Advanced configuration. + // Note: Timeout field no longer exists — the scanner uses a dynamic + // RTT-based timeout with a hardcoded upper bound of 10 s. scanner := sdk.NewScanner(&sdk.Config{ Bandwidth: "10m", Retry: 5, - Timeout: 10, Resolvers: []string{"8.8.8.8", "1.1.1.1"}, Predict: true, WildcardFilter: "advanced", Silent: false, }) - // Create context with timeout + // Context with a hard wall-clock limit for the whole scan. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - // Start scanning fmt.Println("🚀 Starting advanced scan...") start := time.Now() results, err := scanner.EnumWithContext(ctx, "example.com") if err != nil { - if err == context.DeadlineExceeded { - fmt.Println("⏰ Scan timeout, showing partial results") - } else { + switch { + case errors.Is(err, context.DeadlineExceeded): + fmt.Println("⏰ Scan reached 2-minute wall-clock limit; showing partial results") + case errors.Is(err, sdk.ErrPermissionDenied): + log.Fatal("permission denied — run with sudo or grant CAP_NET_RAW") + default: log.Fatal(err) } } elapsed := time.Since(start) - // Statistics + // Aggregate stats typeCount := make(map[string]int) for _, r := range results { typeCount[r.Type]++ } - // Print results fmt.Printf("\n📊 Scan Results:\n") - fmt.Printf(" Total: %d subdomains\n", len(results)) - fmt.Printf(" Time: %v\n", elapsed.Round(time.Millisecond)) - fmt.Printf(" Speed: %.0f domains/s\n\n", float64(len(results))/elapsed.Seconds()) + fmt.Printf(" Total: %d subdomains\n", len(results)) + fmt.Printf(" Time: %v\n", elapsed.Round(time.Millisecond)) + if elapsed.Seconds() > 0 { + fmt.Printf(" Speed: %.0f domains/s\n", float64(len(results))/elapsed.Seconds()) + } - fmt.Println("📋 Record Types:") + fmt.Println("\n📋 Record Types:") for recType, count := range typeCount { fmt.Printf(" %s: %d\n", recType, count) } - fmt.Println("\n✅ Discovered Subdomains:") - for i, result := range results { + fmt.Println("\n✅ Discovered Subdomains (first 10):") + for i, r := range results { if i >= 10 { fmt.Printf(" ... and %d more\n", len(results)-10) break } - fmt.Printf(" %-30s [%s] %v\n", result.Domain, result.Type, result.Records) + fmt.Printf(" %-40s [%-5s] %v\n", r.Domain, r.Type, r.Records) } } diff --git a/sdk/examples/simple/main.go b/sdk/examples/simple/main.go index 68a5a86..df464a4 100644 --- a/sdk/examples/simple/main.go +++ b/sdk/examples/simple/main.go @@ -1,26 +1,34 @@ package main import ( + "errors" "fmt" "log" + "strings" "github.com/boy-hack/ksubdomain/v2/sdk" ) func main() { - // Create scanner with default configuration + // Create scanner with default configuration. + // Timeout is managed automatically (dynamic RTT-based, upper bound 10 s). scanner := sdk.NewScanner(sdk.DefaultConfig) - // Enumerate subdomains fmt.Println("Scanning example.com...") results, err := scanner.Enum("example.com") if err != nil { - log.Fatal(err) + switch { + case errors.Is(err, sdk.ErrPermissionDenied): + log.Fatal("permission denied — run with sudo or grant CAP_NET_RAW") + case errors.Is(err, sdk.ErrDeviceNotFound): + log.Fatal("network device not found — check your interface name") + default: + log.Fatal(err) + } } - // Print results fmt.Printf("\nFound %d subdomains:\n\n", len(results)) - for _, result := range results { - fmt.Printf("%-30s => %s\n", result.Domain, result.Records[0]) + for _, r := range results { + fmt.Printf("%-40s [%-5s] %s\n", r.Domain, r.Type, strings.Join(r.Records, ", ")) } } diff --git a/sdk/sdk.go b/sdk/sdk.go index 096b486..805cdef 100644 --- a/sdk/sdk.go +++ b/sdk/sdk.go @@ -1,20 +1,28 @@ // Package sdk provides a simple Go SDK for ksubdomain -// -// Example: +// +// # Basic usage (blocking, collect all results): // // scanner := sdk.NewScanner(&sdk.Config{ // Bandwidth: "5m", // Retry: 3, // }) -// +// // results, err := scanner.Enum("example.com") // if err != nil { // log.Fatal(err) // } -// -// for _, result := range results { -// fmt.Printf("%s => %s\n", result.Domain, result.IP) +// for _, r := range results { +// fmt.Printf("%s => %v\n", r.Domain, r.Records) // } +// +// # Stream usage (callback-based, real-time): +// +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// err := scanner.EnumStream(ctx, "example.com", func(r sdk.Result) { +// fmt.Printf("%s => %v\n", r.Domain, r.Records) +// }) package sdk import ( @@ -22,8 +30,8 @@ import ( "fmt" "strings" "sync" - "time" + kserrors "github.com/boy-hack/ksubdomain/v2/pkg/core/errors" "github.com/boy-hack/ksubdomain/v2/pkg/core/options" "github.com/boy-hack/ksubdomain/v2/pkg/runner" "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" @@ -31,41 +39,59 @@ import ( "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" ) -// Config Scanner configuration +// Re-export sentinel errors so SDK users need only import this package. +var ( + ErrPermissionDenied = kserrors.ErrPermissionDenied + ErrDeviceNotFound = kserrors.ErrDeviceNotFound + ErrDeviceNotActive = kserrors.ErrDeviceNotActive + ErrPcapInit = kserrors.ErrPcapInit + ErrDomainChanNil = kserrors.ErrDomainChanNil +) + +// Config holds scanner configuration. +// Timeout is no longer configurable: the scanner uses a dynamic RTT-based +// timeout with a hardcoded upper bound of 10 s. type Config struct { - // Bandwidth downstream speed (e.g., "5m", "10m", "100m") + // Bandwidth limit (e.g., "5m", "10m", "100m") Bandwidth string - // Retry count (-1 for infinite) + // Retry count for timed-out queries (-1 for infinite) Retry int - // Timeout in seconds - Timeout int - - // DNS resolvers (nil for default) + // DNS resolvers; nil means built-in defaults Resolvers []string - // Network adapter name (empty for auto-detect) + // Network adapter name; empty means auto-detect (single NIC, kept for backward-compat) Device string - // Dictionary file path (for enum mode) + // Devices specifies one or more network interface names for multi-NIC parallel sending. + // If non-empty, takes precedence over Device. + Devices []string + + // Dictionary file path (for Enum mode); empty means built-in list Dictionary string - // Enable prediction mode + // Enable AI-powered subdomain prediction Predict bool - // Wildcard filter mode: "none", "basic", "advanced" + // Wildcard filter mode: "none" (default), "basic", "advanced" WildcardFilter string - // Silent mode (no progress bar) + // Silent disables the progress bar Silent bool + + // ExtraWriters allows callers to inject custom output sinks. + // Each writer's WriteDomainResult is called for every resolved result, + // in addition to the SDK's internal collection/stream logic. + // Writers must implement outputter.Output (WriteDomainResult + Close). + // Close() on each writer is called after the scan completes. + ExtraWriters []outputter.Output } -// DefaultConfig returns default configuration +// DefaultConfig is a ready-to-use Config with sensible defaults. var DefaultConfig = &Config{ Bandwidth: "5m", Retry: 3, - Timeout: 6, Resolvers: nil, Device: "", Dictionary: "", @@ -74,132 +100,194 @@ var DefaultConfig = &Config{ Silent: false, } -// Result scan result +// Result represents a single resolved subdomain. type Result struct { - Domain string // Subdomain - Type string // Record type (A, CNAME, NS, etc.) + Domain string // Resolved subdomain + Type string // DNS record type (A, CNAME, NS, PTR, …) Records []string // Record values } -// Scanner subdomain scanner +// Scanner performs subdomain enumeration and verification. type Scanner struct { config *Config } -// NewScanner creates a new scanner with given config +// NewScanner creates a Scanner. If config is nil, DefaultConfig is used. func NewScanner(config *Config) *Scanner { if config == nil { config = DefaultConfig } - - // Apply defaults if config.Bandwidth == "" { config.Bandwidth = "5m" } if config.Retry == 0 { config.Retry = 3 } - if config.Timeout == 0 { - config.Timeout = 6 - } - - return &Scanner{ - config: config, - } + return &Scanner{config: config} } -// Enum enumerates subdomains for given domain +// --------------------------------------------------------------------------- +// Blocking API +// --------------------------------------------------------------------------- + +// Enum enumerates subdomains for domain, returning all results when done. func (s *Scanner) Enum(domain string) ([]Result, error) { return s.EnumWithContext(context.Background(), domain) } -// EnumWithContext enumerates subdomains with context support +// EnumWithContext is like Enum but respects ctx for cancellation. func (s *Scanner) EnumWithContext(ctx context.Context, domain string) ([]Result, error) { - // Load dictionary - var dictChan chan string - if s.config.Dictionary != "" { - // TODO: Load from file - return nil, fmt.Errorf("dictionary file not yet implemented in SDK") - } else { - // Use built-in default dictionary - dictChan = make(chan string, 1000) - go func() { - defer close(dictChan) - // Load built-in subdomain list - // This will be implemented using core.GetDefaultSubdomainData() - subdomains := []string{"www", "mail", "ftp", "blog", "api", "dev", "test"} - for _, sub := range subdomains { - dictChan <- sub + "." + domain - } - }() + dictChan, err := s.buildDictChan(domain) + if err != nil { + return nil, err } - - return s.scan(ctx, dictChan, options.EnumType) + return s.scanCollect(ctx, dictChan, options.EnumType) } -// Verify verifies a list of domains +// Verify verifies each domain in domains, returning those that resolve. func (s *Scanner) Verify(domains []string) ([]Result, error) { return s.VerifyWithContext(context.Background(), domains) } -// VerifyWithContext verifies domains with context support +// VerifyWithContext is like Verify but respects ctx. func (s *Scanner) VerifyWithContext(ctx context.Context, domains []string) ([]Result, error) { - domainChan := make(chan string, len(domains)) - for _, domain := range domains { - domainChan <- domain + ch := make(chan string, len(domains)) + for _, d := range domains { + ch <- d } - close(domainChan) + close(ch) + return s.scanCollect(ctx, ch, options.VerifyType) +} + +// --------------------------------------------------------------------------- +// Stream API +// --------------------------------------------------------------------------- - return s.scan(ctx, domainChan, options.VerifyType) +// EnumStream enumerates subdomains for domain and calls callback for each +// result as it arrives. It blocks until scanning is complete or ctx is +// cancelled. +// +// callback is called from multiple goroutines; implementations must be +// goroutine-safe (e.g., protect shared state with a mutex). +// +// Example: +// +// err := scanner.EnumStream(ctx, "example.com", func(r sdk.Result) { +// fmt.Println(r.Domain) +// }) +func (s *Scanner) EnumStream(ctx context.Context, domain string, callback func(Result)) error { + dictChan, err := s.buildDictChan(domain) + if err != nil { + return err + } + return s.scanStream(ctx, dictChan, options.EnumType, callback) } -// scan internal scan implementation -func (s *Scanner) scan(ctx context.Context, domainChan chan string, method string) ([]Result, error) { - // Collect results - collector := &resultCollector{ - results: make([]Result, 0), +// VerifyStream verifies domains and calls callback for each resolved result. +// It blocks until scanning is complete or ctx is cancelled. +func (s *Scanner) VerifyStream(ctx context.Context, domains []string, callback func(Result)) error { + ch := make(chan string, len(domains)) + for _, d := range domains { + ch <- d } + close(ch) + return s.scanStream(ctx, ch, options.VerifyType, callback) +} - // Get resolvers +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +// buildDictChan returns a channel that emits fully-qualified domain names +// to enumerate under domain. +func (s *Scanner) buildDictChan(domain string) (chan string, error) { + if s.config.Dictionary != "" { + // TODO: load from file + return nil, fmt.Errorf("dictionary file loading not yet implemented in SDK") + } + ch := make(chan string, 1000) + go func() { + defer close(ch) + // Built-in minimal list; full implementation uses core.GetDefaultSubdomainData() + subdomains := []string{"www", "mail", "ftp", "blog", "api", "dev", "test"} + for _, sub := range subdomains { + ch <- sub + "." + domain + } + }() + return ch, nil +} + +// buildOptions constructs runner.Options from the scanner config. +// primaryWriter is the SDK-internal writer (resultCollector or streamCollector). +// Any ExtraWriters from config are appended after it. +func (s *Scanner) buildOptions(domainChan chan string, method options.OptionMethod, primaryWriter outputter.Output) *options.Options { resolvers := options.GetResolvers(s.config.Resolvers) - // Build options + writers := make([]outputter.Output, 0, 1+len(s.config.ExtraWriters)) + writers = append(writers, primaryWriter) + writers = append(writers, s.config.ExtraWriters...) + opt := &options.Options{ Rate: options.Band2Rate(s.config.Bandwidth), Domain: domainChan, Resolvers: resolvers, Silent: s.config.Silent, - TimeOut: s.config.Timeout, Retry: s.config.Retry, Method: method, - Writer: []outputter.Output{collector}, - ProcessBar: &processbar2.FakeProcess{}, - EtherInfo: options.GetDeviceConfig(resolvers), + Writer: writers, + ProcessBar: &processbar2.FakeScreenProcess{}, WildcardFilterMode: s.config.WildcardFilter, Predict: s.config.Predict, } - // Override device if specified - if s.config.Device != "" { - opt.EtherInfo.Device = s.config.Device + // 多网卡支持：Devices 优先，否则退回 Device 单卡或自动探测 + if len(s.config.Devices) > 0 { + etherInfos := options.GetDeviceConfigs(s.config.Devices, resolvers) + opt.EtherInfos = etherInfos + opt.EtherInfo = etherInfos[0] + } else { + et := options.GetDeviceConfig(resolvers) + if s.config.Device != "" { + et.Device = s.config.Device + } + opt.EtherInfo = et } opt.Check() + return opt +} - // Create runner +// scanCollect runs the scan and returns collected results. +func (s *Scanner) scanCollect(ctx context.Context, domainChan chan string, method options.OptionMethod) ([]Result, error) { + collector := &resultCollector{results: make([]Result, 0)} + opt := s.buildOptions(domainChan, method, collector) r, err := runner.New(opt) if err != nil { return nil, fmt.Errorf("failed to create runner: %w", err) } - - // Run enumeration r.RunEnumeration(ctx) r.Close() - return collector.results, nil } -// resultCollector collects scan results +// scanStream runs the scan, calling callback for each result in real-time. +func (s *Scanner) scanStream(ctx context.Context, domainChan chan string, method options.OptionMethod, callback func(Result)) error { + streamer := &streamCollector{callback: callback} + opt := s.buildOptions(domainChan, method, streamer) + r, err := runner.New(opt) + if err != nil { + return fmt.Errorf("failed to create runner: %w", err) + } + r.RunEnumeration(ctx) + r.Close() + return nil +} + +// --------------------------------------------------------------------------- +// outputter.Output implementations +// --------------------------------------------------------------------------- + +// resultCollector accumulates results for the blocking API. type resultCollector struct { results []Result mu sync.Mutex @@ -208,39 +296,52 @@ type resultCollector struct { func (rc *resultCollector) WriteDomainResult(r result.Result) error { rc.mu.Lock() defer rc.mu.Unlock() + rc.results = append(rc.results, parseResult(r)) + return nil +} + +func (rc *resultCollector) Close() error { return nil } + +// streamCollector forwards each result to a user-supplied callback. +// WriteDomainResult may be called from multiple goroutines; the callback +// itself is invoked under no lock — callers are responsible for thread safety. +type streamCollector struct { + callback func(Result) +} + +func (sc *streamCollector) WriteDomainResult(r result.Result) error { + sc.callback(parseResult(r)) + return nil +} + +func (sc *streamCollector) Close() error { return nil } - // Parse record type +// parseResult converts an internal result.Result to the public Result type. +func parseResult(r result.Result) Result { recordType := "A" records := make([]string, 0, len(r.Answers)) for _, answer := range r.Answers { - if strings.HasPrefix(answer, "CNAME ") { + switch { + case strings.HasPrefix(answer, "CNAME "): recordType = "CNAME" records = append(records, answer[6:]) - } else if strings.HasPrefix(answer, "NS ") { + case strings.HasPrefix(answer, "NS "): recordType = "NS" records = append(records, answer[3:]) - } else if strings.HasPrefix(answer, "PTR ") { + case strings.HasPrefix(answer, "PTR "): recordType = "PTR" records = append(records, answer[4:]) - } else { + default: records = append(records, answer) } } - if len(records) == 0 { records = r.Answers } - - rc.results = append(rc.results, Result{ + return Result{ Domain: r.Subdomain, Type: recordType, Records: records, - }) - - return nil -} - -func (rc *resultCollector) Close() error { - return nil + } }