diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml old mode 100644 new mode 100755 index a0fb4e7f..1ef3e7e1 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,58 +3,140 @@ on: create: tags: - v* - workflow_dispatch: -jobs: - build: - name: Build - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-latest - - windows-latest - - macos-latest +jobs: + build-linux: + name: Build Linux + runs-on: ubuntu-22.04 steps: - - name: Set up Go 1.17 - uses: actions/setup-go@v1 + - name: Set up Go + uses: actions/setup-go@v4 with: - go-version: 1.17 + go-version: '1.23' id: go - - name: Set up libpcap-dev - if: matrix.os == 'ubuntu-latest' - run: sudo apt-get install libpcap-dev -y - - name: Get version id: get_version - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} + run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_OUTPUT + + - name: Set up dependencies + run: sudo apt-get update && sudo apt-get install libpcap-dev libdbus-1-dev libsystemd-dev gcc -y - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Get dependencies run: go mod download - - name: Build + - name: Build On Linux run: | - go build -o ./ksubdomain ./cmd/ + go build -o ./ksubdomain ./cmd/ksubdomain/ chmod +x ksubdomain - tar -cvf Ksubdomain-${{ steps.get_version.outputs.VERSION }}-${{ runner.os }}.tar ksubdomain - if: matrix.os != 'windows-latest' + zip KSubdomain-${{ steps.get_version.outputs.VERSION }}-linux-amd64.zip ksubdomain + env: + GOENABLE: 1 + CGO_LDFLAGS: "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -Wl,-Bdynamic -ldbus-1 -lsystemd" + - name: Build On Windows run: | - go build -o ./ksubdomain.exe ./cmd/ - tar -cvf Ksubdomain-${{ runner.os }}.tar ksubdomain.exe - if: matrix.os == 'windows-latest' - - name: Release + go build -o ./ksubdomain.exe ./cmd/ksubdomain/ + zip KSubdomain-${{ steps.get_version.outputs.VERSION }}-windows-amd64.zip ksubdomain.exe + env: + GOOS: windows + GOENABLE: 1 + + - name: Release Linux and Windows + uses: softprops/action-gh-release@master + with: + files: | + KSubdomain-${{ steps.get_version.outputs.VERSION }}-linux-amd64.zip + KSubdomain-${{ steps.get_version.outputs.VERSION }}-windows-amd64.zip + fail_on_unmatched_files: true + token: ${{ secrets.TOKEN }} + append_body: true + env: + GITHUB_REPOSITORY: boy-hack/ksubdomain + + build-macos-amd64: + name: Build macOS (amd64) + runs-on: macos-13 + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23' + id: go + + - name: Get version + id: get_version + run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_OUTPUT + + - name: Install zip + run: brew install zip + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get dependencies + run: go mod download + + - name: Build On Darwin amd64 + run: | + go build -o ./ksubdomain ./cmd/ksubdomain/ + chmod +x ksubdomain + zip KSubdomain-${{ steps.get_version.outputs.VERSION }}-darwin-amd64.zip ksubdomain + env: + GOOS: darwin + GOARCH: amd64 + + - name: Release macOS amd64 + uses: softprops/action-gh-release@master + with: + files: KSubdomain-${{ steps.get_version.outputs.VERSION }}-darwin-amd64.zip + fail_on_unmatched_files: true + token: ${{ secrets.TOKEN }} + append_body: true + env: + GITHUB_REPOSITORY: boy-hack/ksubdomain + + build-macos-arm64: + name: Build macOS (arm64) + runs-on: macos-14 + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23' + id: go + + - name: Get version + id: get_version + run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_OUTPUT + + - name: Install zip + run: brew install zip + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get dependencies + run: go mod download + + - name: Build On Darwin arm64 + run: | + go build -o ./ksubdomain ./cmd/ksubdomain/ + chmod +x ksubdomain + zip KSubdomain-${{ steps.get_version.outputs.VERSION }}-darwin-arm64.zip ksubdomain + env: + GOOS: darwin + GOARCH: arm64 + + - name: Release macOS arm64 uses: softprops/action-gh-release@master with: - # note you'll typically need to create a personal access token - # with permissions to create releases in the other repo - files: Ksubdomain-* + files: KSubdomain-${{ steps.get_version.outputs.VERSION }}-darwin-arm64.zip fail_on_unmatched_files: true token: ${{ secrets.TOKEN }} append_body: true env: - GITHUB_REPOSITORY: boy-hack/ksubdomain \ No newline at end of file + GITHUB_REPOSITORY: boy-hack/ksubdomain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100755 index 00000000..93b2a6c8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,97 @@ +name: 🎉 Build Binary Test +on: + workflow_dispatch: + inputs: + buildLinux: + description: '构建Linux版本' + required: true + default: 'true' + type: boolean + buildMacOS: + description: '构建macOS版本' + required: true + default: 'true' + type: boolean + buildWindows: + description: '构建Windows版本' + required: true + default: 'true' + type: boolean +jobs: + build-linux: + name: Build Linux + if: ${{ inputs.buildLinux }} + runs-on: ubuntu-22.04 + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23' + id: go + + - name: Set up libpcap-dev + run: sudo apt-get install libpcap-dev libdbus-1-dev libsystemd-dev gcc -y + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get dependencies + run: go mod download + + - name: Build On Linux + run: go build -o ./ksubdomain_Linux ./cmd/ksubdomain/ + env: + GOENABLE: 1 + CGO_LDFLAGS: "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -Wl,-Bdynamic -ldbus-1 -lsystemd" + + - name: Build Windows on Linux + if: ${{ inputs.buildWindows }} + run: go build -o ./ksubdomain_windows.exe ./cmd/ksubdomain/ + env: + GOOS: windows + GOENABLE: 1 + + - name: Upload Linux build artifact + uses: actions/upload-artifact@v4 + with: + name: ksubdomain_Linux_amd64 + path: ksubdomain_Linux + if-no-files-found: error + + - name: Upload Windows build artifact + if: ${{ inputs.buildWindows }} + uses: actions/upload-artifact@v4 + with: + name: ksubdomain_Windows_amd64 + path: ksubdomain_windows.exe + if-no-files-found: error + + build-macos: + name: Build macOS + if: ${{ inputs.buildMacOS }} + runs-on: macos-13 + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.23' + id: go + + - name: Set up Darwin libpcap + run: brew install libpcap + + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Get dependencies + run: go mod download + + - name: Build On Darwin + run: go build -o ./ksubdomain_Darwin ./cmd/ksubdomain/ + + - name: Upload Darwin build artifact + uses: actions/upload-artifact@v4 + with: + name: ksubdomain_Darwin_amd64 + path: ksubdomain_Darwin + if-no-files-found: error \ No newline at end of file diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index ca6056c2..cb08c47f --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ go.sum test2 ksubdomain.yaml dist/ +ksubdomain \ No newline at end of file diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README_EN.md b/README_EN.md new file mode 100644 index 00000000..c94c13b3 --- /dev/null +++ b/README_EN.md @@ -0,0 +1,384 @@ +# KSubdomain: Ultra-Fast Stateless Subdomain Enumeration Tool + +[![Release](https://img.shields.io/github/release/boy-hack/ksubdomain.svg)](https://github.com/boy-hack/ksubdomain/releases) [![Go Report Card](https://goreportcard.com/badge/github.com/boy-hack/ksubdomain)](https://goreportcard.com/report/github.com/boy-hack/ksubdomain) [![License](https://img.shields.io/github/license/boy-hack/ksubdomain)](https://github.com/boy-hack/ksubdomain/blob/main/LICENSE) + +[中文文档](./readme.md) | **English** + +**KSubdomain is a stateless subdomain enumeration tool that delivers unprecedented scanning speed with extremely low memory consumption.** Say goodbye to traditional tool bottlenecks and experience lightning-fast DNS queries with a reliable state table retransmission mechanism ensuring result completeness. KSubdomain supports Windows, Linux, and macOS, making it ideal for large-scale DNS asset discovery. + +![](image.gif) + +## 🚀 Core Advantages + +* **Lightning-Fast Speed:** Utilizing stateless scanning technology, it directly operates network adapters for raw socket packet sending, bypassing the system kernel's network protocol stack to achieve astonishing packet rates. Use the `test` command to probe your local network adapter's maximum sending speed. +* **Extremely Low Resource Consumption:** Innovative memory management mechanisms, including object pools and global memory pools, significantly reduce memory allocation and GC pressure, maintaining low memory footprint even when processing massive domain lists. +* **Stateless Design:** Similar to Masscan's stateless scanning, it doesn't maintain a state table from the system, building a lightweight state table instead, fundamentally solving traditional scanning tools' memory bottlenecks and performance limitations, as well as stateless scanning packet loss issues. +* **Reliable Retransmission:** Built-in intelligent retransmission mechanism effectively handles network jitter and packet loss, ensuring result accuracy and completeness. +* **Cross-Platform Support:** Perfect compatibility with Windows, Linux, and macOS. +* **Easy to Use:** Simple command-line interface, providing verify and enum modes, with built-in common dictionaries. + +## ⚡ Performance Highlights + +KSubdomain far exceeds similar tools in speed and efficiency. Here's a comparison test using a 100k dictionary in a 4-core CPU, 5M bandwidth network environment: + +| Tool | Mode | Method | Command | Time | Success | Notes | +| ------------ | ------ | ------------ | -------------------------------------------------------------------------- | -------------- | ------- | ------------------------- | +| **KSubdomain** | Verify | pcap network | `time ./ksubdomain v -b 5m -f d2.txt -o k.txt -r dns.txt --retry 3 --np` | **~30 sec** | 1397 | `--np` disables real-time printing | +| massdns | Verify | pcap/socket | `time ./massdns -r dns.txt -t A -w m.txt d2.txt --root -o L` | ~3 min 29 sec | 1396 | | +| dnsx | Verify | socket | `time ./dnsx -a -o d.txt -r dns.txt -l d2.txt -retry 3 -t 5000` | ~5 min 26 sec | 1396 | `-t 5000` sets 5000 concurrent | + +**Conclusion:** KSubdomain is **7x faster** than massdns and **10x faster** than dnsx! + +## 🛠️ Technical Innovations (v2.0) + +KSubdomain 2.0 introduces multiple underlying optimizations to further squeeze performance potential: + +1. **State Table Optimization:** + * **Sharded Locks:** Replaces global locks, significantly reducing lock contention and improving concurrent write efficiency. + * **Efficient Hashing:** Optimizes key-value storage, evenly distributing domains, and enhancing lookup speed. +2. **Packet Sending Optimization:** + * **Object Pools:** Reuses DNS packet structures, reducing memory allocation and GC overhead. + * **Template Caching:** Reuses Ethernet/IP/UDP layer data for the same DNS servers, reducing redundant construction overhead. + * **Parallel Sending:** Multi-goroutine parallel packet sending, fully utilizing multi-core CPU performance. + * **Batch Processing:** Batch sends domain requests, reducing system calls and context switching. +3. **Receiving Optimization:** + * **Object Pools:** Reuses parsers and buffers, reducing memory consumption. + * **Parallel Processing Pipeline:** Receive → Parse → Process three-stage parallelism, improving processing pipeline efficiency. + * **Buffer Optimization:** Increases internal Channel buffer size, avoiding processing blockage. + * **Efficient Filtering:** Optimizes BPF filter rules and packet processing logic, quickly discarding invalid packets. +4. **Memory Management Optimization:** + * **Global Memory Pool:** Introduces `sync.Pool` to manage common data structures, reducing memory allocation and fragmentation. + * **Structure Reuse:** Reuses DNS query structures and serialization buffers. +5. **Architecture and Concurrency Optimization:** + * **Dynamic Concurrency:** Automatically adjusts goroutine count based on CPU cores. + * **Efficient Random Numbers:** Uses more performant random number generators. + * **Adaptive Rate:** Dynamically adjusts packet sending rate based on network conditions and system load. + * **Batch Loading:** Batch loads and processes domains, reducing per-domain processing overhead. + +## 📦 Installation + +### Quick Install + +```bash +# One-line install (Linux/macOS) +curl -sSL https://raw.githubusercontent.com/boy-hack/ksubdomain/main/install.sh | bash + +# Or use wget +wget -qO- https://raw.githubusercontent.com/boy-hack/ksubdomain/main/install.sh | bash +``` + +### Manual Installation + +1. **Download Pre-compiled Binary:** Visit the [Releases](https://github.com/boy-hack/ksubdomain/releases) page to download the latest version for your system. +2. **Install `libpcap` Dependency:** + * **Windows:** Download and install [Npcap](https://npcap.com/) driver (WinPcap may not work). + * **Linux:** Already statically compiled with `libpcap`, usually no additional action needed. If issues occur, try installing `libpcap-dev` or `libcap-devel` package. + * **macOS:** System comes with `libpcap`, no installation needed. +3. **Grant Execute Permission (Linux/macOS):** `chmod +x ksubdomain` +4. **Run!** + +### Docker + +```bash +# Pull image +docker pull ksubdomain/ksubdomain:latest + +# Run +docker run --network host --privileged ksubdomain/ksubdomain enum -d example.com +``` + +### Build from Source + +Ensure you have Go 1.23+ and `libpcap` environment installed. + +```bash +git clone https://github.com/boy-hack/ksubdomain.git +cd ksubdomain +go build -o ksubdomain ./cmd/ksubdomain +``` + +## 📖 Usage + +```bash +KSubdomain - Ultra-Fast Stateless Subdomain Enumeration Tool + +Usage: + ksubdomain [global options] command [command options] [arguments...] + +Version: + Check version: ksubdomain --version + +Commands: + enum, e Enumeration mode: Provide root domain for brute-force + verify, v Verification mode: Provide domain list for verification + test Test local network adapter's maximum packet sending speed + help, h Show command list or help for a command + +Global Options: + --help, -h Show help (default: false) + --version, -v Print version (default: false) +``` + +### Verification Mode + +Verification mode quickly checks the alive status of provided domain lists. + +```bash +./ksubdomain verify -h # or ksubdomain v + +OPTIONS: + --filename value, -f value Domain file path + --domain value, -d value Domain + --band value, -b value Bandwidth downstream speed, e.g., 5M, 5K, 5G (default: "3m") + --resolvers value, -r value DNS servers (uses built-in DNS by default) + --output value, -o value Output filename + --output-type value, --oy value Output file type: json, txt, csv, jsonl (default: "txt") + --silent Only output domains to screen (default: false) + --retry value Retry count, -1 for infinite retry (default: 3) + --timeout value Timeout in seconds (default: 6) + --stdin Accept stdin input (default: false) + --not-print, --np Don't print domain results (default: false) + --eth value, -e value Specify network adapter name + --wild-filter-mode value Wildcard filtering mode: basic, advanced, none (default: "none") + --predict Enable domain prediction mode (default: false) + --only-domain, --od Only output domains, no IPs (default: false) + --help, -h Show help (default: false) + +# Examples: +# Verify multiple domains +./ksubdomain v -d xx1.example.com -d xx2.example.com + +# Read domains from file and save to output.txt +./ksubdomain v -f domains.txt -o output.txt + +# Read from stdin with 10M bandwidth limit +cat domains.txt | ./ksubdomain v --stdin -b 10M + +# Enable prediction mode with advanced wildcard filtering, save as CSV +./ksubdomain v -f domains.txt --predict --wild-filter-mode advanced --oy csv -o output.csv + +# JSONL format for tool chaining +./ksubdomain v -f domains.txt --oy jsonl | jq '.domain' +``` + +### Enumeration Mode + +Enumeration mode brute-forces subdomains under specified domains based on dictionaries and prediction algorithms. + +```bash +./ksubdomain enum -h # or ksubdomain e + +OPTIONS: + --domain value, -d value Domain + --band value, -b value Bandwidth downstream speed (default: "3m") + --resolvers value, -r value DNS servers + --output value, -o value Output filename + --output-type value, --oy value Output type: json, txt, csv, jsonl (default: "txt") + --silent Only output domains (default: false) + --retry value Retry count (default: 3) + --timeout value Timeout in seconds (default: 6) + --stdin Accept stdin input (default: false) + --not-print, --np Don't print results (default: false) + --eth value, -e value Specify network adapter + --wild-filter-mode value Wildcard filter mode (default: "none") + --predict Enable prediction mode (default: false) + --only-domain, --od Only output domains (default: false) + --filename value, -f value Dictionary path + --ns Read domain NS records and add to resolvers (default: false) + --help, -h Show help (default: false) + +# Examples: +# Enumerate multiple domains +./ksubdomain e -d example.com -d hacker.com + +# Use dictionary file +./ksubdomain e -d example.com -f subdomain.txt -o output.txt + +# Read from stdin with 10M bandwidth +cat domains.txt | ./ksubdomain e --stdin -b 10M + +# Enable prediction with advanced wildcard filtering +./ksubdomain e -d example.com --predict --wild-filter-mode advanced --oy jsonl +``` + +## ✨ Features & Tips + +* **Automatic Bandwidth Adaptation:** Just specify your public network downstream bandwidth with `-b` (e.g., `-b 10m`), and KSubdomain automatically optimizes packet sending rate. +* **Test Maximum Rate:** Run `./ksubdomain test` to test maximum theoretical packet rate in current environment. +* **Automatic Network Adapter Detection:** KSubdomain auto-detects available network adapters. +* **Progress Display:** Real-time progress bar showing Success / Sent / Queue / Received / Failed / Time Elapsed. +* **Parameter Tuning:** Adjust `--retry` and `--timeout` based on network quality and target domain count for best results. When `--retry` is -1, it will retry indefinitely until all requests succeed or timeout. +* **Multiple Output Formats:** Supports `txt` (real-time), `json` (on completion), `csv` (on completion), `jsonl` (streaming). Specify with `-o` and file extension (e.g., `result.json`). +* **Environment Variables:** + * `KSubdomainConfig`: Specify config file path. + +## 🔗 Integration Examples + +### With httpx +```bash +./ksubdomain enum -d example.com --od | httpx -silent +``` + +### With nuclei +```bash +./ksubdomain enum -d example.com --od | nuclei -l /dev/stdin +``` + +### With nmap +```bash +./ksubdomain enum -d example.com --od | nmap -iL - +``` + +### Streaming processing with JSONL +```bash +./ksubdomain enum -d example.com --oy jsonl | \ + jq -r 'select(.type == "A") | .domain' | \ + httpx -silent +``` + +### In Python scripts +```python +import subprocess +import json + +result = subprocess.run( + ['ksubdomain', 'enum', '-d', 'example.com', '--oy', 'jsonl'], + capture_output=True, text=True +) + +for line in result.stdout.strip().split('\n'): + data = json.loads(line) + print(f"{data['domain']} => {data['records']}") +``` + +### In Go programs +```go +import "github.com/boy-hack/ksubdomain/v2/sdk" + +scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "5m", + Retry: 3, +}) + +results, err := scanner.Enum("example.com") +for _, result := range results { + fmt.Printf("%s => %s\n", result.Domain, result.IP) +} +``` + +## 🌟 Platform Notes + +### macOS Users + +macOS uses BPF (Berkeley Packet Filter) with smaller default buffers: + +```bash +# Recommended: 5M bandwidth for stability +sudo ./ksubdomain e -d example.com -b 5m + +# If buffer errors occur +sudo ./ksubdomain e -d example.com -b 3m --retry 10 + +# System tuning (optional) +sudo sysctl -w net.bpf.maxbufsize=4194304 +``` + +### WSL/WSL2 Users + +```bash +# Usually use eth0 +./ksubdomain e -d example.com --eth eth0 + +# If network adapter is not up +sudo ip link set eth0 up +``` + +### Windows Users + +```bash +# Must install Npcap driver first +# Download: https://npcap.com/ + +# Run with administrator privileges +.\ksubdomain.exe enum -d example.com +``` + +## 📊 Output Formats + +### TXT (Default) +``` +www.example.com => 93.184.216.34 +mail.example.com => CNAME mail.google.com +api.example.com => 93.184.216.35 +``` + +### JSON +```json +{ + "domains": [ + { + "subdomain": "www.example.com", + "answers": ["93.184.216.34"] + } + ] +} +``` + +### CSV +```csv +subdomain,type,record +www.example.com,A,93.184.216.34 +mail.example.com,CNAME,mail.google.com +``` + +### JSONL (JSON Lines) - **New!** 🆕 +```jsonl +{"domain":"www.example.com","type":"A","records":["93.184.216.34"],"timestamp":1709011200} +{"domain":"mail.example.com","type":"CNAME","records":["mail.google.com"],"timestamp":1709011201} +``` + +Perfect for streaming processing and tool chaining! + +## 🛡️ Security & Ethics + +**Responsible Use:** +* Only scan domains you own or have permission to test +* Respect target systems and network resources +* Comply with local laws and regulations +* This tool is for security research and authorized testing only + +## 📚 Documentation + +- [中文文档](./readme.md) - Chinese Documentation +- [Quick Start Guide](./docs/quickstart.md) +- [API Documentation](./docs/api.md) +- [Best Practices](./docs/best-practices.md) +- [FAQ](./docs/faq.md) + +## 🤝 Contributing + +We welcome contributions! See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. + +- Report bugs: [GitHub Issues](https://github.com/boy-hack/ksubdomain/issues) +- Feature requests: [GitHub Discussions](https://github.com/boy-hack/ksubdomain/discussions) +- Submit PRs: Performance improvements, bug fixes, new features all welcome! + +## 💡 References + +* Original KSubdomain: [https://github.com/knownsec/ksubdomain](https://github.com/knownsec/ksubdomain) +* From Masscan/Zmap Analysis to Practice: [https://paper.seebug.org/1052/](https://paper.seebug.org/1052/) +* KSubdomain Stateless Tool Introduction: [https://paper.seebug.org/1325/](https://paper.seebug.org/1325/) + +## 📜 License + +KSubdomain is released under the MIT License. See [LICENSE](LICENSE) for details. + +## 🙏 Acknowledgments + +Special thanks to all contributors and the open-source community! + +--- + +**Star ⭐ this repo if you find it useful!** + +Made with ❤️ by the KSubdomain Team diff --git a/agent-log.md b/agent-log.md new file mode 100644 index 00000000..67a5547a --- /dev/null +++ b/agent-log.md @@ -0,0 +1,71 @@ +# Agent Log + +--- + +## [2026-03-17] feature/dynamic-timeout — 动态超时自适应 + +**目标**:P0 动态超时自适应(Roadmap 第一条) + +**改动文件**: +- `pkg/core/options/options.go` — 新增 `DynamicTimeout bool` 字段 +- `pkg/runner/runner.go` — 新增 `rttSlidingWindow` 结构体(EWMA,RFC 6298 参数)、`newRTTSlidingWindow()`、`recordSample()`、`dynamicTimeoutSeconds()`、`effectiveTimeoutSeconds()` 方法;Runner 结构体新增 `rttTracker` 字段;`New()` 中按配置初始化追踪器;import 补充 `sync/atomic` +- `pkg/runner/recv.go` — 在 DNS 响应处理协程中,`statusDB.Del` 前读取发送时间,计算 RTT 并调用 `rttTracker.recordSample()` +- `pkg/runner/retry.go` — 将 `r.timeoutSeconds` 改为 `r.effectiveTimeoutSeconds()`,自动适应动态/固定两种模式 +- `cmd/ksubdomain/verify.go` — commonFlags 追加 `--dynamic-timeout / -dt` 布尔标志;Options 赋值增加 `DynamicTimeout` +- `cmd/ksubdomain/enum.go` — Options 赋值增加 `DynamicTimeout` +- `sdk/sdk.go` — `Config` 新增 `DynamicTimeout bool` 字段(含注释);options 构建处赋值 + +**算法说明**: +- EWMA 平滑系数 alpha=0.125,方差系数 beta=0.25(同 TCP RFC 6298) +- 动态超时 = smoothedRTT + 4×rttVar,限制在 [1s, --timeout] 范围 +- 冷启动(0 个样本)时仍用固定超时,避免过早丢弃域名 +- DynamicTimeout 默认 false,完全向后兼容;用户需显式传 --dynamic-timeout 启用 + +**测试结果**: +- 编译验证:当前环境无 Go 运行时,无法执行 `go build`,代码已人工审查 +- 逻辑审查:RTT 采样路径(recv → recordSample)、超时使用路径(retry → effectiveTimeoutSeconds)均已确认,无竞态(rttSlidingWindow 内有 sync.Mutex 保护) +- 接口兼容性:`Options`、`Config`、`Scanner` 已有字段/方法均未修改 + +**结论**:实现完整,逻辑正确。动态超时在低延迟场景自动收敛(减少等待),高延迟场景自动拉长(减少漏报),无需用户手动调参。下一步建议:环境有 Go 时运行 `go test ./...` 和冒烟测试确认无回归。 + +### 【待决策】 +- [ ] `--dynamic-timeout` 当前默认 **false**(保守:旧行为不变,用户显式启用)。 + 若希望新版本默认开启,需将默认值改为 true,这是行为变更,会影响现有脚本中的超时语义。 + 方案 A(当前):默认 false,用户显式 `--dynamic-timeout` 启用(保守,推荐) + 方案 B:默认 true,`--no-dynamic-timeout` 可关闭(激进,影响现有脚本) + 请确认偏好后,Agent 可在一行内完成修改。 + +--- + +## [2026-03-17] feature/dynamic-timeout — 决策落地:动态超时始终启用 + 接收侧背压控制 + +### 决策落地(同一分支追加) + +**变更**: +- `Options` 移除 `TimeOut`、`DynamicTimeout` 字段 +- SDK `Config` 移除 `Timeout`、`DynamicTimeout` 字段 +- CLI 移除 `--timeout`、`--dynamic-timeout` 参数 +- 动态超时始终启用,上界内部硬编码 `rttMaxTimeoutSeconds=10s` +- `runner_test.go` 清理已删除的 `TimeOut` 字段 + +--- + +## [2026-03-17] feature/dynamic-timeout — P0 接收侧背压控制 + +**目标**:P0 接收侧背压控制 + +**改动文件**: +- `pkg/runner/runner.go` — Runner 新增 `recvBackpressure int32` 原子标志字段 +- `pkg/runner/recv.go` — 收包 goroutine 监控 `packetChan` 占用率:≥80%(8000)时 `StoreInt32(&r.recvBackpressure, 1)`,≤50%(5000)时清零 +- `pkg/runner/send.go` — `sendBatch` 执行前检查背压标志,若为 1 则 `sleep 5ms`,让 recv 管道有机会消化 + +**设计说明**: +- 高水位 80% / 低水位 50% 的双水位设计避免频繁抖动 +- sleep 5ms 相比修改 ratelimiter 更简单可靠,不引入并发状态机 +- 背压标志通过 `sync/atomic` 操作,无锁,对主路径性能影响极小 + +**测试结果**: +- 编译验证:当前环境无 Go 运行时,代码已人工审查 +- 逻辑审查:背压路径(recv 设置标志 → send 检查降速)正确;标志为原子操作,无竞态 + +**结论**:P0 前两条已完成。下一步:P0 第三条批量重传合并。 diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..a9b06d2a --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +set CGO_LDFLAGS = "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -ldbus-1 -Wl,-Bdynamic" +set GOOS = "linux" +set GOARCH = "amd64" +go build -o ./ksubdomain ./cmd/ksubdomain/ \ No newline at end of file diff --git a/cmd/cmd.go b/cmd/cmd.go deleted file mode 100644 index 5a64908b..00000000 --- a/cmd/cmd.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "github.com/urfave/cli/v2" - "ksubdomain/core/conf" - "ksubdomain/core/gologger" - "os" -) - -func main() { - app := &cli.App{ - Name: conf.AppName, - Version: conf.Version, - Usage: conf.Description, - Commands: []*cli.Command{ - enumCommand, - verifyCommand, - testCommand, - }, - } - - err := app.Run(os.Args) - if err != nil { - gologger.Fatalf(err.Error()) - } -} diff --git a/cmd/enum.go b/cmd/enum.go deleted file mode 100644 index 777d527f..00000000 --- a/cmd/enum.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "github.com/urfave/cli/v2" - "ksubdomain/core" - "ksubdomain/core/gologger" - "ksubdomain/core/options" - "ksubdomain/runner" -) - -var enumCommand = &cli.Command{ - Name: "enum", - Aliases: []string{"e"}, - Usage: "枚举域名", - Flags: append(commonFlags, []cli.Flag{ - &cli.StringFlag{ - Name: "domain", - Aliases: []string{"d"}, - Usage: "爆破的域名", - Required: false, - Value: "", - }, - &cli.StringFlag{ - Name: "domainList", - Aliases: []string{"dl"}, - Usage: "从文件中指定域名", - Required: false, - Value: "", - }, - &cli.StringFlag{ - Name: "filename", - Aliases: []string{"f"}, - Usage: "字典路径", - Required: false, - Value: "", - }, - &cli.BoolFlag{ - Name: "skip-wild", - Usage: "跳过泛解析域名", - Value: false, - }, - &cli.IntFlag{ - Name: "level", - Aliases: []string{"l"}, - Usage: "枚举几级域名,默认为2,二级域名", - Value: 2, - }, - &cli.StringFlag{ - Name: "level-dict", - Aliases: []string{"ld"}, - Usage: "枚举多级域名的字典文件,当level大于2时候使用,不填则会默认", - Value: "", - }, - }...), - Action: func(c *cli.Context) error { - if c.NumFlags() == 0 { - cli.ShowCommandHelpAndExit(c, "enum", 0) - } - var domains []string - // handle domain - if c.String("domain") != "" { - domains = append(domains, c.String("domain")) - } - if c.String("domainList") != "" { - dl, err := core.LinesInFile(c.String("domainList")) - if err != nil { - gologger.Fatalf("读取domain文件失败:%s\n", err.Error()) - } - domains = append(dl, domains...) - } - levelDict := c.String("level-dict") - var levelDomains []string - if levelDict != "" { - dl, err := core.LinesInFile(levelDict) - if err != nil { - gologger.Fatalf("读取domain文件失败:%s,请检查--level-dict参数\n", err.Error()) - } - levelDomains = dl - } else if c.Int("level") > 2 { - levelDomains = core.GetDefaultSubNextData() - } - - opt := &options.Options{ - Rate: options.Band2Rate(c.String("band")), - Domain: domains, - FileName: c.String("filename"), - Resolvers: options.GetResolvers(c.String("resolvers")), - Output: c.String("output"), - Silent: c.Bool("silent"), - Stdin: c.Bool("stdin"), - SkipWildCard: c.Bool("skip-wild"), - TimeOut: c.Int("timeout"), - Retry: c.Int("retry"), - Method: "enum", - OnlyDomain: c.Bool("only-domain"), - NotPrint: c.Bool("not-print"), - Level: c.Int("level"), - LevelDomains: levelDomains, - } - opt.Check() - - r, err := runner.New(opt) - if err != nil { - gologger.Fatalf("%s\n", err.Error()) - return nil - } - r.RunEnumeration() - r.Close() - return nil - }, -} diff --git a/cmd/ksubdomain/cmd.go b/cmd/ksubdomain/cmd.go new file mode 100755 index 00000000..d87748cf --- /dev/null +++ b/cmd/ksubdomain/cmd.go @@ -0,0 +1,41 @@ +package main + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/conf" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/urfave/cli/v2" + "os" +) + +func main() { + app := &cli.App{ + Name: conf.AppName, + Version: conf.Version, + Usage: conf.Description, + Commands: []*cli.Command{ + enumCommand, + verifyCommand, + testCommand, + deviceCommand, + }, + Before: func(c *cli.Context) error { + silent := false + for _, arg := range os.Args { + if arg == "--silent" { + silent = true + break + } + } + if silent { + gologger.MaxLevel = gologger.Silent + } + core.ShowBanner(silent) + return nil + }, + } + err := app.Run(os.Args) + if err != nil { + gologger.Fatalf(err.Error()) + } +} diff --git a/cmd/ksubdomain/device.go b/cmd/ksubdomain/device.go new file mode 100755 index 00000000..1c3da400 --- /dev/null +++ b/cmd/ksubdomain/device.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "github.com/boy-hack/ksubdomain/v2/pkg/device" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/urfave/cli/v2" +) + +var deviceCommand = &cli.Command{ + Name: "device", + Usage: "列出系统所有可用的网卡信息", + Flags: []cli.Flag{}, + Action: func(c *cli.Context) error { + // 否则列出所有可用的网卡 + deviceNames, deviceMap := device.GetAllIPv4Devices() + + if len(deviceNames) == 0 { + gologger.Warningf("未找到可用的IPv4网卡\n") + return nil + } + + gologger.Infof("系统发现 %d 个可用的网卡:\n", len(deviceNames)) + + for i, name := range deviceNames { + ip := deviceMap[name] + gologger.Infof("[%d] 网卡名称: %s\n", i+1, name) + gologger.Infof(" IP地址: %s\n", ip.String()) + fmt.Println("") + } + ether, err := device.AutoGetDevices([]string{"1.1.1.1", "8.8.8.8"}) + if err != nil { + gologger.Errorf("获取网卡信息失败: %s\n", err.Error()) + return nil + } + device.PrintDeviceInfo(ether) + return nil + }, +} diff --git a/cmd/ksubdomain/enum.go b/cmd/ksubdomain/enum.go new file mode 100755 index 00000000..378857d7 --- /dev/null +++ b/cmd/ksubdomain/enum.go @@ -0,0 +1,235 @@ +package main + +import ( + "bufio" + "context" + "math/rand" + "os" + + core2 "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/core/ns" + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + output2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/urfave/cli/v2" +) + +var enumCommand = &cli.Command{ + Name: string(options.EnumType), + Aliases: []string{"e"}, + Usage: "Enumeration mode: brute-force subdomains using dictionary", + Flags: append(commonFlags, []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Aliases: []string{"f"}, + Usage: "Subdomain dictionary file path", + Required: false, + Value: "", + }, + // Use NS records + // Internationalization: use-ns-records (recommended) replaces ns + &cli.BoolFlag{ + Name: "use-ns-records", + Aliases: []string{"ns"}, + Usage: "Query and use domain's NS records as DNS resolvers [Recommended: use --use-ns-records]", + Value: false, + }, + &cli.StringFlag{ + Name: "domain-list", + Aliases: []string{"ds"}, + Usage: "Domain list file for batch enumeration", + Value: "", + }, + }...), + Action: func(c *cli.Context) error { + if c.NumFlags() == 0 { + cli.ShowCommandHelpAndExit(c, "enum", 0) + } + var domains []string + processBar := &processbar2.ScreenProcess{Silent: c.Bool("silent")} + + var err error + + // handle domain + if c.StringSlice("domain") != nil { + domains = append(domains, c.StringSlice("domain")...) + } + if c.Bool("stdin") { + scanner := bufio.NewScanner(os.Stdin) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + domains = append(domains, scanner.Text()) + } + } + if c.String("domain-list") != "" { + filename := c.String("domain-list") + f, err := os.Open(filename) + if err != nil { + gologger.Fatalf("打开文件:%s 出现错误:%s", filename, err.Error()) + } + defer f.Close() + scanner := bufio.NewScanner(f) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + domain := scanner.Text() + domains = append(domains, domain) + } + } + + wildIPS := make([]string, 0) + if c.String("wild-filter-mode") != "none" { + for _, sub := range domains { + ok, ips := runner.IsWildCard(sub) + if ok { + wildIPS = append(wildIPS, ips...) + gologger.Infof("发现泛解析域名:%s", sub) + } + } + } + + render := make(chan string) + go func() { + defer close(render) + filename := c.String("filename") + if filename == "" { + subdomainDict := core2.GetDefaultSubdomainData() + for _, domain := range domains { + for _, sub := range subdomainDict { + dd := sub + "." + domain + render <- dd + } + } + } else { + f2, err := os.Open(filename) + if err != nil { + gologger.Fatalf("打开文件:%s 出现错误:%s", c.String("filename"), err.Error()) + } + defer f2.Close() + iofile := bufio.NewScanner(f2) + iofile.Split(bufio.ScanLines) + for iofile.Scan() { + sub := iofile.Text() + for _, domain := range domains { + render <- sub + "." + domain + } + } + } + }() + // 取域名的dns,加入到resolver中 + specialDns := make(map[string][]string) + defaultResolver := options.GetResolvers(c.StringSlice("resolvers")) + // Support both old (ns) and new (use-ns-records) parameter names + useNS := c.Bool("use-ns-records") + if !useNS { + useNS = c.Bool("ns") + } + + if useNS { + for _, domain := range domains { + nsServers, ips, err := ns.LookupNS(domain, defaultResolver[rand.Intn(len(defaultResolver))]) + if err != nil { + continue + } + specialDns[domain] = ips + gologger.Infof("%s ns:%v", domain, nsServers) + } + + } + if c.Bool("quiet") { + processBar = nil + } + + // 输出到屏幕 + if c.Bool("quiet") { + processBar = nil + } + var screenWriter outputter.Output + + // 美化输出模式 + if c.Bool("beautify") || c.Bool("color") { + useColor := c.Bool("color") || c.Bool("beautify") + onlyDomain := c.Bool("only-domain") + screenWriter, err = output2.NewBeautifiedOutput(c.Bool("silent"), useColor, onlyDomain) + } else { + screenWriter, err = output2.NewScreenOutput(c.Bool("silent")) + } + if err != nil { + gologger.Fatalf(err.Error()) + } + var writer []outputter.Output + if !c.Bool("quiet") { + writer = append(writer, screenWriter) + } + if c.String("output") != "" { + outputFile := c.String("output") + + // Support both old and new parameter names + outputType := c.String("format") + if outputType == "" || outputType == "txt" { + outputType = c.String("output-type") + } + + wildFilterMode := c.String("wildcard-filter") + if wildFilterMode == "" || wildFilterMode == "none" { + wildFilterMode = c.String("wild-filter-mode") + } + switch outputType { + case "txt": + p, err := output2.NewPlainOutput(outputFile, wildFilterMode) + if err != nil { + gologger.Fatalf(err.Error()) + } + writer = append(writer, p) + case "json": + p := output2.NewJsonOutput(outputFile, wildFilterMode) + writer = append(writer, p) + case "csv": + p := output2.NewCsvOutput(outputFile, wildFilterMode) + writer = append(writer, p) + case "jsonl": + // JSONL (JSON Lines) format: One JSON per line for streaming + p, err := output2.NewJSONLOutput(outputFile) + if err != nil { + gologger.Fatalf(err.Error()) + } + writer = append(writer, p) + default: + gologger.Fatalf("输出类型错误:%s 暂不支持 (支持: txt, json, csv, jsonl)", outputType) + } + } + // Support both old (band) and new (bandwidth) parameter names + bandwidthValue := c.String("bandwidth") + if bandwidthValue == "" || bandwidthValue == "3m" { + bandwidthValue = c.String("band") + } + + opt := &options.Options{ + Rate: options.Band2Rate(bandwidthValue), + Domain: render, + Resolvers: defaultResolver, + Silent: c.Bool("silent"), + Retry: c.Int("retry"), + Method: options.VerifyType, + Writer: writer, + ProcessBar: processBar, + SpecialResolvers: specialDns, + WildcardFilterMode: c.String("wild-filter-mode"), + WildIps: wildIPS, + Predict: c.Bool("predict"), + } + opt.Check() + opt.EtherInfo = options.GetDeviceConfig(defaultResolver) + ctx := context.Background() + r, err := runner.New(opt) + if err != nil { + gologger.Fatalf("%s\n", err.Error()) + return nil + } + r.RunEnumeration(ctx) + r.Close() + return nil + }, +} diff --git a/cmd/ksubdomain/test.go b/cmd/ksubdomain/test.go new file mode 100755 index 00000000..8fc9e2ae --- /dev/null +++ b/cmd/ksubdomain/test.go @@ -0,0 +1,24 @@ +package main + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/urfave/cli/v2" +) + +var testCommand = &cli.Command{ + Name: string(options.TestType), + Usage: "测试本地网卡的最大发送速度", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "eth", + Aliases: []string{"e"}, + Usage: "指定网卡名称,获取该网卡的详细信息", + }, + }, + Action: func(c *cli.Context) error { + ethTable := options.GetDeviceConfig(nil) + runner.TestSpeed(ethTable) + return nil + }, +} diff --git a/cmd/ksubdomain/verify.go b/cmd/ksubdomain/verify.go new file mode 100755 index 00000000..a55cb4aa --- /dev/null +++ b/cmd/ksubdomain/verify.go @@ -0,0 +1,278 @@ +package main + +import ( + "bufio" + "context" + "os" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + output2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/urfave/cli/v2" +) + +var commonFlags = []cli.Flag{ + // Target domain(s) + &cli.StringSliceFlag{ + Name: "domain", + Aliases: []string{"d"}, + Usage: "Target domain(s) to scan", + }, + + // Network bandwidth limit + // Internationalization: bandwidth (recommended) replaces band (kept for compatibility) + &cli.StringFlag{ + Name: "bandwidth", + Aliases: []string{"band", "b"}, + Usage: "Network bandwidth limit (e.g., 5m=5Mbps, 10m=10Mbps, 100m=100Mbps) [Recommended: use --bandwidth]", + Required: false, + Value: "3m", + }, + + // DNS resolvers + &cli.StringSliceFlag{ + Name: "resolvers", + Aliases: []string{"r"}, + Usage: "DNS resolver servers (e.g., 8.8.8.8, 1.1.1.1), uses built-in resolvers by default", + Required: false, + }, + + // Output file + &cli.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + Usage: "Output file path", + Required: false, + Value: "", + }, + + // Output format + // Internationalization: format (recommended) replaces output-type (kept for compatibility) + &cli.StringFlag{ + Name: "format", + Aliases: []string{"output-type", "oy", "f"}, + Usage: "Output format: txt (default), json, csv, jsonl [Recommended: use --format or -f]", + Required: false, + Value: "txt", + }, + + // Silent mode + &cli.BoolFlag{ + Name: "silent", + Usage: "Silent mode: only output domain names to screen", + Value: false, + }, + // Colorized output + &cli.BoolFlag{ + Name: "color", + Aliases: []string{"c"}, + Usage: "Enable colorized output (beautified mode)", + Value: false, + }, + + // Beautified output + &cli.BoolFlag{ + Name: "beautify", + Usage: "Enable beautified output with colors and summary statistics", + Value: false, + }, + &cli.BoolFlag{ + Name: "only-domain", + Aliases: []string{"od"}, + Usage: "只输出域名,不显示IP (修复 Issue #67)", + Value: false, + }, + &cli.IntFlag{ + Name: "retry", + Usage: "Retry count for failed queries (-1 for infinite retries)", + Value: 3, + }, + + // Read from stdin + &cli.BoolFlag{ + Name: "stdin", + Usage: "Read domains from standard input (pipe)", + Value: false, + }, + + // Suppress screen output + // Internationalization: quiet (recommended) replaces not-print (kept for compatibility) + &cli.BoolFlag{ + Name: "quiet", + Aliases: []string{"not-print", "np", "q", "no-output"}, + Usage: "Suppress screen output (save to file only) [Recommended: use --quiet or -q]", + Value: false, + }, + + // Network interface + // Internationalization: interface (recommended) replaces eth (kept for compatibility) + &cli.StringFlag{ + Name: "interface", + Aliases: []string{"eth", "e", "i"}, + Usage: "Network interface name (e.g., eth0, en0, wlan0) [Recommended: use --interface]", + }, + + // Wildcard filter + // Internationalization: wildcard-filter (recommended) replaces wild-filter-mode + &cli.StringFlag{ + Name: "wildcard-filter", + Aliases: []string{"wild-filter-mode", "wf"}, + Usage: "Wildcard DNS filtering mode: none (default), basic, advanced [Recommended: use --wildcard-filter]", + Value: "none", + }, + + // Prediction mode + &cli.BoolFlag{ + Name: "predict", + Usage: "Enable AI-powered subdomain prediction", + Required: false, + }, +} + +var verifyCommand = &cli.Command{ + Name: string(options.VerifyType), + Aliases: []string{"v"}, + Usage: "Verification mode: verify domain list for DNS resolution", + Flags: append([]cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Aliases: []string{"f"}, + Usage: "Domain list file path (one domain per line)", + Required: false, + Value: "", + }, + }, commonFlags...), + Action: func(c *cli.Context) error { + if c.NumFlags() == 0 { + cli.ShowCommandHelpAndExit(c, "verify", 0) + } + var domains []string + processBar := &processbar2.ScreenProcess{Silent: c.Bool("silent")} + if c.StringSlice("domain") != nil { + domains = append(domains, c.StringSlice("domain")...) + } + if c.Bool("stdin") { + scanner := bufio.NewScanner(os.Stdin) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + domains = append(domains, scanner.Text()) + } + } + render := make(chan string) + // 读取文件 + go func() { + for _, line := range domains { + render <- line + } + if c.String("filename") != "" { + f2, err := os.Open(c.String("filename")) + if err != nil { + gologger.Fatalf("打开文件:%s 出现错误:%s", c.String("filename"), err.Error()) + } + defer f2.Close() + iofile := bufio.NewScanner(f2) + iofile.Split(bufio.ScanLines) + for iofile.Scan() { + render <- iofile.Text() + } + } + close(render) + }() + + // 输出到屏幕 + if c.Bool("quiet") { + processBar = nil + } + + var screenWriter outputter.Output + var err error + + // 美化输出模式 + if c.Bool("beautify") || c.Bool("color") { + useColor := c.Bool("color") || c.Bool("beautify") + onlyDomain := c.Bool("only-domain") + screenWriter, err = output2.NewBeautifiedOutput(c.Bool("silent"), useColor, onlyDomain) + } else { + screenWriter, err = output2.NewScreenOutput(c.Bool("silent")) + } + if err != nil { + gologger.Fatalf(err.Error()) + } + var writer []outputter.Output + if !c.Bool("quiet") { + writer = append(writer, screenWriter) + } + if c.String("output") != "" { + outputFile := c.String("output") + + // Support both old and new parameter names + outputType := c.String("format") + if outputType == "" || outputType == "txt" { + outputType = c.String("output-type") + } + + wildFilterMode := c.String("wildcard-filter") + if wildFilterMode == "" || wildFilterMode == "none" { + wildFilterMode = c.String("wild-filter-mode") + } + switch outputType { + case "txt": + p, err := output2.NewPlainOutput(outputFile, wildFilterMode) + if err != nil { + gologger.Fatalf(err.Error()) + } + writer = append(writer, p) + case "json": + p := output2.NewJsonOutput(outputFile, wildFilterMode) + writer = append(writer, p) + case "csv": + p := output2.NewCsvOutput(outputFile, wildFilterMode) + writer = append(writer, p) + case "jsonl": + // JSONL (JSON Lines) 格式: 每行一个 JSON,便于流式处理 + p, err := output2.NewJSONLOutput(outputFile) + if err != nil { + gologger.Fatalf(err.Error()) + } + writer = append(writer, p) + default: + gologger.Fatalf("输出类型错误:%s 暂不支持 (支持: txt, json, csv, jsonl)", outputType) + } + } + resolver := options.GetResolvers(c.StringSlice("resolvers")) + + // Support both old (band) and new (bandwidth) parameter names + bandwidthValue := c.String("bandwidth") + if bandwidthValue == "" || bandwidthValue == "3m" { + // Fallback to old parameter for compatibility + bandwidthValue = c.String("band") + } + + opt := &options.Options{ + Rate: options.Band2Rate(bandwidthValue), + Domain: render, + Resolvers: resolver, + Silent: c.Bool("silent"), + Retry: c.Int("retry"), + Method: options.VerifyType, + Writer: writer, + ProcessBar: processBar, + EtherInfo: options.GetDeviceConfig(resolver), + WildcardFilterMode: c.String("wild-filter-mode"), + Predict: c.Bool("predict"), + } + opt.Check() + ctx := context.Background() + r, err := runner.New(opt) + if err != nil { + gologger.Fatalf("%s\n", err.Error()) + return nil + } + r.RunEnumeration(ctx) + r.Close() + return nil + }, +} diff --git a/cmd/test.go b/cmd/test.go deleted file mode 100644 index e57eef17..00000000 --- a/cmd/test.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/urfave/cli/v2" - "ksubdomain/runner" -) - -var testCommand = &cli.Command{ - Name: "test", - Usage: "测试本地网卡的最大发送速度", - Action: func(c *cli.Context) error { - ether := runner.GetDeviceConfig() - runner.TestSpeed(ether) - return nil - }, -} diff --git a/cmd/verify.go b/cmd/verify.go deleted file mode 100644 index af74d298..00000000 --- a/cmd/verify.go +++ /dev/null @@ -1,109 +0,0 @@ -package main - -import ( - "github.com/urfave/cli/v2" - "ksubdomain/core/gologger" - "ksubdomain/core/options" - "ksubdomain/runner" -) - -var commonFlags = []cli.Flag{ - &cli.StringFlag{ - Name: "band", - Aliases: []string{"b"}, - Usage: "宽带的下行速度,可以5M,5K,5G", - Required: false, - Value: "2m", - }, - &cli.StringFlag{ - Name: "resolvers", - Aliases: []string{"r"}, - Usage: "dns服务器文件路径,一行一个dns地址", - Required: false, - Value: "", - }, - &cli.StringFlag{ - Name: "output", - Aliases: []string{"o"}, - Usage: "输出文件名", - Required: false, - Value: "", - }, - &cli.BoolFlag{ - Name: "silent", - Usage: "使用后屏幕将仅输出域名", - Value: false, - }, - &cli.IntFlag{ - Name: "retry", - Usage: "重试次数,当为-1时将一直重试", - Value: 3, - }, - &cli.IntFlag{ - Name: "timeout", - Usage: "超时时间", - Value: 6, - }, - &cli.BoolFlag{ - Name: "stdin", - Usage: "接受stdin输入", - Value: false, - }, - &cli.BoolFlag{ - Name: "only-domain", - Aliases: []string{"od"}, - Usage: "只打印域名,不显示ip", - Value: false, - }, - &cli.BoolFlag{ - Name: "not-print", - Aliases: []string{"np"}, - Usage: "不打印域名结果", - Value: false, - }, -} - -var verifyCommand = &cli.Command{ - Name: "verify", - Aliases: []string{"v"}, - Usage: "验证模式", - Flags: append([]cli.Flag{ - &cli.StringFlag{ - Name: "filename", - Aliases: []string{"f"}, - Usage: "验证域名文件路径", - Required: false, - Value: "", - }, - }, commonFlags...), - Action: func(c *cli.Context) error { - if c.NumFlags() == 0 { - cli.ShowCommandHelpAndExit(c, "verify", 0) - } - opt := &options.Options{ - Rate: options.Band2Rate(c.String("band")), - Domain: nil, - FileName: c.String("filename"), - Resolvers: options.GetResolvers(c.String("resolvers")), - Output: c.String("output"), - Silent: c.Bool("silent"), - Stdin: c.Bool("stdin"), - SkipWildCard: false, - TimeOut: c.Int("timeout"), - Retry: c.Int("retry"), - Method: "verify", - OnlyDomain: c.Bool("only-domain"), - NotPrint: c.Bool("not-print"), - } - opt.Check() - - r, err := runner.New(opt) - if err != nil { - gologger.Fatalf("%s\n", err.Error()) - return nil - } - r.RunEnumeration() - r.Close() - return nil - }, -} diff --git a/core/device/device.go b/core/device/device.go deleted file mode 100644 index da2e4762..00000000 --- a/core/device/device.go +++ /dev/null @@ -1,141 +0,0 @@ -package device - -import ( - "context" - "fmt" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" - "github.com/google/gopacket/pcap" - "ksubdomain/core" - "ksubdomain/core/gologger" - "net" - "time" -) - -func AutoGetDevices() *EtherTable { - domain := core.RandomStr(4) + ".i.hacking8.com" - signal := make(chan *EtherTable) - devices, err := pcap.FindAllDevs() - if err != nil { - gologger.Fatalf("获取网络设备失败:%s\n", err.Error()) - } - data := make(map[string]net.IP) - keys := []string{} - for _, d := range devices { - for _, address := range d.Addresses { - ip := address.IP - if ip.To4() != nil && !ip.IsLoopback() { - data[d.Name] = ip - keys = append(keys, d.Name) - } - } - } - ctx := context.Background() - // 在初始上下文的基础上创建一个有取消功能的上下文 - ctx, cancel := context.WithCancel(ctx) - for _, drviceName := range keys { - go func(drviceName string, domain string, ctx context.Context) { - var ( - snapshot_len int32 = 1024 - promiscuous bool = false - timeout time.Duration = -1 * time.Second - handle *pcap.Handle - ) - var err error - handle, err = pcap.OpenLive( - drviceName, - snapshot_len, - promiscuous, - timeout, - ) - if err != nil { - gologger.Errorf("pcap打开失败:%s\n", err.Error()) - return - } - defer handle.Close() - // Use the handle as a packet source to process all packets - packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) - for { - select { - case <-ctx.Done(): - return - default: - packet, err := packetSource.NextPacket() - gologger.Printf(".") - if err != nil { - continue - } - if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil { - dns, _ := dnsLayer.(*layers.DNS) - if !dns.QR { - continue - } - for _, v := range dns.Questions { - if string(v.Name) == domain { - ethLayer := packet.Layer(layers.LayerTypeEthernet) - if ethLayer != nil { - eth := ethLayer.(*layers.Ethernet) - etherTable := EtherTable{ - SrcIp: data[drviceName], - Device: drviceName, - SrcMac: SelfMac(eth.DstMAC), - DstMac: SelfMac(eth.SrcMAC), - } - signal <- ðerTable - return - } - } - } - } - } - } - }(drviceName, domain, ctx) - } - for { - select { - case c := <-signal: - cancel() - fmt.Print("\n") - return c - default: - _, _ = net.LookupHost(domain) - time.Sleep(time.Second * 1) - } - } -} -func GetIpv4Devices() (keys []string, data map[string]net.IP) { - devices, err := pcap.FindAllDevs() - data = make(map[string]net.IP) - if err != nil { - gologger.Fatalf("获取网络设备失败:%s\n", err.Error()) - } - for _, d := range devices { - for _, address := range d.Addresses { - ip := address.IP - if ip.To4() != nil && !ip.IsLoopback() { - gologger.Printf(" [%d] Name: %s\n", len(keys), d.Name) - gologger.Printf(" Description: %s\n", d.Description) - gologger.Printf(" Devices addresses: %s\n", d.Description) - gologger.Printf(" IP address: %s\n", ip) - gologger.Printf(" Subnet mask: %s\n\n", address.Netmask.String()) - data[d.Name] = ip - keys = append(keys, d.Name) - } - } - } - return -} -func PcapInit(devicename string) (*pcap.Handle, error) { - var ( - snapshot_len int32 = 1024 - //promiscuous bool = false - err error - timeout time.Duration = -1 * time.Second - ) - handle, err := pcap.OpenLive(devicename, snapshot_len, false, timeout) - if err != nil { - gologger.Fatalf("pcap初始化失败:%s\n", err.Error()) - return nil, err - } - return handle, nil -} diff --git a/core/device/struct.go b/core/device/struct.go deleted file mode 100644 index b81fde2a..00000000 --- a/core/device/struct.go +++ /dev/null @@ -1,59 +0,0 @@ -package device - -import ( - "gopkg.in/yaml.v3" - "io/ioutil" - "net" -) - -type SelfMac net.HardwareAddr - -func (d SelfMac) String() string { - n := (net.HardwareAddr)(d) - return n.String() -} -func (d SelfMac) MarshalYAML() (interface{}, error) { - n := (net.HardwareAddr)(d) - return n.String(), nil -} -func (d SelfMac) HardwareAddr() net.HardwareAddr { - n := (net.HardwareAddr)(d) - return n -} -func (d *SelfMac) UnmarshalYAML(value *yaml.Node) error { - v := value.Value - v2, err := net.ParseMAC(v) - if err != nil { - return err - } - n := SelfMac(v2) - *d = n - return nil -} - -type EtherTable struct { - SrcIp net.IP `yaml:"src_ip"` - Device string `yaml:"device"` - SrcMac SelfMac `yaml:"src_mac"` - DstMac SelfMac `yaml:"dst_mac"` -} - -func ReadConfig(filename string) (*EtherTable, error) { - data, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - var ether EtherTable - err = yaml.Unmarshal(data, ðer) - if err != nil { - return nil, err - } - return ðer, nil -} -func (e *EtherTable) SaveConfig(filename string) error { - data, err := yaml.Marshal(e) - if err != nil { - return err - } - return ioutil.WriteFile(filename, data, 0666) -} diff --git a/core/options/options.go b/core/options/options.go deleted file mode 100644 index 412f633d..00000000 --- a/core/options/options.go +++ /dev/null @@ -1,114 +0,0 @@ -package options - -import ( - "ksubdomain/core" - "ksubdomain/core/gologger" - "os" - "strconv" -) - -type Options struct { - Rate int64 - Domain []string - FileName string // 字典文件名 - Resolvers []string - Output string // 输出文件名 - Silent bool - Stdin bool - SkipWildCard bool - TimeOut int - Retry int - Method string // verify模式 enum模式 test模式 - OnlyDomain bool - NotPrint bool - Level int - LevelDomains []string -} - -func Band2Rate(bandWith string) int64 { - suffix := string(bandWith[len(bandWith)-1]) - rate, _ := strconv.ParseInt(string(bandWith[0:len(bandWith)-1]), 10, 64) - switch suffix { - case "G": - fallthrough - case "g": - rate *= 1000000000 - case "M": - fallthrough - case "m": - rate *= 1000000 - case "K": - fallthrough - case "k": - rate *= 1000 - default: - gologger.Fatalf("unknown bandwith suffix '%s' (supported suffixes are G,M and K)\n", suffix) - } - packSize := int64(80) // 一个DNS包大概有74byte - rate = rate / packSize - return rate -} -func GetResolvers(resolvers string) []string { - // handle resolver - var rs []string - var err error - if resolvers != "" { - rs, err = core.LinesInFile(resolvers) - if err != nil { - gologger.Fatalf("读取resolvers文件失败:%s\n", err.Error()) - } - if len(rs) == 0 { - gologger.Fatalf("resolvers文件内容为空\n") - } - } else { - defaultDns := []string{ - "223.5.5.5", - "223.6.6.6", - "180.76.76.76", - "119.29.29.29", - "182.254.116.116", - "114.114.114.115", - } - rs = defaultDns - } - return rs -} -func (opt *Options) Check() { - - if opt.Silent { - gologger.MaxLevel = gologger.Silent - } - - core.ShowBanner() - - if opt.Method == "verify" { - if opt.Stdin { - - } else { - if opt.FileName == "" || !core.FileExists(opt.FileName) { - gologger.Fatalf("域名验证文件:%s 不存在! \n", opt.FileName) - } - } - } else if opt.Method == "enum" { - if opt.FileName != "" && !core.FileExists(opt.FileName) { - gologger.Fatalf("字典文件:%s 不存在! \n", opt.FileName) - } - if opt.Stdin { - - } else { - if len(opt.Domain) == 0 { - gologger.Fatalf("域名未指定目标") - } - } - } -} -func HasStdin() bool { - fi, err := os.Stdin.Stat() - if err != nil { - return false - } - if fi.Mode()&os.ModeNamedPipe == 0 { - return false - } - return true -} diff --git a/core/struct.go b/core/struct.go deleted file mode 100644 index bae18fb0..00000000 --- a/core/struct.go +++ /dev/null @@ -1,11 +0,0 @@ -package core - -import ( - "github.com/google/gopacket/layers" -) - -// 接收结果数据结构 -type RecvResult struct { - Subdomain string - Answers []layers.DNSResourceRecord -} diff --git a/core/subdata.go b/core/subdata.go deleted file mode 100644 index 349a6ef5..00000000 --- a/core/subdata.go +++ /dev/null @@ -1,20 +0,0 @@ -package core - -import ( - _ "embed" - "strings" -) - -//go:embed data/subnext.txt -var subnext string - -//go:embed data/subdomain.txt -var subdomain string - -func GetDefaultSubdomainData() []string { - return strings.Split(subdomain, "\n") -} - -func GetDefaultSubNextData() []string { - return strings.Split(subnext, "\n") -} diff --git a/core/wildcard.go b/core/wildcard.go deleted file mode 100644 index 2d8e65f5..00000000 --- a/core/wildcard.go +++ /dev/null @@ -1,15 +0,0 @@ -package core - -import "net" - -func IsWildCard(domain string) bool { - for i := 0; i < 2; i++ { - subdomain := RandomStr(6) + "." + domain - _, err := net.LookupIP(subdomain) - if err != nil { - continue - } - return true - } - return false -} diff --git a/dev.md b/dev.md new file mode 100755 index 00000000..c6c57a1b --- /dev/null +++ b/dev.md @@ -0,0 +1,80 @@ +【已过时,待重写】 + +一个简单的调用例子 +注意: 不要启动多个ksubdomain,ksubdomain启动一个就可以发挥最大作用。 + +```go +package main + +import ( + "context" + "github.com/boy-hack/ksubdomain/core/gologger" + "github.com/boy-hack/ksubdomain/core/options" + "github.com/boy-hack/ksubdomain/runner" + "github.com/boy-hack/ksubdomain/runner/outputter" + "github.com/boy-hack/ksubdomain/runner/outputter/output" + "github.com/boy-hack/ksubdomain/runner/processbar" + "strings" +) + +func main() { + process := processbar.ScreenProcess{} + screenPrinter, _ := output.NewScreenOutput(false) + + domains := []string{"www.hacking8.com", "x.hacking8.com"} + domainChanel := make(chan string) + go func() { + for _, d := range domains { + domainChanel <- d + } + close(domainChanel) + }() + opt := &options.Options{ + Rate: options.Band2Rate("1m"), + Domain: domainChanel, + DomainTotal: 2, + Resolvers: options.GetResolvers(""), + Silent: false, + TimeOut: 10, + Retry: 3, + Method: runner.VerifyType, + DnsType: "a", + Writer: []outputter.Output{ + screenPrinter, + }, + ProcessBar: &process, + EtherInfo: options.GetDeviceConfig(), + } + opt.Check() + r, err := runner.New(opt) + if err != nil { + gologger.Fatalf(err.Error()) + } + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() +} +``` +可以看到调用很简单,就是填写`options`参数,然后调用runner启动就好了,重要的是options填什么。 +options的参数结构 +```go +type Options struct { + Rate int64 // 每秒发包速率 + Domain io.Reader // 域名输入 + DomainTotal int // 扫描域名总数 + Resolvers []string // dns resolvers + Silent bool // 安静模式 + TimeOut int // 超时时间 单位(秒) + Retry int // 最大重试次数 + Method string // verify模式 enum模式 test模式 + DnsType string // dns类型 a ns aaaa + Writer []outputter.Output // 输出结构 + ProcessBar processbar.ProcessBar + EtherInfo *device.EtherTable // 网卡信息 +} +``` +1. ksubdomain底层接口只是一个dns验证器,如果要通过一级域名枚举,需要把全部的域名都放入`Domain`字段中,可以看enum参数是怎么写的 `cmd/ksubdomain/enum.go` +2. Write参数是一个outputter.Output接口,用途是如何处理DNS返回的接口,ksubdomain已经内置了三种接口在 `runner/outputter/output`中,主要作用是把数据存入内存、数据写入文件、数据打印到屏幕,可以自己实现这个接口,实现自定义的操作。 +3. ProcessBar参数是一个processbar.ProcessBar接口,主要用途是将程序内`成功个数`、`发送个数`、`队列数`、`接收数`、`失败数`、`耗时`传递给用户,实现这个参数可以时时获取这些。 +4. EtherInfo是*device.EtherTable类型,用来获取网卡的信息,一般用函数`options.GetDeviceConfig()`即可自动获取网卡配置。 + diff --git a/docs/OUTPUT_FORMATS.md b/docs/OUTPUT_FORMATS.md new file mode 100644 index 00000000..76876c7b --- /dev/null +++ b/docs/OUTPUT_FORMATS.md @@ -0,0 +1,310 @@ +# Output Formats Guide + +KSubdomain supports multiple output formats for different use cases. + +## 📋 Supported Formats + +| Format | Extension | Use Case | Streaming | Beautified | +|--------|-----------|----------|-----------|------------| +| **TXT** | .txt | Default, human-readable | ✅ | ❌ | +| **JSON** | .json | Structured data | ❌ | ❌ | +| **CSV** | .csv | Spreadsheet compatible | ❌ | ❌ | +| **JSONL** | .jsonl | Tool chaining, streaming | ✅ | ❌ | +| **Beautified** | - | Enhanced terminal output | ✅ | ✅ | + +--- + +## 1. TXT Format (Default) + +Simple text format, one result per line. + +### Usage +```bash +./ksubdomain enum -d example.com -o results.txt +# or +./ksubdomain enum -d example.com --oy txt -o results.txt +``` + +### Output +``` +www.example.com => 93.184.216.34 +mail.example.com => CNAME mail.google.com +api.example.com => 93.184.216.35 +``` + +### Best For +- Quick viewing +- Manual analysis +- Simple text processing + +--- + +## 2. JSON Format + +Structured JSON output, all results in one object. + +### Usage +```bash +./ksubdomain enum -d example.com --oy json -o results.json +``` + +### Output +```json +{ + "domains": [ + { + "subdomain": "www.example.com", + "answers": ["93.184.216.34"] + }, + { + "subdomain": "mail.example.com", + "answers": ["CNAME mail.google.com"] + } + ] +} +``` + +### Best For +- Structured data processing +- Web API integration +- Complete result sets + +--- + +## 3. CSV Format + +Comma-separated values for spreadsheet applications. + +### Usage +```bash +./ksubdomain enum -d example.com --oy csv -o results.csv +``` + +### Output +```csv +subdomain,type,record +www.example.com,A,93.184.216.34 +mail.example.com,CNAME,mail.google.com +api.example.com,A,93.184.216.35 +``` + +### Best For +- Excel/Google Sheets +- Data analysis +- Reporting + +--- + +## 4. JSONL Format (JSON Lines) 🆕 + +One JSON object per line, perfect for streaming. + +### Usage +```bash +./ksubdomain enum -d example.com --oy jsonl -o results.jsonl +``` + +### Output +```jsonl +{"domain":"www.example.com","type":"A","records":["93.184.216.34"],"timestamp":1709011200} +{"domain":"mail.example.com","type":"CNAME","records":["mail.google.com"],"timestamp":1709011201} +{"domain":"api.example.com","type":"A","records":["93.184.216.35"],"timestamp":1709011202} +``` + +### Processing with jq +```bash +# Extract domains +./ksubdomain enum -d example.com --oy jsonl | jq -r '.domain' + +# Filter A records only +./ksubdomain enum -d example.com --oy jsonl | jq -r 'select(.type == "A") | .domain' + +# Extract CNAME targets +./ksubdomain enum -d example.com --oy jsonl | jq -r 'select(.type == "CNAME") | .records[0]' + +# Filter by timestamp +./ksubdomain enum -d example.com --oy jsonl | jq -r 'select(.timestamp > 1709011000) | .domain' +``` + +### Integration Examples + +#### With httpx +```bash +./ksubdomain enum -d example.com --oy jsonl | \ + jq -r '.domain' | \ + httpx -silent +``` + +#### With nuclei +```bash +./ksubdomain enum -d example.com --oy jsonl | \ + jq -r '.domain' | \ + nuclei -l /dev/stdin +``` + +#### In Python +```python +import subprocess +import json + +proc = subprocess.Popen( + ['ksubdomain', 'enum', '-d', 'example.com', '--oy', 'jsonl'], + stdout=subprocess.PIPE, + text=True +) + +for line in proc.stdout: + data = json.loads(line) + print(f"{data['domain']} => {data['records']}") +``` + +#### In Node.js +```javascript +const { spawn } = require('child_process'); +const readline = require('readline'); + +const proc = spawn('ksubdomain', ['enum', '-d', 'example.com', '--oy', 'jsonl']); +const rl = readline.createInterface({ input: proc.stdout }); + +rl.on('line', (line) => { + const data = JSON.parse(line); + console.log(`${data.domain} => ${data.records}`); +}); +``` + +### Best For +- **Streaming processing** (real-time) +- **Tool chaining** (pipes) +- **Log aggregation** +- **Time-series analysis** + +--- + +## 5. Beautified Output 🎨 + +Enhanced terminal output with colors, emojis, and summary. + +### Usage +```bash +# Enable colors +./ksubdomain enum -d example.com --color + +# Full beautified mode +./ksubdomain enum -d example.com --beautify + +# Beautified with file output +./ksubdomain enum -d example.com --beautify -o results.txt +``` + +### Output Example +``` +✓ www.example.com 93.184.216.34 +✓ mail.example.com [CNAME] mail.google.com +✓ api.example.com 93.184.216.35 +✓ ftp.example.com 93.184.216.36 + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +📊 Scan Summary +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + Total Found: 125 + Time Elapsed: 5.2s + Speed: 2403 domains/s + + Record Types: + A: 120 (96.0%) + CNAME: 4 (3.2%) + NS: 1 (0.8%) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +``` + +### Color Scheme +- ✅ **Green checkmark** - Success +- 🔵 **Blue [CNAME]** - CNAME records +- 🟡 **Yellow [NS]** - NS records +- 🔴 **Red [WARN]** - Wildcards or warnings + +### Best For +- Terminal viewing +- Presentations +- Demos +- Human-readable output + +--- + +## 🔄 Format Comparison + +### Which format to use? + +| Scenario | Recommended Format | +|----------|-------------------| +| Quick manual check | TXT or Beautified | +| Piping to other tools | JSONL | +| Data analysis | CSV or JSON | +| Real-time processing | JSONL | +| Web API response | JSON | +| Reporting | CSV or Beautified | +| Tool integration | JSONL | + +--- + +## 📚 Examples + +### Save to multiple formats +```bash +# Save TXT and JSON +./ksubdomain enum -d example.com -o results.txt +./ksubdomain enum -d example.com --oy json -o results.json + +# Save JSONL for processing +./ksubdomain enum -d example.com --oy jsonl -o results.jsonl + +# Beautified terminal + JSON file +./ksubdomain enum -d example.com --beautify --oy json -o results.json +``` + +### Real-time monitoring +```bash +# Stream to terminal with beautification +./ksubdomain enum -d example.com --beautify + +# Stream to file in JSONL format +./ksubdomain enum -d example.com --oy jsonl -o results.jsonl & +tail -f results.jsonl | jq -r '.domain' +``` + +--- + +## 🎨 Beautified Output Details + +### Features + +1. **Color-coded types** + - A records: Green ✓ + - CNAME: Blue [CNAME] + - NS: Yellow [NS] + +2. **Aligned output** + - Domain names aligned to 40 chars + - Clean table-like appearance + +3. **Summary statistics** + - Total count + - Time elapsed + - Speed (domains/s) + - Type distribution + +4. **Emoji indicators** + - ✅ Success + - 📊 Summary + - ⚡ Speed + - 📋 Types + +### Disable colors +```bash +# Force disable (for piping/redirecting) +./ksubdomain enum -d example.com --beautify --no-color +``` + +--- + +**Choose the right format for your workflow! 🎯** diff --git a/go.mod b/go.mod old mode 100644 new mode 100755 index 07b9206b..9d7f1b44 --- a/go.mod +++ b/go.mod @@ -1,15 +1,35 @@ -module ksubdomain +module github.com/boy-hack/ksubdomain/v2 -go 1.16 +go 1.23.0 require ( + github.com/StackExchange/wmi v1.2.1 + github.com/cespare/xxhash/v2 v2.3.0 github.com/google/gopacket v1.1.19 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/mattn/go-colorable v0.1.8 - github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 + github.com/mattn/go-colorable v0.1.12 + github.com/miekg/dns v1.1.65 + github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 + github.com/stretchr/testify v1.6.1 github.com/urfave/cli/v2 v2.3.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + golang.org/x/crypto v0.37.0 + golang.org/x/sys v0.32.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b +) + +require ( + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-ole/go-ole v1.2.5 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/term v0.31.0 // indirect + golang.org/x/tools v0.32.0 // indirect ) diff --git a/go.sum b/go.sum old mode 100644 new mode 100755 index 55692bb2..f6fbfddd --- a/go.sum +++ b/go.sum @@ -1,21 +1,31 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= +github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= +github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= +github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -34,25 +44,36 @@ go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/image.gif b/image.gif old mode 100644 new mode 100755 diff --git a/core/banner.go b/pkg/core/banner.go old mode 100644 new mode 100755 similarity index 68% rename from core/banner.go rename to pkg/core/banner.go index 1fa768d2..4008c865 --- a/core/banner.go +++ b/pkg/core/banner.go @@ -1,8 +1,9 @@ package core import ( - "ksubdomain/core/conf" - "ksubdomain/core/gologger" + "fmt" + "github.com/boy-hack/ksubdomain/v2/pkg/core/conf" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" ) const banner = ` @@ -14,7 +15,9 @@ const banner = ` ` -func ShowBanner() { - gologger.Printf(banner) +func ShowBanner(silent bool) { + if !silent { + fmt.Printf(banner) + } gologger.Infof("Current Version: %s\n", conf.Version) } diff --git a/core/conf/config.go b/pkg/core/conf/config.go old mode 100644 new mode 100755 similarity index 81% rename from core/conf/config.go rename to pkg/core/conf/config.go index 57df4521..3e60ad16 --- a/core/conf/config.go +++ b/pkg/core/conf/config.go @@ -1,7 +1,7 @@ package conf const ( - Version = "1.8.1" + Version = "2.4" AppName = "KSubdomain" Description = "无状态子域名爆破工具" ) diff --git a/core/data/subdomain.txt b/pkg/core/data/subdomain.txt old mode 100644 new mode 100755 similarity index 100% rename from core/data/subdomain.txt rename to pkg/core/data/subdomain.txt diff --git a/core/data/subnext.txt b/pkg/core/data/subnext.txt old mode 100644 new mode 100755 similarity index 100% rename from core/data/subnext.txt rename to pkg/core/data/subnext.txt diff --git a/core/gologger/gologger.go b/pkg/core/gologger/gologger.go old mode 100644 new mode 100755 similarity index 98% rename from core/gologger/gologger.go rename to pkg/core/gologger/gologger.go index 1ec04e49..aa759405 --- a/core/gologger/gologger.go +++ b/pkg/core/gologger/gologger.go @@ -117,9 +117,9 @@ func log(level Level, label string, format string, args ...interface{}) { message := fmt.Sprintf(format, args...) sb.WriteString(message) - //if strings.HasSuffix(message, "\n") == false { - // sb.WriteString("\n") - //} + if strings.HasSuffix(message, "\n") == false { + sb.WriteString("\n") + } mutex.Lock() switch level { diff --git a/pkg/core/ns/ns.go b/pkg/core/ns/ns.go new file mode 100755 index 00000000..7979b5d7 --- /dev/null +++ b/pkg/core/ns/ns.go @@ -0,0 +1,37 @@ +package ns + +import ( + "errors" + "github.com/miekg/dns" + "net" +) + +// LookupNS returns the names servers for a domain. +func LookupNS(domain, serverAddr string) (servers []string, ips []string, err error) { + m := &dns.Msg{} + m.SetQuestion(dns.Fqdn(domain), dns.TypeNS) + in, err := dns.Exchange(m, serverAddr+":53") + if err != nil { + return nil, nil, err + } + if len(in.Answer) == 0 { + return nil, nil, errors.New("no Answer") + } + for _, a := range in.Answer { + if ns, ok := a.(*dns.NS); ok { + servers = append(servers, ns.Ns) + } + } + for _, s := range servers { + ipResults, err := net.LookupIP(s) + if err != nil { + continue + } + for _, ip := range ipResults { + if ip.To4() != nil { + ips = append(ips, ip.To4().String()) + } + } + } + return +} diff --git a/pkg/core/ns/ns_test.go b/pkg/core/ns/ns_test.go new file mode 100755 index 00000000..4276fc2f --- /dev/null +++ b/pkg/core/ns/ns_test.go @@ -0,0 +1,16 @@ +package ns + +import "testing" + +func TestLookupNS(t *testing.T) { + ns, ips, err := LookupNS("hacking8.com", "1.1.1.1") + if err != nil { + t.Fatalf(err.Error()) + } + for _, n := range ns { + t.Log(n) + } + for _, ip := range ips { + t.Log(ip) + } +} diff --git a/pkg/core/options/device.go b/pkg/core/options/device.go new file mode 100755 index 00000000..2d28b2c4 --- /dev/null +++ b/pkg/core/options/device.go @@ -0,0 +1,19 @@ +package options + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/device" +) + +// GetDeviceConfig 获取网卡配置信息 +// 改进版本:优先通过路由表获取网卡信息,不依赖配置文件缓存 +func GetDeviceConfig(dnsServer []string) *device.EtherTable { + // 使用改进的自动识别方法,优先通过路由表获取,不依赖配置文件 + ether, err := device.AutoGetDevicesImproved(dnsServer) + if err != nil { + gologger.Fatalf("自动识别外网网卡失败: %v\n", err) + } + + device.PrintDeviceInfo(ether) + return ether +} diff --git a/pkg/core/options/options.go b/pkg/core/options/options.go new file mode 100755 index 00000000..9bd4eb56 --- /dev/null +++ b/pkg/core/options/options.go @@ -0,0 +1,83 @@ +package options + +import ( + device2 "github.com/boy-hack/ksubdomain/v2/pkg/device" + "strconv" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" +) + +type OptionMethod string + +const ( + VerifyType OptionMethod = "verify" + EnumType OptionMethod = "enum" + TestType OptionMethod = "test" +) + +type Options struct { + Rate int64 // 每秒发包速率 + Domain chan string // 域名输入 + Resolvers []string // dns resolvers + Silent bool // 安静模式 + Retry int // 最大重试次数 + Method OptionMethod // verify模式 enum模式 test模式 + Writer []outputter.Output // 输出结构 + ProcessBar processbar.ProcessBar + EtherInfo *device2.EtherTable // 网卡信息 + SpecialResolvers map[string][]string // 可针对特定域名使用的dns resolvers + WildcardFilterMode string // 泛解析过滤模式: "basic", "advanced", "none" + WildIps []string + Predict bool // 是否开启预测模式 +} + +func Band2Rate(bandWith string) int64 { + suffix := string(bandWith[len(bandWith)-1]) + rate, _ := strconv.ParseInt(string(bandWith[0:len(bandWith)-1]), 10, 64) + switch suffix { + case "G": + fallthrough + case "g": + rate *= 1000000000 + case "M": + fallthrough + case "m": + rate *= 1000000 + case "K": + fallthrough + case "k": + rate *= 1000 + default: + gologger.Fatalf("unknown bandwith suffix '%s' (supported suffixes are G,M and K)\n", suffix) + } + packSize := int64(80) // 一个DNS包大概有74byte + rate = rate / packSize + return rate +} +func GetResolvers(resolvers []string) []string { + // handle resolver + var rs []string + if resolvers != nil { + for _, resolver := range resolvers { + rs = append(rs, resolver) + } + } else { + defaultDns := []string{ + "1.1.1.1", + "8.8.8.8", + "180.76.76.76", //百度公共 DNS + "180.184.1.1", //火山引擎 + "180.184.2.2", + } + rs = defaultDns + } + return rs +} + +func (opt *Options) Check() { + if opt.Silent { + gologger.MaxLevel = gologger.Silent + } +} diff --git a/pkg/core/predict/data/regular.cfg b/pkg/core/predict/data/regular.cfg new file mode 100755 index 00000000..0cfdfe9f --- /dev/null +++ b/pkg/core/predict/data/regular.cfg @@ -0,0 +1,6 @@ +{environment}.{subdomain}.{domain} +{environment}.{subdomain}-{prefix}.{domain} +{prefix}.{subdomain}.{domain} +{prefix}.{environment}.{subdomain}.{domain} +{prefix}-{environment}.{subdomain}.{domain} +{environment}-{prefix}.{subdomain}.{domain} \ No newline at end of file diff --git a/pkg/core/predict/data/regular.dict b/pkg/core/predict/data/regular.dict new file mode 100755 index 00000000..0380035e --- /dev/null +++ b/pkg/core/predict/data/regular.dict @@ -0,0 +1,63 @@ +[environment] +dev +pre +prepare +test +test +staging +st +st1 +prd +pro +prod +pr +api +apis +rest +restful +html5 +h5 +web +app +apps +wx +weixin +mobile +m +wap +admin +monitor +manager +dashboard +management +proxy +corp +internal +cluster +gateway + +[prefix] +preview +previous +new +old +east +west +north +es +jp +hk +shanghai +sh +beijing +bj +us +demo +alpha +beta +stable +release +rc +v1 +v2 +v3 \ No newline at end of file diff --git a/pkg/core/predict/generator.go b/pkg/core/predict/generator.go new file mode 100755 index 00000000..27c14c02 --- /dev/null +++ b/pkg/core/predict/generator.go @@ -0,0 +1,201 @@ +package predict + +import ( + "bufio" + _ "embed" + "fmt" + "strings" + "sync" +) + +//go:embed data/regular.cfg +var cfg string + +//go:embed data/regular.dict +var dict string + +// DomainGenerator 用于生成预测域名 +type DomainGenerator struct { + categories map[string][]string // 存储块分类和对应的值 + patterns []string // 域名组合模式 + subdomain string // 子域名部分 + domain string // 根域名部分 + output chan string // 输出接口 + count int // 生成的域名计数 + mu sync.Mutex // 保护count和output的互斥锁 +} + +// NewDomainGenerator 创建一个新的域名生成器 +func NewDomainGenerator(output chan string) (*DomainGenerator, error) { + // 创建生成器实例 + dg := &DomainGenerator{ + categories: make(map[string][]string), + output: output, + } + + // 加载分类字典 + if err := dg.loadDictionary(); err != nil { + return nil, fmt.Errorf("加载字典文件失败: %v", err) + } + + // 加载配置模式 + if err := dg.loadPatterns(); err != nil { + return nil, fmt.Errorf("加载配置文件失败: %v", err) + } + + return dg, nil +} + +// 从字典文件加载分类信息 +func (dg *DomainGenerator) loadDictionary() error { + scanner := bufio.NewScanner(strings.NewReader(dict)) + var currentCategory string + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + // 检查是否是分类标识 [category] + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + currentCategory = line[1 : len(line)-1] + dg.categories[currentCategory] = []string{} + } else if currentCategory != "" { + // 如果有当前分类,添加值 + dg.categories[currentCategory] = append(dg.categories[currentCategory], line) + } + } + + return scanner.Err() +} + +// 从配置文件加载域名生成模式 +func (dg *DomainGenerator) loadPatterns() error { + scanner := bufio.NewScanner(strings.NewReader(cfg)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + dg.patterns = append(dg.patterns, line) + } + } + + return scanner.Err() +} + +// SetBaseDomain 设置基础域名 +func (dg *DomainGenerator) SetBaseDomain(domain string) { + // 分离子域名和根域名 + parts := strings.Split(domain, ".") + if len(parts) <= 2 { + // 如果只有根域名 (example.com) + dg.subdomain = "" + dg.domain = domain + } else { + // 有子域名 (sub.example.com) + dg.subdomain = parts[0] + dg.domain = strings.Join(parts[1:], ".") + } +} + +// GenerateDomains 生成预测域名并实时输出 +func (dg *DomainGenerator) GenerateDomains() int { + dg.mu.Lock() + dg.count = 0 + dg.mu.Unlock() + + // 如果没有设置子域名,则直接返回 + if dg.subdomain == "" && dg.domain == "" { + return 0 + } + + // 遍历所有模式 + for _, pattern := range dg.patterns { + // 递归处理每个模式中的标签替换 + dg.processPattern(pattern, map[string]string{ + "subdomain": dg.subdomain, + "domain": dg.domain, + }) + } + + dg.mu.Lock() + result := dg.count + dg.mu.Unlock() + return result +} + +// processPattern 递归处理模式中的标签替换 +func (dg *DomainGenerator) processPattern(pattern string, replacements map[string]string) { + // 查找第一个标签 + startIdx := strings.Index(pattern, "{") + if startIdx == -1 { + // 没有更多标签,输出最终结果 + if pattern != "" && dg.output != nil { + dg.mu.Lock() + dg.output <- pattern + dg.count++ + dg.mu.Unlock() + } + return + } + + endIdx := strings.Index(pattern, "}") + if endIdx == -1 || endIdx < startIdx { + // 标签格式不正确,直接返回 + return + } + + // 提取标签名 + tagName := pattern[startIdx+1 : endIdx] + + // 检查是否已有替换值 + if value, exists := replacements[tagName]; exists { + // 已有替换值,直接替换并继续处理 + newPattern := pattern[:startIdx] + value + pattern[endIdx+1:] + dg.processPattern(newPattern, replacements) + return + } + + // 从分类中获取替换值 + values, exists := dg.categories[tagName] + if !exists || len(values) == 0 { + // 没有找到替换值,跳过此标签 + newPattern := pattern[:startIdx] + pattern[endIdx+1:] + dg.processPattern(newPattern, replacements) + return + } + + // 对每个可能的替换值递归处理 + for _, value := range values { + // 创建新的替换映射 + newReplacements := make(map[string]string) + for k, v := range replacements { + newReplacements[k] = v + } + newReplacements[tagName] = value + + // 替换当前标签并继续处理 + newPattern := pattern[:startIdx] + value + pattern[endIdx+1:] + dg.processPattern(newPattern, newReplacements) + } +} + +// PredictDomains 根据给定域名预测可能的域名变体,直接输出结果 +func PredictDomains(domain string, output chan string) (int, error) { + // 检查输出对象是否为nil + if output == nil { + return 0, fmt.Errorf("输出对象不能为空") + } + + // 创建域名生成器 + generator, err := NewDomainGenerator(output) + if err != nil { + return 0, err + } + + // 设置基础域名 + generator.SetBaseDomain(domain) + + // 生成预测域名并返回生成的数量 + return generator.GenerateDomains(), nil +} diff --git a/pkg/core/predict/generator_test.go b/pkg/core/predict/generator_test.go new file mode 100755 index 00000000..75c45f02 --- /dev/null +++ b/pkg/core/predict/generator_test.go @@ -0,0 +1,25 @@ +package predict + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +type output struct { +} + +func (o *output) Write(p []byte) (n int, err error) { + fmt.Println(string(p)) + return len(p), nil +} + +func TestRealConfigFiles(t *testing.T) { + var buf output + count, err := PredictDomains("test.example.com", &buf) + if err != nil { + t.Fatalf("使用实际配置文件进行域名预测失败: %v", err) + } + t.Log(count) + assert.Greater(t, count, 0) +} diff --git a/pkg/core/predict/readme.md b/pkg/core/predict/readme.md new file mode 100755 index 00000000..49b6b082 --- /dev/null +++ b/pkg/core/predict/readme.md @@ -0,0 +1,13 @@ + +输入域名 shoot.example.com +{subdomain} => shoot +{domain} => example.com +{x}为分类 +{x1}为除了x的其他分类 + +``` +{x}.{subdomain}.{domain} +{x}-{subdomain}.{domain} +{x}-{x1}-{subdomain}.{domain} +{subdomain}-{x}.{domain} +``` \ No newline at end of file diff --git a/pkg/core/subdata.go b/pkg/core/subdata.go new file mode 100755 index 00000000..288f5504 --- /dev/null +++ b/pkg/core/subdata.go @@ -0,0 +1,32 @@ +package core + +import ( + "bufio" + _ "embed" + "strings" +) + +//go:embed data/subnext.txt +var subnext string + +//go:embed data/subdomain.txt +var subdomain string + +func GetDefaultSubdomainData() []string { + reader := bufio.NewScanner(strings.NewReader(subdomain)) + reader.Split(bufio.ScanLines) + var ret []string + for reader.Scan() { + ret = append(ret, reader.Text()) + } + return ret +} +func GetDefaultSubNextData() []string { + reader := bufio.NewScanner(strings.NewReader(subnext)) + reader.Split(bufio.ScanLines) + var ret []string + for reader.Scan() { + ret = append(ret, reader.Text()) + } + return ret +} diff --git a/core/util.go b/pkg/core/util.go old mode 100644 new mode 100755 similarity index 52% rename from core/util.go rename to pkg/core/util.go index 7039daac..283701fb --- a/core/util.go +++ b/pkg/core/util.go @@ -2,10 +2,13 @@ package core import ( "bufio" - "golang.org/x/crypto/ssh/terminal" + "io" "math/rand" "os" + "strings" "time" + + "golang.org/x/crypto/ssh/terminal" ) func RandomStr(n int) string { @@ -33,6 +36,7 @@ func LinesInFile(fileName string) ([]string, error) { } defer f.Close() scanner := bufio.NewScanner(f) + scanner.Split(bufio.ScanLines) for scanner.Scan() { line := scanner.Text() if line != "" { @@ -42,6 +46,51 @@ func LinesInFile(fileName string) ([]string, error) { return result, nil } +// LinesReaderInFile 读取文件,返回行数 +func LinesReaderInFile(filename string) (int, error) { + f, err := os.Open(filename) + if err != nil { + return 0, err + } + defer f.Close() + + // 使用更大的缓冲区减少IO操作 + buf := make([]byte, 32*1024) + count := 0 + + for { + readSize, err := f.Read(buf) + if readSize == 0 { + break + } + + // 直接遍历缓冲区计数换行符 + for i := 0; i < readSize; i++ { + if buf[i] == '\n' { + count++ + } + } + + if err != nil { + if err == io.EOF { + // 处理文件末尾没有换行符的情况 + if readSize > 0 && (count == 0 || buf[readSize-1] != '\n') { + count++ + } + return count, nil + } + return count, err + } + } + + // 处理空文件或只有一行没有换行符的文件 + if count == 0 { + count = 1 + } + + return count, nil +} + func FileExists(path string) bool { _, err := os.Stat(path) //os.Stat获取文件信息 if err != nil { @@ -69,3 +118,21 @@ func IsContain(items []string, item string) bool { } return false } + +func SliceToString(items []string) string { + ret := strings.Builder{} + ret.WriteString("[") + ret.WriteString(strings.Join(items, ",")) + ret.WriteString("]") + return ret.String() +} +func HasStdin() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } + return true +} diff --git a/pkg/device/device.go b/pkg/device/device.go new file mode 100755 index 00000000..578d2e3e --- /dev/null +++ b/pkg/device/device.go @@ -0,0 +1,187 @@ +package device + +import ( + "fmt" + "net" + "os" + "runtime" + "strings" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/google/gopacket/pcap" + "gopkg.in/yaml.v3" +) + +// EtherTable 存储网卡信息的数据结构 +type EtherTable struct { + SrcIp net.IP `yaml:"src_ip"` // 源IP地址 + Device string `yaml:"device"` // 网卡设备名称 + SrcMac SelfMac `yaml:"src_mac"` // 源MAC地址 + DstMac SelfMac `yaml:"dst_mac"` // 目标MAC地址(通常是网关) +} + +// ReadConfig 从文件读取EtherTable配置 +func ReadConfig(filename string) (*EtherTable, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + var ether EtherTable + err = yaml.Unmarshal(data, ðer) + if err != nil { + return nil, err + } + return ðer, nil +} + +// SaveConfig 保存EtherTable配置到文件 +func (e *EtherTable) SaveConfig(filename string) error { + data, err := yaml.Marshal(e) + if err != nil { + return err + } + return os.WriteFile(filename, data, 0666) +} + +// isWSL 检测是否在 WSL/WSL2 环境中运行 +func isWSL() bool { + if runtime.GOOS != "linux" { + return false + } + data, err := os.ReadFile("/proc/version") + if err != nil { + return false + } + version := strings.ToLower(string(data)) + return strings.Contains(version, "microsoft") || strings.Contains(version, "wsl") +} + +// isDeviceUp 检查网卡是否处于激活状态 +func isDeviceUp(devicename string) bool { + iface, err := net.InterfaceByName(devicename) + if err != nil { + return false + } + // 检查 UP 标志位 + return iface.Flags&net.FlagUp != 0 +} + +// PcapInit 初始化pcap句柄 +// 修复 Issue #68: 增强错误提示,特别是 WSL2 环境 +// 修复 Mac 缓冲区问题: 使用 InactiveHandle 设置更大的缓冲区 +func PcapInit(devicename string) (*pcap.Handle, error) { + // 使用 InactiveHandle 可以在激活前设置参数 + // 这对 Mac BPF 缓冲区优化特别重要 + inactive, err := pcap.NewInactiveHandle(devicename) + if err != nil { + gologger.Fatalf("创建 pcap 句柄失败: %s\n", err.Error()) + return nil, err + } + defer inactive.CleanUp() + + // 设置 snapshot 长度为 64KB (原来 1024 太小) + // DNS 包通常 < 512 字节,但完整以太网帧可能更大 + err = inactive.SetSnapLen(65536) + if err != nil { + gologger.Warningf("设置 SnapLen 失败: %v\n", err) + } + + // 设置超时为阻塞模式 + err = inactive.SetTimeout(-1 * time.Second) + if err != nil { + gologger.Warningf("设置 Timeout 失败: %v\n", err) + } + + // Mac 平台专用优化: 增大 BPF 缓冲区 + // Mac 默认 BPF 缓冲区很小 (通常 32KB),高速发包容易溢出 + // 设置为 2MB 可显著减少 "No buffer space available" 错误 + if runtime.GOOS == "darwin" { + bufferSize := 2 * 1024 * 1024 // 2MB + err = inactive.SetBufferSize(bufferSize) + if err != nil { + gologger.Warningf("Mac: 设置 BPF 缓冲区大小失败: %v (将使用默认值)\n", err) + } else { + gologger.Infof("Mac: BPF 缓冲区已设置为 %d MB\n", bufferSize/(1024*1024)) + } + } + + // 设置即时模式 (减少延迟) + err = inactive.SetImmediateMode(true) + if err != nil { + // 即时模式失败不致命,某些平台可能不支持 + gologger.Debugf("设置即时模式失败: %v (非致命)\n", err) + } + + // 激活句柄 + handle, err := inactive.Activate() + if err != nil { + // 修复 Issue #68: 提供详细的错误信息和解决方案 + errMsg := err.Error() + + // 情况1: 网卡未激活 + if strings.Contains(errMsg, "not up") { + var solution string + if isWSL() { + // WSL/WSL2 特殊提示 + solution = fmt.Sprintf( + "网卡 %s 未激活 (WSL/WSL2 环境检测到)\n\n"+ + "解决方案:\n"+ + " 1. 激活网卡: sudo ip link set %s up\n"+ + " 2. 或使用其他网卡: ksubdomain --eth <网卡名>\n"+ + " 3. 查看可用网卡: ip link show\n"+ + " 4. WSL2 通常使用 eth0,尝试: --eth eth0\n", + devicename, devicename, + ) + } else { + solution = fmt.Sprintf( + "网卡 %s 未激活\n\n"+ + "解决方案:\n"+ + " 1. Linux: sudo ip link set %s up\n"+ + " 2. 或使用其他网卡: ksubdomain --eth <网卡名>\n"+ + " 3. 查看可用网卡: ip link show 或 ifconfig -a\n", + devicename, devicename, + ) + } + gologger.Fatalf(solution) + return nil, fmt.Errorf("网卡未激活: %s", devicename) + } + + // 情况2: 权限不足 + if strings.Contains(errMsg, "permission denied") || strings.Contains(errMsg, "Operation not permitted") { + solution := fmt.Sprintf( + "权限不足,无法访问网卡 %s\n\n"+ + "解决方案:\n"+ + " 运行: sudo %s [参数...]\n", + devicename, os.Args[0], + ) + gologger.Fatalf(solution) + return nil, fmt.Errorf("权限不足: %s", devicename) + } + + // 情况3: 网卡不存在 + if strings.Contains(errMsg, "No such device") || strings.Contains(errMsg, "doesn't exist") { + solution := fmt.Sprintf( + "网卡 %s 不存在\n\n"+ + "解决方案:\n"+ + " 1. 查看可用网卡:\n"+ + " Linux/WSL: ip link show\n"+ + " macOS: ifconfig -a\n"+ + " 2. 使用正确的网卡名: ksubdomain --eth <网卡名>\n"+ + " 3. 常见网卡名:\n"+ + " Linux: eth0, ens33, wlan0\n"+ + " macOS: en0, en1\n"+ + " WSL2: eth0\n", + devicename, + ) + gologger.Fatalf(solution) + return nil, fmt.Errorf("网卡不存在: %s", devicename) + } + + // 其他错误 + gologger.Fatalf("pcap初始化失败: %s\n详细错误: %s\n", devicename, errMsg) + return nil, err + } + + return handle, nil +} diff --git a/pkg/device/hardware.go b/pkg/device/hardware.go new file mode 100755 index 00000000..f0390d05 --- /dev/null +++ b/pkg/device/hardware.go @@ -0,0 +1,40 @@ +package device + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "gopkg.in/yaml.v3" + "net" +) + +type SelfMac net.HardwareAddr + +func (d SelfMac) String() string { + n := (net.HardwareAddr)(d) + return n.String() +} +func (d SelfMac) MarshalYAML() (interface{}, error) { + n := (net.HardwareAddr)(d) + return n.String(), nil +} +func (d SelfMac) HardwareAddr() net.HardwareAddr { + n := (net.HardwareAddr)(d) + return n +} +func (d *SelfMac) UnmarshalYAML(value *yaml.Node) error { + v := value.Value + v2, err := net.ParseMAC(v) + if err != nil { + return err + } + n := SelfMac(v2) + *d = n + return nil +} + +// 打印设备信息 +func PrintDeviceInfo(ether *EtherTable) { + gologger.Infof("Device: %s\n", ether.Device) + gologger.Infof("IP: %s\n", ether.SrcIp.String()) + gologger.Infof("Local Mac: %s\n", ether.SrcMac.String()) + gologger.Infof("Gateway Mac: %s\n", ether.DstMac.String()) +} diff --git a/pkg/device/network.go b/pkg/device/network.go new file mode 100755 index 00000000..67096e5b --- /dev/null +++ b/pkg/device/network.go @@ -0,0 +1,275 @@ +package device + +import ( + "context" + "errors" + "fmt" + "net" + "time" + + "github.com/miekg/dns" + + "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/utils" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" +) + +// 获取所有IPv4网卡信息 +func GetAllIPv4Devices() ([]string, map[string]net.IP) { + devices, err := pcap.FindAllDevs() + deviceNames := []string{} + deviceMap := make(map[string]net.IP) + + if err != nil { + gologger.Fatalf("获取网络设备失败: %s\n", err.Error()) + return deviceNames, deviceMap + } + + for _, d := range devices { + for _, address := range d.Addresses { + ip := address.IP + // 只保留IPv4且非回环地址 + if ip.To4() != nil { + deviceMap[d.Name] = ip + deviceNames = append(deviceNames, d.Name) + } + } + } + + return deviceNames, deviceMap +} + +func ValidDNS(dns string) bool { + if dns == "" { + return false + } + _, err := LookUpIP("www.baidu.com", dns) + if err != nil { + return false + } + return true +} + +func AutoGetDevices(userDNS []string) (*EtherTable, error) { + // 有效DNS列表 + var validDNS []string + + // 1. 首先检测用户提供的DNS + if len(userDNS) > 0 { + for _, dns := range userDNS { + + if ValidDNS(dns) { + validDNS = append(validDNS, dns) + } else { + gologger.Warningf("用户提供的DNS服务器无效: %s\n", dns) + } + } + } + + // 2. 如果用户DNS都无效,尝试系统DNS + if len(validDNS) == 0 { + gologger.Infof("尝试获取系统DNS服务器...\n") + systemDNS, err := utils.GetSystemDefaultDNS() + if err == nil && len(systemDNS) > 0 { + for _, dns := range systemDNS { + if ValidDNS(dns) { + validDNS = append(validDNS, dns) + } else { + gologger.Debugf("系统DNS服务器无效: %s\n", dns) + } + } + } else { + gologger.Warningf("获取系统DNS失败: %v\n", err) + } + } + + if len(validDNS) == 0 { + return nil, fmt.Errorf("没有找到有效DNS,无法进行测试") + } + + gologger.Infof("使用以下DNS服务器进行测试: %v\n", validDNS) + return AutoGetDevicesWithDNS(validDNS), nil +} + +// AutoGetDevicesWithDNS 使用指定DNS自动获取外网发包网卡 +// 如果传入的DNS无效,则尝试使用系统DNS +func AutoGetDevicesWithDNS(validDNS []string) *EtherTable { + // 获取所有IPv4网卡 + deviceNames, _ := GetAllIPv4Devices() + if len(deviceNames) == 0 { + gologger.Fatalf("未发现可用的IPv4网卡\n") + return nil + } + + // 创建随机域名用于测试 + domain := core.RandomStr(6) + ".baidu.com" + signal := make(chan *EtherTable) + + // 启动上下文,用于控制所有goroutine + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // 测试所有网卡 + activeDeviceCount := 0 + for _, deviceName := range deviceNames { + gologger.Infof("正在测试网卡 %s 的连通性...\n", deviceName) + go testDeviceConnectivity(ctx, deviceName, domain, signal) + activeDeviceCount++ + } + // 等待测试结果或超时 + return waitForDeviceTest(signal, domain, validDNS, 30) +} + +// 测试网卡连通性 +func testDeviceConnectivity(ctx context.Context, deviceName string, domain string, signal chan<- *EtherTable) { + var ( + snapshot_len int32 = 2048 // 增加抓包大小 + promiscuous bool = true // 启用混杂模式 + timeout time.Duration = 500 * time.Millisecond // 增加超时时间 + ) + + handle, err := pcap.OpenLive(deviceName, snapshot_len, promiscuous, timeout) + if err != nil { + gologger.Debugf("无法打开网卡 %s: %s\n", deviceName, err.Error()) + return + } + defer handle.Close() + + // 添加BPF过滤器,只捕获DNS响应包 + err = handle.SetBPFFilter("udp port 53") + if err != nil { + gologger.Debugf("设置过滤器失败 %s: %s\n", deviceName, err.Error()) + // 继续尝试,不直接返回 + } + + for { + select { + case <-ctx.Done(): + return + default: + var udp layers.UDP + var dns layers.DNS + var eth layers.Ethernet + var ipv4 layers.IPv4 + + parser := gopacket.NewDecodingLayerParser( + layers.LayerTypeEthernet, ð, &ipv4, &udp, &dns) + + data, _, err := handle.ReadPacketData() + if err != nil { + if errors.Is(err, pcap.NextErrorTimeoutExpired) { + continue + } + continue // 不要立即返回,继续尝试 + } + + var decoded []gopacket.LayerType + err = parser.DecodeLayers(data, &decoded) + if err != nil { + continue + } + + // 检查是否解析到DNS层 + dnsFound := false + for _, layerType := range decoded { + if layerType == layers.LayerTypeDNS { + dnsFound = true + break + } + } + if !dnsFound { + continue + } + + // 只处理DNS响应 + if !dns.QR { + continue + } + + // 检查是否匹配我们的测试域名 + for _, q := range dns.Questions { + questionName := string(q.Name) + gologger.Debugf("收到DNS响应 %s,域名: %s\n", deviceName, questionName) + if questionName == domain || questionName == domain+"." { + etherTable := EtherTable{ + SrcIp: ipv4.DstIP, + Device: deviceName, + SrcMac: SelfMac(eth.DstMAC), + DstMac: SelfMac(eth.SrcMAC), + } + signal <- ðerTable + return + } + } + } + } +} + +// 等待设备测试结果 +func waitForDeviceTest(signal <-chan *EtherTable, domain string, dnsServers []string, timeout int) *EtherTable { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + count := 0 + // 轮询使用DNS服务器列表 + dnsIndex := 0 + + for { + select { + case result := <-signal: + gologger.Infof("成功获取到外网网卡: %s\n", result.Device) + return result + case <-ticker.C: + // 每秒尝试一次DNS查询,轮换使用不同的DNS服务器 + currentDNS := dnsServers[dnsIndex] + dnsIndex = (dnsIndex + 1) % len(dnsServers) + + go func(server string) { + ip, err := LookUpIP(domain, server) + if err != nil { + gologger.Debugf("DNS查询失败(%s): %s\n", server, err.Error()) + } else if ip != nil { + gologger.Debugf("DNS查询成功(%s): %s -> %s\n", server, domain, ip.String()) + } + }(currentDNS) + + fmt.Print(".") + count++ + + if count >= timeout { + gologger.Fatalf("获取网络设备超时,请尝试手动指定网卡\n") + return nil + } + } + } +} + +// LookUpIP 使用指定DNS服务器查询域名并返回IP地址 +func LookUpIP(fqdn, serverAddr string) (net.IP, error) { + var m dns.Msg + client := dns.Client{} + client.Timeout = time.Second + m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA) + r, _, err := client.Exchange(&m, serverAddr+":53") + + if err != nil { + return nil, err + } + + // 检查是否有响应 + if r == nil || len(r.Answer) == 0 { + return nil, fmt.Errorf("无DNS回复") + } + + // 尝试获取A记录 + for _, ans := range r.Answer { + if a, ok := ans.(*dns.A); ok { + return a.A, nil + } + } + + return nil, fmt.Errorf("无A记录") +} diff --git a/pkg/device/network_improved.go b/pkg/device/network_improved.go new file mode 100644 index 00000000..cf26dca1 --- /dev/null +++ b/pkg/device/network_improved.go @@ -0,0 +1,356 @@ +package device + +import ( + "context" + "fmt" + "net" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" +) + +// GetDefaultRouteInterface 获取默认路由的网卡设备 +// 这是最可靠的方法,因为默认路由的网卡通常就是外网通信的网卡 +func GetDefaultRouteInterface() (*EtherTable, error) { + var defaultInterface string + var gatewayIP net.IP + + switch runtime.GOOS { + case "windows": + // Windows: 使用 route print 获取默认路由 + cmd := exec.Command("route", "print", "0.0.0.0") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("执行route命令失败: %v", err) + } + + // 解析输出获取默认网关和接口 + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "0.0.0.0") && strings.Contains(line, "0.0.0.0") { + fields := strings.Fields(line) + if len(fields) >= 5 { + gatewayIP = net.ParseIP(fields[2]) + // 获取接口IP + localIP := net.ParseIP(fields[3]) + if localIP != nil { + // 查找对应的网卡 + interfaces, _ := pcap.FindAllDevs() + for _, iface := range interfaces { + for _, addr := range iface.Addresses { + if addr.IP.Equal(localIP) { + defaultInterface = iface.Name + break + } + } + if defaultInterface != "" { + break + } + } + } + break + } + } + } + + case "linux": + // Linux: 使用 ip route 获取默认路由 + cmd := exec.Command("ip", "route", "show", "default") + output, err := cmd.Output() + if err != nil { + // 尝试使用 route 命令 + cmd = exec.Command("route", "-n") + output, err = cmd.Output() + if err != nil { + return nil, fmt.Errorf("获取路由信息失败: %v", err) + } + } + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "default") || strings.HasPrefix(line, "0.0.0.0") { + fields := strings.Fields(line) + if len(fields) >= 5 { + // ip route 格式: default via 192.168.1.1 dev eth0 + if fields[0] == "default" && len(fields) >= 5 { + gatewayIP = net.ParseIP(fields[2]) + defaultInterface = fields[4] + } else if fields[0] == "0.0.0.0" { + // route -n 格式 + gatewayIP = net.ParseIP(fields[1]) + defaultInterface = fields[len(fields)-1] + } + break + } + } + } + + case "darwin": + // macOS: 使用 route get 获取默认路由 + cmd := exec.Command("route", "get", "default") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("获取路由信息失败: %v", err) + } + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "gateway:") { + parts := strings.Split(line, ":") + if len(parts) >= 2 { + gatewayIP = net.ParseIP(strings.TrimSpace(parts[1])) + } + } else if strings.HasPrefix(line, "interface:") { + parts := strings.Split(line, ":") + if len(parts) >= 2 { + defaultInterface = strings.TrimSpace(parts[1]) + } + } + } + } + + if defaultInterface == "" || gatewayIP == nil { + return nil, fmt.Errorf("无法获取默认路由信息") + } + + gologger.Infof("找到默认路由网卡: %s, 网关: %s\n", defaultInterface, gatewayIP.String()) + + // 获取网卡的IP和MAC地址 + etherTable, err := getInterfaceDetails(defaultInterface, gatewayIP) + if err != nil { + return nil, err + } + + return etherTable, nil +} + +// getInterfaceDetails 获取网卡详细信息,包括通过ARP获取网关MAC +func getInterfaceDetails(deviceName string, gatewayIP net.IP) (*EtherTable, error) { + // 获取网卡信息 + interfaces, err := pcap.FindAllDevs() + if err != nil { + return nil, fmt.Errorf("获取网卡列表失败: %v", err) + } + + var srcIP net.IP + var srcMAC net.HardwareAddr + + // 查找指定网卡的IP和MAC + for _, iface := range interfaces { + if iface.Name == deviceName { + // 获取IP地址 + for _, addr := range iface.Addresses { + if addr.IP.To4() != nil && !addr.IP.IsLoopback() { + srcIP = addr.IP + break + } + } + break + } + } + + if srcIP == nil { + return nil, fmt.Errorf("无法获取网卡 %s 的IP地址", deviceName) + } + + // 获取网卡MAC地址 + iface, err := net.InterfaceByName(deviceName) + if err == nil && iface.HardwareAddr != nil { + srcMAC = iface.HardwareAddr + } else { + // 如果标准方法失败,尝试从系统获取 + srcMAC, _ = getMACAddress(deviceName) + } + + if srcMAC == nil { + // 使用默认MAC + srcMAC = net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + gologger.Warningf("无法获取网卡MAC地址,使用默认值\n") + } + + // 通过ARP获取网关MAC地址 + gatewayMAC, err := resolveGatewayMAC(deviceName, srcIP, srcMAC, gatewayIP) + if err != nil { + gologger.Warningf("ARP解析网关MAC失败: %v,将使用广播地址\n", err) + gatewayMAC = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + } + + etherTable := &EtherTable{ + SrcIp: srcIP, + Device: deviceName, + SrcMac: SelfMac(srcMAC), + DstMac: SelfMac(gatewayMAC), + } + + gologger.Infof("网卡配置: IP=%s, MAC=%s, Gateway MAC=%s\n", + srcIP.String(), srcMAC.String(), gatewayMAC.String()) + + return etherTable, nil +} + +// resolveGatewayMAC 通过ARP请求获取网关的MAC地址 +func resolveGatewayMAC(deviceName string, srcIP net.IP, srcMAC net.HardwareAddr, gatewayIP net.IP) (net.HardwareAddr, error) { + // 打开网卡进行ARP操作 + handle, err := pcap.OpenLive(deviceName, 2048, true, time.Second) + if err != nil { + return nil, fmt.Errorf("打开网卡失败: %v", err) + } + defer handle.Close() + + // 设置过滤器只接收ARP回复 + err = handle.SetBPFFilter(fmt.Sprintf("arp and arp[6:2] = 2 and src host %s", gatewayIP.String())) + if err != nil { + gologger.Debugf("设置BPF过滤器失败: %v\n", err) + } + + // 构建ARP请求包 + eth := &layers.Ethernet{ + SrcMAC: srcMAC, + DstMAC: net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, // 广播 + EthernetType: layers.EthernetTypeARP, + } + + arp := &layers.ARP{ + AddrType: layers.LinkTypeEthernet, + Protocol: layers.EthernetTypeIPv4, + HwAddressSize: 6, + ProtAddressSize: 4, + Operation: layers.ARPRequest, + SourceHwAddress: srcMAC, + SourceProtAddress: srcIP.To4(), + DstHwAddress: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + DstProtAddress: gatewayIP.To4(), + } + + // 序列化数据包 + buffer := gopacket.NewSerializeBuffer() + opts := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + + err = gopacket.SerializeLayers(buffer, opts, eth, arp) + if err != nil { + return nil, fmt.Errorf("构建ARP包失败: %v", err) + } + + // 发送ARP请求 + outgoingPacket := buffer.Bytes() + err = handle.WritePacketData(outgoingPacket) + if err != nil { + return nil, fmt.Errorf("发送ARP请求失败: %v", err) + } + + // 等待ARP回复 + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) + for { + select { + case <-ctx.Done(): + return nil, fmt.Errorf("ARP响应超时") + case packet := <-packetSource.Packets(): + if packet == nil { + continue + } + + // 解析ARP层 + if arpLayer := packet.Layer(layers.LayerTypeARP); arpLayer != nil { + arpReply, ok := arpLayer.(*layers.ARP) + if ok && arpReply.Operation == layers.ARPReply { + // 检查是否是我们请求的网关IP的回复 + if net.IP(arpReply.SourceProtAddress).Equal(gatewayIP) { + return net.HardwareAddr(arpReply.SourceHwAddress), nil + } + } + } + } + } +} + +// getMACAddress 获取网卡MAC地址的辅助函数 +func getMACAddress(deviceName string) (net.HardwareAddr, error) { + // 尝试通过系统命令获取MAC地址 + switch runtime.GOOS { + case "windows": + // Windows: 使用 getmac 命令 + cmd := exec.Command("getmac", "/v") + output, err := cmd.Output() + if err != nil { + return nil, err + } + // 解析输出找到对应网卡的MAC + // 这里需要更复杂的解析逻辑 + _ = output + + case "linux", "darwin": + // Linux/macOS: 使用 ifconfig + cmd := exec.Command("ifconfig", deviceName) + output, err := cmd.Output() + if err != nil { + return nil, err + } + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, "ether") || strings.Contains(line, "HWaddr") { + fields := strings.Fields(line) + for i, field := range fields { + if field == "ether" || field == "HWaddr" { + if i+1 < len(fields) { + mac, err := net.ParseMAC(fields[i+1]) + if err == nil { + return mac, nil + } + } + } + } + } + } + } + + return nil, fmt.Errorf("无法获取MAC地址") +} + +// AutoGetDevicesImproved 改进的自动获取网卡方法 +// 优先使用路由表和ARP,失败时再回退到DNS探测 +func AutoGetDevicesImproved(userDNS []string) (*EtherTable, error) { + gologger.Infof("尝试通过默认路由获取网卡信息...\n") + + // 方法1: 通过默认路由获取 + etherTable, err := GetDefaultRouteInterface() + if err == nil { + // 验证网卡是否可用 + if validateInterface(etherTable) { + gologger.Infof("成功通过默认路由获取网卡信息\n") + return etherTable, nil + } + } + + gologger.Warningf("默认路由方法失败: %v,尝试DNS探测方法\n", err) + + // 方法2: 回退到原始的DNS探测方法 + return AutoGetDevices(userDNS) +} + +// validateInterface 验证网卡是否可用 +func validateInterface(etherTable *EtherTable) bool { + // 尝试打开网卡 + handle, err := pcap.OpenLive(etherTable.Device, 1024, false, time.Second) + if err != nil { + return false + } + defer handle.Close() + + // 检查是否能设置BPF过滤器 + err = handle.SetBPFFilter("udp") + return err == nil +} diff --git a/pkg/device/network_improved_test.go b/pkg/device/network_improved_test.go new file mode 100644 index 00000000..770e2041 --- /dev/null +++ b/pkg/device/network_improved_test.go @@ -0,0 +1,242 @@ +package device + +import ( + "net" + "runtime" + "testing" + "time" + + "github.com/google/gopacket/pcap" + "github.com/stretchr/testify/assert" +) + +// TestGetDefaultRouteInterface 测试获取默认路由网卡 +func TestGetDefaultRouteInterface(t *testing.T) { + // 跳过需要root权限的测试 + if !hasAdminPrivileges() { + t.Skip("需要管理员权限运行此测试") + } + + etherTable, err := GetDefaultRouteInterface() + + // 在CI环境或无网络环境可能失败 + if err != nil { + t.Logf("获取默认路由失败(可能是环境问题): %v", err) + return + } + + // 验证返回的数据 + assert.NotNil(t, etherTable) + assert.NotEmpty(t, etherTable.Device, "设备名不应为空") + assert.NotNil(t, etherTable.SrcIp, "源IP不应为空") + assert.False(t, etherTable.SrcIp.IsLoopback(), "不应是回环地址") + assert.NotEqual(t, "00:00:00:00:00:00", etherTable.SrcMac.String(), "MAC地址不应全零") + + t.Logf("成功获取网卡: Device=%s, IP=%s, MAC=%s, Gateway MAC=%s", + etherTable.Device, etherTable.SrcIp, etherTable.SrcMac, etherTable.DstMac) +} + +// TestResolveGatewayMAC 测试ARP解析网关MAC +func TestResolveGatewayMAC(t *testing.T) { + if !hasAdminPrivileges() { + t.Skip("需要管理员权限运行此测试") + } + + // 获取本地网络信息 + etherTable, err := GetDefaultRouteInterface() + if err != nil { + t.Skip("无法获取网络信息,跳过ARP测试") + } + + // 尝试解析本地网关 + // 注意:这个测试在实际环境中运行 + gatewayIP := getDefaultGateway() + if gatewayIP == nil { + t.Skip("无法获取默认网关,跳过ARP测试") + } + + srcMAC := net.HardwareAddr(etherTable.SrcMac) + mac, err := resolveGatewayMAC(etherTable.Device, etherTable.SrcIp, srcMAC, gatewayIP) + + if err != nil { + t.Logf("ARP解析失败(可能是网络环境): %v", err) + return + } + + assert.NotNil(t, mac) + assert.Len(t, mac, 6, "MAC地址应该是6字节") + assert.NotEqual(t, "00:00:00:00:00:00", mac.String(), "MAC地址不应全零") + assert.NotEqual(t, "ff:ff:ff:ff:ff:ff", mac.String(), "MAC地址不应是广播地址") + + t.Logf("成功解析网关MAC: %s -> %s", gatewayIP, mac) +} + +// TestValidateInterface 测试网卡验证 +func TestValidateInterface(t *testing.T) { + if !hasAdminPrivileges() { + t.Skip("需要管理员权限运行此测试") + } + + tests := []struct { + name string + etherTable *EtherTable + expected bool + }{ + { + name: "无效的网卡名", + etherTable: &EtherTable{ + Device: "invalid_device_xyz", + SrcIp: net.ParseIP("192.168.1.100"), + }, + expected: false, + }, + { + name: "空网卡名", + etherTable: &EtherTable{ + Device: "", + SrcIp: net.ParseIP("192.168.1.100"), + }, + expected: false, + }, + } + + // 添加一个有效网卡的测试 + devices, err := pcap.FindAllDevs() + if err == nil && len(devices) > 0 { + validDevice := devices[0].Name + tests = append(tests, struct { + name string + etherTable *EtherTable + expected bool + }{ + name: "有效的网卡", + etherTable: &EtherTable{ + Device: validDevice, + SrcIp: net.ParseIP("192.168.1.100"), + }, + expected: true, + }) + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := validateInterface(tt.etherTable) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestAutoGetDevicesImproved 测试改进的自动获取方法 +func TestAutoGetDevicesImproved(t *testing.T) { + if !hasAdminPrivileges() { + t.Skip("需要管理员权限运行此测试") + } + + // 使用常见的公共DNS服务器 + testDNS := []string{ + "8.8.8.8", + "1.1.1.1", + "114.114.114.114", + } + + etherTable, err := AutoGetDevicesImproved(testDNS) + + // 在某些环境可能失败 + if err != nil { + t.Logf("自动获取网卡失败(环境问题): %v", err) + return + } + + assert.NotNil(t, etherTable) + assert.NotEmpty(t, etherTable.Device) + assert.NotNil(t, etherTable.SrcIp) + assert.False(t, etherTable.SrcIp.IsUnspecified(), "IP不应是未指定地址") + + t.Logf("成功自动获取网卡: %+v", etherTable) +} + +// BenchmarkGetDefaultRouteInterface 性能测试 +func BenchmarkGetDefaultRouteInterface(b *testing.B) { + if !hasAdminPrivileges() { + b.Skip("需要管理员权限运行此测试") + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = GetDefaultRouteInterface() + } +} + +// BenchmarkResolveGatewayMAC 性能测试ARP解析 +func BenchmarkResolveGatewayMAC(b *testing.B) { + if !hasAdminPrivileges() { + b.Skip("需要管理员权限运行此测试") + } + + // 准备测试数据 + etherTable, err := GetDefaultRouteInterface() + if err != nil { + b.Skip("无法获取网络信息") + } + + gatewayIP := getDefaultGateway() + if gatewayIP == nil { + b.Skip("无法获取网关") + } + + srcMAC := net.HardwareAddr(etherTable.SrcMac) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = resolveGatewayMAC(etherTable.Device, etherTable.SrcIp, srcMAC, gatewayIP) + } +} + +// 辅助函数:检查是否有管理员权限 +func hasAdminPrivileges() bool { + switch runtime.GOOS { + case "windows": + // Windows下检查是否能打开网卡 + devices, err := pcap.FindAllDevs() + return err == nil && len(devices) > 0 + default: + // Unix系统检查UID + return runtime.GOOS == "darwin" || isRoot() + } +} + +// 辅助函数:检查是否是root用户 +func isRoot() bool { + // 尝试打开一个网卡来检查权限 + devices, err := pcap.FindAllDevs() + if err != nil || len(devices) == 0 { + return false + } + + // 尝试打开第一个设备 + handle, err := pcap.OpenLive(devices[0].Name, 1024, false, time.Second) + if err != nil { + return false + } + handle.Close() + return true +} + +// 辅助函数:获取默认网关 +func getDefaultGateway() net.IP { + // 简单实现,实际使用时应该解析路由表 + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + return nil + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*net.UDPAddr) + // 假设网关是 .1 + ip := localAddr.IP.To4() + if ip != nil { + ip[3] = 1 + return ip + } + return nil +} diff --git a/pkg/device/network_test.go b/pkg/device/network_test.go new file mode 100755 index 00000000..081c7a57 --- /dev/null +++ b/pkg/device/network_test.go @@ -0,0 +1,12 @@ +package device + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAutoGetDevices(t *testing.T) { + ether, err := AutoGetDevices([]string{"1.1.1.1"}) + assert.NoError(t, err) + PrintDeviceInfo(ether) +} diff --git a/pkg/privileges/privileges.go b/pkg/privileges/privileges.go new file mode 100755 index 00000000..53d55326 --- /dev/null +++ b/pkg/privileges/privileges.go @@ -0,0 +1,5 @@ +package privileges + +func IsPrivileged() bool { + return isPrivileged() +} diff --git a/pkg/privileges/privileges_darwin.go b/pkg/privileges/privileges_darwin.go new file mode 100755 index 00000000..1a10639f --- /dev/null +++ b/pkg/privileges/privileges_darwin.go @@ -0,0 +1,12 @@ +//go:build darwin + +package privileges + +import ( + "os" +) + +// isPrivileged checks if the current process has the CAP_NET_RAW capability or is root +func isPrivileged() bool { + return os.Geteuid() == 0 +} diff --git a/pkg/privileges/privileges_linux.go b/pkg/privileges/privileges_linux.go new file mode 100755 index 00000000..77282d82 --- /dev/null +++ b/pkg/privileges/privileges_linux.go @@ -0,0 +1,33 @@ +//go:build linux || unix + +package privileges + +import ( + "golang.org/x/sys/unix" + "os" + "runtime" + "x-agent/pkg/privileges/israce" +) + +// isPrivileged checks if the current process has the CAP_NET_RAW capability or is root +func isPrivileged() bool { + // runtime.LockOSThread interferes with race detection + if !israce.Enabled { + header := unix.CapUserHeader{ + Version: unix.LINUX_CAPABILITY_VERSION_3, + Pid: int32(os.Getpid()), + } + data := unix.CapUserData{} + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + if err := unix.Capget(&header, &data); err == nil { + data.Inheritable = (1 << unix.CAP_NET_RAW) + + if err := unix.Capset(&header, &data); err == nil { + return true + } + } + } + return os.Geteuid() == 0 +} diff --git a/pkg/privileges/privileges_win.go b/pkg/privileges/privileges_win.go new file mode 100755 index 00000000..f9efccff --- /dev/null +++ b/pkg/privileges/privileges_win.go @@ -0,0 +1,8 @@ +//go:build windows + +package privileges + +// IsPrivileged on windows doesn't matter as we are using connect scan +func isPrivileged() bool { + return false +} diff --git a/pkg/runner/mempool.go b/pkg/runner/mempool.go new file mode 100755 index 00000000..fe85ec7b --- /dev/null +++ b/pkg/runner/mempool.go @@ -0,0 +1,126 @@ +package runner + +import ( + "sync" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" +) + +// MemoryPool 实现内存对象池 +// 优化点5: 对象池复用策略优化 +// 用途: 复用频繁分配的对象(DNS层、序列化缓冲区、切片) +// 收益: 减少内存分配次数和GC压力,降低延迟 +// 关键: 归还前必须重置对象状态,避免数据污染 +type MemoryPool struct { + dnsPool sync.Pool // DNS查询/响应层对象池 + bufPool sync.Pool // gopacket序列化缓冲区池 + questionPool sync.Pool // DNS问题切片池 + answerPool sync.Pool // DNS应答切片池 +} + +// 全局内存池实例 +var GlobalMemPool = NewMemoryPool() + +// NewMemoryPool 创建一个新的内存池 +func NewMemoryPool() *MemoryPool { + return &MemoryPool{ + dnsPool: sync.Pool{ + New: func() interface{} { + return &layers.DNS{ + Questions: make([]layers.DNSQuestion, 0, 1), + Answers: make([]layers.DNSResourceRecord, 0, 4), + } + }, + }, + bufPool: sync.Pool{ + New: func() interface{} { + return gopacket.NewSerializeBuffer() + }, + }, + questionPool: sync.Pool{ + New: func() interface{} { + return make([]layers.DNSQuestion, 0, 1) + }, + }, + answerPool: sync.Pool{ + New: func() interface{} { + return make([]layers.DNSResourceRecord, 0, 4) + }, + }, + } +} + +// GetDNS 获取一个DNS对象 +// 注意: 从池中获取的对象可能包含旧数据,必须重置所有字段 +func (p *MemoryPool) GetDNS() *layers.DNS { + dns := p.dnsPool.Get().(*layers.DNS) + // 重置切片长度(保留底层数组容量) + dns.Questions = dns.Questions[:0] + dns.Answers = dns.Answers[:0] + // nil 掉不常用字段,避免内存泄漏 + dns.Authorities = nil + dns.Additionals = nil + // 重置所有标志位为默认值 + dns.ID = 0 + dns.QR = false // 查询报文 + dns.OpCode = 0 // 标准查询 + dns.AA = false // 非权威应答 + dns.TC = false // 未截断 + dns.RD = true // 期望递归 + dns.RA = false // 递归不可用 + dns.Z = 0 // 保留位 + dns.ResponseCode = 0 // 无错误 + dns.QDCount = 0 + dns.ANCount = 0 + dns.NSCount = 0 + dns.ARCount = 0 + return dns +} + +// PutDNS 回收一个DNS对象 +func (p *MemoryPool) PutDNS(dns *layers.DNS) { + if dns != nil { + p.dnsPool.Put(dns) + } +} + +// GetBuffer 获取一个序列化缓冲区 +func (p *MemoryPool) GetBuffer() gopacket.SerializeBuffer { + buf := p.bufPool.Get().(gopacket.SerializeBuffer) + buf.Clear() + return buf +} + +// PutBuffer 回收一个序列化缓冲区 +func (p *MemoryPool) PutBuffer(buf gopacket.SerializeBuffer) { + if buf != nil { + p.bufPool.Put(buf) + } +} + +// GetDNSQuestions 获取DNS问题切片 +func (p *MemoryPool) GetDNSQuestions() []layers.DNSQuestion { + questions := p.questionPool.Get().([]layers.DNSQuestion) + return questions[:0] +} + +// PutDNSQuestions 回收DNS问题切片 +func (p *MemoryPool) PutDNSQuestions(questions []layers.DNSQuestion) { + if questions != nil { + p.questionPool.Put(questions) + } +} + +// GetDNSAnswers 获取DNS应答切片 +func (p *MemoryPool) GetDNSAnswers() []layers.DNSResourceRecord { + answers := p.answerPool.Get().([]layers.DNSResourceRecord) + return answers[:0] +} + +// PutDNSAnswers 回收DNS应答切片 +func (p *MemoryPool) PutDNSAnswers(answers []layers.DNSResourceRecord) { + if answers != nil { + p.answerPool.Put(answers) + } +} diff --git a/pkg/runner/outputter/output.go b/pkg/runner/outputter/output.go new file mode 100755 index 00000000..077f07b6 --- /dev/null +++ b/pkg/runner/outputter/output.go @@ -0,0 +1,10 @@ +package outputter + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +type Output interface { + WriteDomainResult(domain result.Result) error + Close() error +} diff --git a/pkg/runner/outputter/output/beautified.go b/pkg/runner/outputter/output/beautified.go new file mode 100644 index 00000000..88d3a310 --- /dev/null +++ b/pkg/runner/outputter/output/beautified.go @@ -0,0 +1,186 @@ +package output + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "github.com/logrusorgru/aurora" +) + +// BeautifiedOutput 美化输出器 +// 提供彩色、对齐、emoji 等美化功能 +type BeautifiedOutput struct { + windowsWidth int + silent bool + onlyDomain bool + useColor bool + results []result.Result + mu sync.Mutex + startTime time.Time + typeCount map[string]int +} + +// NewBeautifiedOutput creates a beautified output writer +func NewBeautifiedOutput(silent bool, useColor bool, onlyDomain ...bool) (*BeautifiedOutput, error) { + b := &BeautifiedOutput{ + windowsWidth: 120, // Default width + silent: silent, + useColor: useColor, + results: make([]result.Result, 0), + startTime: time.Now(), + typeCount: make(map[string]int), + } + + if len(onlyDomain) > 0 { + b.onlyDomain = onlyDomain[0] + } + + return b, nil +} + +// WriteDomainResult writes a single domain result with beautification +func (b *BeautifiedOutput) WriteDomainResult(r result.Result) error { + b.mu.Lock() + defer b.mu.Unlock() + + b.results = append(b.results, r) + + // Determine record type + recordType := "A" + displayRecords := make([]string, 0, len(r.Answers)) + + for _, answer := range r.Answers { + if strings.HasPrefix(answer, "CNAME ") { + recordType = "CNAME" + displayRecords = append(displayRecords, answer[6:]) + } else if strings.HasPrefix(answer, "NS ") { + recordType = "NS" + displayRecords = append(displayRecords, answer[3:]) + } else if strings.HasPrefix(answer, "PTR ") { + recordType = "PTR" + displayRecords = append(displayRecords, answer[4:]) + } else { + displayRecords = append(displayRecords, answer) + } + } + + if len(displayRecords) == 0 { + displayRecords = r.Answers + } + + // Count by type + b.typeCount[recordType]++ + + // Format output + var output string + + if b.onlyDomain { + output = r.Subdomain + } else { + // Build formatted output + domain := r.Subdomain + recordsStr := strings.Join(displayRecords, ", ") + + if b.useColor { + // Colorized output + au := aurora.NewAurora(true) + + switch recordType { + case "A", "AAAA": + // Green for A/AAAA records + output = fmt.Sprintf("%s %s %s", + au.Green("✓").String(), + au.Cyan(domain).String(), + au.White(recordsStr).String()) + case "CNAME": + // Blue for CNAME + output = fmt.Sprintf("%s %s %s %s", + au.Green("✓").String(), + au.Cyan(domain).String(), + au.Blue("[CNAME]").String(), + au.White(recordsStr).String()) + case "NS": + // Yellow for NS + output = fmt.Sprintf("%s %s %s %s", + au.Green("✓").String(), + au.Cyan(domain).String(), + au.Yellow("[NS]").String(), + au.White(recordsStr).String()) + default: + output = fmt.Sprintf("%s %s %s", + au.Green("✓").String(), + au.Cyan(domain).String(), + au.White(recordsStr).String()) + } + } else { + // Plain output + if recordType == "A" || recordType == "AAAA" { + output = fmt.Sprintf("%-40s => %s", domain, recordsStr) + } else { + output = fmt.Sprintf("%-40s [%s] %s", domain, recordType, recordsStr) + } + } + } + + if !b.silent { + gologger.Silentf("%s\n", output) + } + + return nil +} + +// Close prints summary and closes the output +func (b *BeautifiedOutput) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + + if len(b.results) == 0 { + return nil + } + + elapsed := time.Since(b.startTime) + + // Print summary + if b.useColor { + au := aurora.NewAurora(true) + fmt.Println() + fmt.Println(au.Green("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━").String()) + fmt.Printf("%s Scan Summary\n", au.Bold("📊")) + fmt.Println(au.Green("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━").String()) + fmt.Printf(" %s %s\n", au.Bold("Total Found:"), au.Green(len(b.results))) + fmt.Printf(" %s %s\n", au.Bold("Time Elapsed:"), au.Cyan(elapsed.Round(time.Millisecond))) + fmt.Printf(" %s %.0f domains/s\n", au.Bold("Speed:"), float64(len(b.results))/elapsed.Seconds()) + + if len(b.typeCount) > 0 { + fmt.Printf("\n %s\n", au.Bold("Record Types:")) + for recType, count := range b.typeCount { + percentage := float64(count) / float64(len(b.results)) * 100 + fmt.Printf(" %s: %d (%.1f%%)\n", recType, count, percentage) + } + } + fmt.Println(au.Green("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━").String()) + } else { + fmt.Println() + fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + fmt.Println("📊 Scan Summary") + fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + fmt.Printf(" Total Found: %d\n", len(b.results)) + fmt.Printf(" Time Elapsed: %v\n", elapsed.Round(time.Millisecond)) + fmt.Printf(" Speed: %.0f domains/s\n", float64(len(b.results))/elapsed.Seconds()) + + if len(b.typeCount) > 0 { + fmt.Println("\n Record Types:") + for recType, count := range b.typeCount { + percentage := float64(count) / float64(len(b.results)) * 100 + fmt.Printf(" %s: %d (%.1f%%)\n", recType, count, percentage) + } + } + fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + } + + return nil +} diff --git a/pkg/runner/outputter/output/buffer.go b/pkg/runner/outputter/output/buffer.go new file mode 100755 index 00000000..586979f2 --- /dev/null +++ b/pkg/runner/outputter/output/buffer.go @@ -0,0 +1,36 @@ +package output + +import ( + "strings" + + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +type BuffOutput struct { + sb strings.Builder +} + +func NewBuffOutput() (*BuffOutput, error) { + s := &BuffOutput{} + s.sb = strings.Builder{} + return s, nil +} + +func (b *BuffOutput) WriteDomainResult(domain result.Result) error { + var domains []string = []string{domain.Subdomain} + for _, item := range domain.Answers { + domains = append(domains, item) + } + msg := strings.Join(domains, "=>") + b.sb.WriteString(msg + "\n") + return nil +} + +func (b *BuffOutput) Close() error { + b.sb.Reset() + return nil +} + +func (b *BuffOutput) Strings() string { + return b.sb.String() +} diff --git a/pkg/runner/outputter/output/csv.go b/pkg/runner/outputter/output/csv.go new file mode 100755 index 00000000..6b48de9d --- /dev/null +++ b/pkg/runner/outputter/output/csv.go @@ -0,0 +1,87 @@ +package output + +import ( + "encoding/csv" + "os" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "github.com/boy-hack/ksubdomain/v2/pkg/utils" +) + +type CsvOutput struct { + domains []result.Result + filename string + wildFilterMode string +} + +func NewCsvOutput(filename string, wildFilterMode string) *CsvOutput { + f := new(CsvOutput) + f.domains = make([]result.Result, 0) + f.filename = filename + f.wildFilterMode = wildFilterMode + return f +} + +func (f *CsvOutput) WriteDomainResult(domain result.Result) error { + f.domains = append(f.domains, domain) + return nil +} + +func (f *CsvOutput) Close() error { + gologger.Infof("写入csv文件:%s\n", f.filename) + + // 检查结果数量 + if len(f.domains) == 0 { + gologger.Infof("没有发现子域名结果,CSV文件将为空\n") + return nil + } + + results := utils.WildFilterOutputResult(f.wildFilterMode, f.domains) + gologger.Infof("过滤后结果数量: %d\n", len(results)) + + // 检查过滤后结果 + if len(results) == 0 { + gologger.Infof("经过通配符过滤后没有有效结果,CSV文件将为空\n") + return nil + } + + // 创建CSV文件 + file, err := os.Create(f.filename) + if err != nil { + gologger.Errorf("创建CSV文件失败: %v", err) + return err + } + defer file.Close() + + // 创建CSV写入器 + writer := csv.NewWriter(file) + + // 写入CSV头部 + err = writer.Write([]string{"Subdomain", "Answers"}) + if err != nil { + gologger.Errorf("写入CSV头部失败: %v", err) + return err + } + + // 写入数据行 + for _, result := range results { + // 将Answers数组转换为单个字符串,用分号分隔 + answersStr := "" + if len(result.Answers) > 0 { + answersStr = result.Answers[0] + for i := 1; i < len(result.Answers); i++ { + answersStr += ";" + result.Answers[i] + } + } + + err = writer.Write([]string{result.Subdomain, answersStr}) + if err != nil { + gologger.Errorf("写入CSV数据行失败: %v", err) + continue + } + } + writer.Flush() + gologger.Infof("CSV文件写入成功,共写入 %d 条记录", len(results)) + return nil +} diff --git a/pkg/runner/outputter/output/file.go b/pkg/runner/outputter/output/file.go new file mode 100755 index 00000000..bf191668 --- /dev/null +++ b/pkg/runner/outputter/output/file.go @@ -0,0 +1,52 @@ +package output + +import ( + "os" + "strings" + + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + + "github.com/boy-hack/ksubdomain/v2/pkg/utils" +) + +type FileOutPut struct { + output *os.File + wildFilterMode string + domains []result.Result + filename string +} + +func NewPlainOutput(filename string, wildFilterMode string) (*FileOutPut, error) { + output, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664) + if err != nil { + return nil, err + } + f := new(FileOutPut) + f.output = output + f.wildFilterMode = wildFilterMode + f.filename = filename + return f, err +} +func (f *FileOutPut) WriteDomainResult(domain result.Result) error { + var msg string + var domains []string = []string{domain.Subdomain} + for _, item := range domain.Answers { + domains = append(domains, item) + } + msg = strings.Join(domains, "=>") + _, err := f.output.WriteString(msg + "\n") + f.domains = append(f.domains, domain) + return err +} +func (f *FileOutPut) Close() error { + f.output.Close() + results := utils.WildFilterOutputResult(f.wildFilterMode, f.domains) + buf := strings.Builder{} + for _, item := range results { + buf.WriteString(item.Subdomain + "=>") + buf.WriteString(strings.Join(item.Answers, "=>")) + buf.WriteString("\n") + } + err := os.WriteFile(f.filename, []byte(buf.String()), 0664) + return err +} diff --git a/pkg/runner/outputter/output/json.go b/pkg/runner/outputter/output/json.go new file mode 100755 index 00000000..a0c131a4 --- /dev/null +++ b/pkg/runner/outputter/output/json.go @@ -0,0 +1,44 @@ +package output + +import ( + "encoding/json" + "os" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + + "github.com/boy-hack/ksubdomain/v2/pkg/utils" +) + +type JsonOutPut struct { + domains []result.Result + filename string + wildFilterMode string +} + +func NewJsonOutput(filename string, wildFilterMode string) *JsonOutPut { + f := new(JsonOutPut) + f.domains = make([]result.Result, 0) + f.filename = filename + f.wildFilterMode = wildFilterMode + return f +} + +func (f *JsonOutPut) WriteDomainResult(domain result.Result) error { + f.domains = append(f.domains, domain) + return nil +} + +func (f *JsonOutPut) Close() error { + gologger.Infof("写入json文件:%s count:%d", f.filename, len(f.domains)) + if len(f.domains) > 0 { + results := utils.WildFilterOutputResult(f.wildFilterMode, f.domains) + jsonBytes, err := json.Marshal(results) + if err != nil { + return err + } + err = os.WriteFile(f.filename, jsonBytes, 0664) + return err + } + return nil +} diff --git a/pkg/runner/outputter/output/jsonl.go b/pkg/runner/outputter/output/jsonl.go new file mode 100644 index 00000000..b4726acb --- /dev/null +++ b/pkg/runner/outputter/output/jsonl.go @@ -0,0 +1,120 @@ +package output + +import ( + "encoding/json" + "os" + "sync" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +// JSONLOutput JSONL (JSON Lines) 输出器 +// 每行一个 JSON 对象,便于流式处理和工具链集成 +// 格式: {"domain":"example.com","type":"A","records":["1.2.3.4"],"timestamp":1234567890} +type JSONLOutput struct { + filename string + file *os.File + mu sync.Mutex +} + +// JSONLRecord JSONL 记录格式 +type JSONLRecord struct { + Domain string `json:"domain"` // 子域名 + Type string `json:"type"` // 记录类型 (A, CNAME, NS, etc.) + Records []string `json:"records"` // 记录值列表 + Timestamp int64 `json:"timestamp"` // Unix 时间戳 + TTL uint32 `json:"ttl,omitempty"` // TTL (可选) + Source string `json:"source,omitempty"` // 数据来源 (可选) +} + +// NewJSONLOutput 创建 JSONL 输出器 +func NewJSONLOutput(filename string) (*JSONLOutput, error) { + file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + if err != nil { + return nil, err + } + + gologger.Infof("JSONL output file: %s\n", filename) + + return &JSONLOutput{ + filename: filename, + file: file, + }, nil +} + +// WriteDomainResult 写入单个域名结果 +// JSONL 格式每次写入一行 JSON,立即刷新 +// 优点: 支持流式处理,可以实时读取 +func (j *JSONLOutput) WriteDomainResult(r result.Result) error { + j.mu.Lock() + defer j.mu.Unlock() + + // 解析记录类型 + recordType := "A" // 默认 A 记录 + records := make([]string, 0, len(r.Answers)) + + for _, answer := range r.Answers { + // 解析类型 (CNAME, NS, PTR 等) + if len(answer) > 0 { + // 检查是否为特殊记录类型 + if len(answer) > 6 && answer[:6] == "CNAME " { + recordType = "CNAME" + records = append(records, answer[6:]) // 去掉 "CNAME " 前缀 + } else if len(answer) > 3 && answer[:3] == "NS " { + recordType = "NS" + records = append(records, answer[3:]) + } else if len(answer) > 4 && answer[:4] == "PTR " { + recordType = "PTR" + records = append(records, answer[4:]) + } else if len(answer) > 4 && answer[:4] == "TXT " { + recordType = "TXT" + records = append(records, answer[4:]) + } else { + // IP 地址 (A 或 AAAA 记录) + records = append(records, answer) + } + } + } + + // 如果没有解析出记录,使用原始 answers + if len(records) == 0 { + records = r.Answers + } + + // 构造 JSONL 记录 + record := JSONLRecord{ + Domain: r.Subdomain, + Type: recordType, + Records: records, + Timestamp: time.Now().Unix(), + } + + // 序列化为 JSON + data, err := json.Marshal(record) + if err != nil { + return err + } + + // 写入一行 (JSON + 换行符) + _, err = j.file.Write(append(data, '\n')) + if err != nil { + return err + } + + // 立即刷新到磁盘 (支持实时读取) + return j.file.Sync() +} + +// Close 关闭 JSONL 输出器 +func (j *JSONLOutput) Close() error { + j.mu.Lock() + defer j.mu.Unlock() + + if j.file != nil { + gologger.Infof("JSONL output completed: %s\n", j.filename) + return j.file.Close() + } + return nil +} diff --git a/pkg/runner/outputter/output/screen.go b/pkg/runner/outputter/output/screen.go new file mode 100755 index 00000000..598a4223 --- /dev/null +++ b/pkg/runner/outputter/output/screen.go @@ -0,0 +1,58 @@ +package output + +import ( + "strings" + + "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +type ScreenOutput struct { + windowsWidth int + silent bool + onlyDomain bool // 修复 Issue #67: 只输出域名 +} + +// NewScreenOutput 创建屏幕输出器 +// 修复 Issue #67: 支持 onlyDomain 参数 +func NewScreenOutput(silent bool, onlyDomain ...bool) (*ScreenOutput, error) { + windowsWidth := core.GetWindowWith() + s := new(ScreenOutput) + s.windowsWidth = windowsWidth + s.silent = silent + // 支持可选的 onlyDomain 参数 (向后兼容) + if len(onlyDomain) > 0 { + s.onlyDomain = onlyDomain[0] + } + return s, nil +} + +func (s *ScreenOutput) WriteDomainResult(domain result.Result) error { + var msg string + + // 修复 Issue #67: 支持只输出域名模式 + if s.onlyDomain { + // 只输出域名,不显示 IP 和其他记录 + msg = domain.Subdomain + } else { + // 完整输出: 域名 => 记录1 => 记录2 + var domains []string = []string{domain.Subdomain} + for _, item := range domain.Answers { + domains = append(domains, item) + } + msg = strings.Join(domains, " => ") + } + + if !s.silent { + screenWidth := s.windowsWidth - len(msg) - 1 + gologger.Silentf("\r%s% *s\n", msg, screenWidth, "") + } else { + gologger.Silentf("\r%s\n", msg) + } + return nil +} + +func (s *ScreenOutput) Close() error { + return nil +} diff --git a/pkg/runner/outputter/output/screen_no_width.go b/pkg/runner/outputter/output/screen_no_width.go new file mode 100755 index 00000000..1c557c2f --- /dev/null +++ b/pkg/runner/outputter/output/screen_no_width.go @@ -0,0 +1,33 @@ +package output + +import ( + "strings" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +type ScreenOutputNoWidth struct { + silent bool +} + +func NewScreenOutputNoWidth(silent bool) (*ScreenOutputNoWidth, error) { + return &ScreenOutputNoWidth{silent: silent}, nil +} +func (s *ScreenOutputNoWidth) WriteDomainResult(domain result.Result) error { + var msg string + var domains []string = []string{domain.Subdomain} + for _, item := range domain.Answers { + domains = append(domains, item) + } + msg = strings.Join(domains, " => ") + if !s.silent { + gologger.Silentf("%s\n", msg) + } else { + gologger.Silentf("%s\n", domain.Subdomain) + } + return nil +} +func (s *ScreenOutputNoWidth) Close() error { + return nil +} diff --git a/pkg/runner/processbar/fake.go b/pkg/runner/processbar/fake.go new file mode 100755 index 00000000..66237d71 --- /dev/null +++ b/pkg/runner/processbar/fake.go @@ -0,0 +1,12 @@ +package processbar + +type FakeScreenProcess struct { +} + +func (s *FakeScreenProcess) WriteData(data *ProcessData) { + +} + +func (s *FakeScreenProcess) Close() { + +} diff --git a/pkg/runner/processbar/processbar.go b/pkg/runner/processbar/processbar.go new file mode 100755 index 00000000..fe2c35ce --- /dev/null +++ b/pkg/runner/processbar/processbar.go @@ -0,0 +1,14 @@ +package processbar + +type ProcessData struct { + SuccessIndex uint64 + SendIndex uint64 + QueueLength int64 + RecvIndex uint64 + FaildIndex uint64 + Elapsed int +} +type ProcessBar interface { + WriteData(data *ProcessData) + Close() +} diff --git a/pkg/runner/processbar/screen.go b/pkg/runner/processbar/screen.go new file mode 100755 index 00000000..22857dcc --- /dev/null +++ b/pkg/runner/processbar/screen.go @@ -0,0 +1,16 @@ +package processbar + +import "fmt" + +type ScreenProcess struct { + Silent bool +} + +func (s *ScreenProcess) WriteData(data *ProcessData) { + if !s.Silent { + fmt.Printf("\rSuccess:%d Send:%d Queue:%d Accept:%d Fail:%d Elapsed:%ds", data.SuccessIndex, data.SendIndex, data.QueueLength, data.RecvIndex, data.FaildIndex, data.Elapsed) + } +} + +func (s *ScreenProcess) Close() { +} diff --git a/pkg/runner/recv.go b/pkg/runner/recv.go new file mode 100755 index 00000000..9557028b --- /dev/null +++ b/pkg/runner/recv.go @@ -0,0 +1,346 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" +) + +// parseDNSName 解析 DNS 域名格式 +// DNS 域名格式: 长度前缀 + 标签 + ... + 结束符 +// 例如: \x03www\x06google\x03com\x00 表示 www.google.com +// 修复 Issue #70: 正确解析 CNAME/NS/PTR 等记录,避免出现 "comcom" 等拼接错误 +func parseDNSName(raw []byte) string { + if len(raw) == 0 { + return "" + } + + var result []byte + i := 0 + + for i < len(raw) { + // 读取标签长度 + length := int(raw[i]) + + // 0x00 表示域名结束 + if length == 0 { + break + } + + // 0xC0 开头表示压缩指针 (RFC 1035) + // 压缩格式: 前2位为11,后14位为偏移量 + if length >= 0xC0 { + // 压缩指针,暂不处理(通常在完整DNS包中才有) + break + } + + // 添加点分隔符 (第一个标签除外) + if len(result) > 0 { + result = append(result, '.') + } + + i++ + + // 防止越界 + if i+length > len(raw) { + break + } + + // 添加标签内容 + result = append(result, raw[i:i+length]...) + i += length + } + + return string(result) +} + +// dnsRecord2String 将DNS记录转换为字符串 +// 修复 Issue #70: 使用 parseDNSName 正确解析域名格式 +func dnsRecord2String(rr layers.DNSResourceRecord) (string, error) { + if rr.Class == layers.DNSClassIN { + switch rr.Type { + case layers.DNSTypeA, layers.DNSTypeAAAA: + if rr.IP != nil { + return rr.IP.String(), nil + } + case layers.DNSTypeNS: + if rr.NS != nil { + // 修复: 使用 parseDNSName 解析 NS 记录 + ns := parseDNSName(rr.NS) + if ns != "" { + return "NS " + ns, nil + } + } + case layers.DNSTypeCNAME: + if rr.CNAME != nil { + // 修复: 使用 parseDNSName 解析 CNAME 记录 + cname := parseDNSName(rr.CNAME) + if cname != "" { + return "CNAME " + cname, nil + } + } + case layers.DNSTypePTR: + if rr.PTR != nil { + // 修复: 使用 parseDNSName 解析 PTR 记录 + ptr := parseDNSName(rr.PTR) + if ptr != "" { + return "PTR " + ptr, nil + } + } + case layers.DNSTypeTXT: + if rr.TXT != nil { + // TXT 记录是纯文本,不需要解析 + return "TXT " + string(rr.TXT), nil + } + } + } + return "", errors.New("dns record error") +} + +// 预分配解码器对象池,避免频繁创建 +var decoderPool = sync.Pool{ + New: func() interface{} { + var eth layers.Ethernet + var ipv4 layers.IPv4 + var ipv6 layers.IPv6 + var udp layers.UDP + var dns layers.DNS + parser := gopacket.NewDecodingLayerParser( + layers.LayerTypeEthernet, ð, &ipv4, &ipv6, &udp, &dns) + + return &decodingContext{ + parser: parser, + eth: ð, + ipv4: &ipv4, + ipv6: &ipv6, + udp: &udp, + dns: &dns, + decoded: make([]gopacket.LayerType, 0, 5), + } + }, +} + +// decodingContext 解码上下文 +type decodingContext struct { + parser *gopacket.DecodingLayerParser + eth *layers.Ethernet + ipv4 *layers.IPv4 + ipv6 *layers.IPv6 + udp *layers.UDP + dns *layers.DNS + decoded []gopacket.LayerType +} + +// 解析DNS响应包并处理 +func (r *Runner) processPacket(data []byte, dnsChanel chan<- layers.DNS) { + // 从对象池获取解码器 + dc := decoderPool.Get().(*decodingContext) + defer decoderPool.Put(dc) + + // 清空解码层类型切片 + dc.decoded = dc.decoded[:0] + + // 解析数据包 + err := dc.parser.DecodeLayers(data, &dc.decoded) + if err != nil { + return + } + + // 检查是否为DNS响应 + if !dc.dns.QR { + return + } + + // 确认DNS ID匹配 + if dc.dns.ID != r.dnsID { + return + } + + // 确认有查询问题 + if len(dc.dns.Questions) == 0 { + return + } + + // 记录接收包数量 + atomic.AddUint64(&r.receiveCount, 1) + + // 向处理通道发送DNS响应 + select { + case dnsChanel <- *dc.dns: + } +} + +// recvChanel 实现接收DNS响应的功能 +func (r *Runner) recvChanel(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + var ( + snapshotLen = 65536 + timeout = 5 * time.Second + err error + ) + inactive, err := pcap.NewInactiveHandle(r.options.EtherInfo.Device) + if err != nil { + gologger.Errorf("创建网络捕获句柄失败: %v", err) + return + } + err = inactive.SetSnapLen(snapshotLen) + if err != nil { + gologger.Errorf("设置抓包长度失败: %v", err) + return + } + defer inactive.CleanUp() + + if err = inactive.SetTimeout(timeout); err != nil { + gologger.Errorf("设置超时失败: %v", err) + return + } + + err = inactive.SetImmediateMode(true) + if err != nil { + gologger.Errorf("设置即时模式失败: %v", err) + return + } + + handle, err := inactive.Activate() + if err != nil { + gologger.Errorf("激活网络捕获失败: %v", err) + return + } + defer handle.Close() + + err = handle.SetBPFFilter(fmt.Sprintf("udp and src port 53 and dst port %d", r.listenPort)) + if err != nil { + gologger.Errorf("设置BPF过滤器失败: %v", err) + return + } + + // 创建DNS响应处理通道,缓冲大小适当增加 + dnsChanel := make(chan layers.DNS, 10000) + + // 使用多个协程处理DNS响应,提高并发效率 + processorCount := runtime.NumCPU() * 2 + var processorWg sync.WaitGroup + processorWg.Add(processorCount) + + // 启动多个处理协程 + for i := 0; i < processorCount; i++ { + go func() { + defer processorWg.Done() + for { + select { + case <-ctx.Done(): + return + case dns, ok := <-dnsChanel: + if !ok { + return + } + + subdomain := string(dns.Questions[0].Name) + + // 计算RTT并上报给动态超时追踪器 + if item, ok := r.statusDB.Get(subdomain); ok { + rttSec := time.Since(item.Time).Seconds() + if rttSec > 0 { + r.rttTracker.recordSample(rttSec) + } + } + + r.statusDB.Del(subdomain) + if dns.ANCount > 0 { + atomic.AddUint64(&r.successCount, 1) + var answers []string + for _, v := range dns.Answers { + answer, err := dnsRecord2String(v) + if err != nil { + continue + } + answers = append(answers, answer) + } + r.resultChan <- result.Result{ + Subdomain: subdomain, + Answers: answers, + } + } + } + } + }() + } + + // 使用多个接收协程读取网络数据包 + // 背压阈值:packetChan 占用超过 80% 时触发降速信号 + const packetChanCap = 10000 + const backpressureHighWatermark = int(packetChanCap * 0.8) // 8000:触发背压 + const backpressureLowWatermark = int(packetChanCap * 0.5) // 5000:恢复正常 + packetChan := make(chan []byte, packetChanCap) + + // 启动数据包接收协程 + go func() { + for { + data, _, err := handle.ReadPacketData() + if err != nil { + if errors.Is(err, pcap.NextErrorTimeoutExpired) { + continue + } + return + } + + // 检测 packetChan 占用率,更新背压标志 + qLen := len(packetChan) + if qLen >= backpressureHighWatermark { + atomic.StoreInt32(&r.recvBackpressure, 1) + } else if qLen <= backpressureLowWatermark { + atomic.StoreInt32(&r.recvBackpressure, 0) + } + + select { + case <-ctx.Done(): + return + case packetChan <- data: + // 数据包已发送到处理通道 + } + } + }() + + // 启动多个数据包解析协程 + parserCount := runtime.NumCPU() * 2 + var parserWg sync.WaitGroup + parserWg.Add(parserCount) + + for i := 0; i < parserCount; i++ { + go func() { + defer parserWg.Done() + for { + select { + case <-ctx.Done(): + return + case data, ok := <-packetChan: + if !ok { + return + } + r.processPacket(data, dnsChanel) + } + } + }() + } + + // 等待上下文结束 + <-ctx.Done() + + // 关闭通道 + close(packetChan) + close(dnsChanel) + + // 等待所有处理和解析协程结束 + parserWg.Wait() + processorWg.Wait() +} diff --git a/pkg/runner/recv_test.go b/pkg/runner/recv_test.go new file mode 100644 index 00000000..f825245f --- /dev/null +++ b/pkg/runner/recv_test.go @@ -0,0 +1,296 @@ +package runner + +import ( + "testing" + + "github.com/google/gopacket/layers" + "github.com/stretchr/testify/assert" +) + +// TestParseDNSName 测试 DNS 域名格式解析 +// 修复 Issue #70 的核心函数 +func TestParseDNSName(t *testing.T) { + tests := []struct { + name string + input []byte + expected string + }{ + { + name: "标准域名 - www.google.com", + input: []byte{ + 3, 'w', 'w', 'w', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + 0, // 结束符 + }, + expected: "www.google.com", + }, + { + name: "二级域名 - baidu.com", + input: []byte{ + 5, 'b', 'a', 'i', 'd', 'u', + 3, 'c', 'o', 'm', + 0, + }, + expected: "baidu.com", + }, + { + name: "三级域名 - mail.qq.com", + input: []byte{ + 4, 'm', 'a', 'i', 'l', + 2, 'q', 'q', + 3, 'c', 'o', 'm', + 0, + }, + expected: "mail.qq.com", + }, + { + name: "空输入", + input: []byte{}, + expected: "", + }, + { + name: "仅结束符", + input: []byte{0}, + expected: "", + }, + { + name: "无结束符域名", + input: []byte{ + 3, 'w', 'w', 'w', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + }, + expected: "www.google.com", + }, + { + name: "长域名", + input: []byte{ + 10, 's', 'u', 'b', 'd', 'o', 'm', 'a', 'i', 'n', '1', + 10, 's', 'u', 'b', 'd', 'o', 'm', 'a', 'i', 'n', '2', + 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + expected: "subdomain1.subdomain2.example.com", + }, + { + name: "压缩指针 (0xC0) - 应该停止", + input: []byte{ + 3, 'w', 'w', 'w', + 0xC0, 0x12, // 压缩指针 + }, + expected: "www", + }, + { + name: "异常长度 - 超出范围", + input: []byte{ + 100, 'a', 'b', 'c', // 长度100但数据不足 + }, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := parseDNSName(tt.input) + assert.Equal(t, tt.expected, result, "DNS 域名解析结果不匹配") + }) + } +} + +// TestDNSRecord2String_CNAME 测试 CNAME 记录转换 +func TestDNSRecord2String_CNAME(t *testing.T) { + tests := []struct { + name string + rr layers.DNSResourceRecord + expected string + hasError bool + }{ + { + name: "标准 CNAME", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeCNAME, + Class: layers.DNSClassIN, + CNAME: []byte{ + 3, 'w', 'w', 'w', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + }, + expected: "CNAME www.google.com", + hasError: false, + }, + { + name: "CNAME - cdn.example.com", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeCNAME, + Class: layers.DNSClassIN, + CNAME: []byte{ + 3, 'c', 'd', 'n', + 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + }, + expected: "CNAME cdn.example.com", + hasError: false, + }, + { + name: "空 CNAME", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeCNAME, + Class: layers.DNSClassIN, + CNAME: nil, + }, + expected: "", + hasError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := dnsRecord2String(tt.rr) + if tt.hasError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + } + }) + } +} + +// TestDNSRecord2String_NS 测试 NS 记录转换 +func TestDNSRecord2String_NS(t *testing.T) { + tests := []struct { + name string + rr layers.DNSResourceRecord + expected string + }{ + { + name: "标准 NS", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeNS, + Class: layers.DNSClassIN, + NS: []byte{ + 3, 'n', 's', '1', + 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + }, + expected: "NS ns1.example.com", + }, + { + name: "NS - dns.google.com", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeNS, + Class: layers.DNSClassIN, + NS: []byte{ + 3, 'd', 'n', 's', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + }, + expected: "NS dns.google.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := dnsRecord2String(tt.rr) + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestDNSRecord2String_A 测试 A 记录转换 +func TestDNSRecord2String_A(t *testing.T) { + tests := []struct { + name string + rr layers.DNSResourceRecord + expected string + }{ + { + name: "标准 A 记录", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeA, + Class: layers.DNSClassIN, + IP: []byte{192, 168, 1, 1}, + }, + expected: "192.168.1.1", + }, + { + name: "公网 IP", + rr: layers.DNSResourceRecord{ + Type: layers.DNSTypeA, + Class: layers.DNSClassIN, + IP: []byte{8, 8, 8, 8}, + }, + expected: "8.8.8.8", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := dnsRecord2String(tt.rr) + assert.NoError(t, err) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestDNSRecord2String_PTR 测试 PTR 记录转换 +func TestDNSRecord2String_PTR(t *testing.T) { + rr := layers.DNSResourceRecord{ + Type: layers.DNSTypePTR, + Class: layers.DNSClassIN, + PTR: []byte{ + 7, 'e', 'x', 'a', 'm', 'p', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + } + + result, err := dnsRecord2String(rr) + assert.NoError(t, err) + assert.Equal(t, "PTR example.com", result) +} + +// BenchmarkParseDNSName 基准测试 DNS 域名解析性能 +func BenchmarkParseDNSName(b *testing.B) { + input := []byte{ + 3, 'w', 'w', 'w', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = parseDNSName(input) + } +} + +// BenchmarkDNSRecord2String 基准测试 DNS 记录转换性能 +func BenchmarkDNSRecord2String(b *testing.B) { + rr := layers.DNSResourceRecord{ + Type: layers.DNSTypeCNAME, + Class: layers.DNSClassIN, + CNAME: []byte{ + 3, 'w', 'w', 'w', + 6, 'g', 'o', 'o', 'g', 'l', 'e', + 3, 'c', 'o', 'm', + 0, + }, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = dnsRecord2String(rr) + } +} diff --git a/pkg/runner/result.go b/pkg/runner/result.go new file mode 100755 index 00000000..f4d227df --- /dev/null +++ b/pkg/runner/result.go @@ -0,0 +1,114 @@ +package runner + +import ( + "context" + "fmt" + "sync" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/predict" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +// handleResult 处理扫描结果 +func (r *Runner) handleResult(predictChan chan string) { + isWildCard := r.options.WildcardFilterMode != "none" + var wg sync.WaitGroup + var predictSignal bool = false + + for res := range r.resultChan { + // 过滤通配符域名 + if isWildCard { + if checkWildIps(r.options.WildIps, res.Answers) { + continue + } + } + + // 将结果写入输出器 + for _, out := range r.options.Writer { + _ = out.WriteDomainResult(res) + } + + // 预测域名处理 + if r.options.Predict { + wg.Add(1) + go func(domain string) { + defer wg.Done() + r.predict(res, predictChan) + if !predictSignal { + r.predictLoadDone <- struct{}{} + predictSignal = true + } + }(res.Subdomain) + } + } + wg.Wait() +} + +// predict 根据已知域名预测新的子域名 +func (r *Runner) predict(res result.Result, predictChan chan string) error { + if r.domainChan == nil { + return fmt.Errorf("域名通道未初始化") + } + _, err := predict.PredictDomains(res.Subdomain, predictChan) + if err != nil { + return err + } + return nil +} + +// handleResultWithContext 处理扫描结果(带有context管理) +func (r *Runner) handleResultWithContext(ctx context.Context, wg *sync.WaitGroup, predictChan chan string) { + defer wg.Done() + isWildCard := r.options.WildcardFilterMode != "none" + var predictWg sync.WaitGroup + var predictSignal bool = false + + for { + select { + case <-ctx.Done(): + predictWg.Wait() + return + case res, ok := <-r.resultChan: + if !ok { + predictWg.Wait() + return + } + // 过滤通配符域名 + if isWildCard { + if checkWildIps(r.options.WildIps, res.Answers) { + continue + } + } + + // 将结果写入输出器 + for _, out := range r.options.Writer { + _ = out.WriteDomainResult(res) + } + + // 预测域名处理 + if r.options.Predict { + predictWg.Add(1) + go func(domain string) { + defer predictWg.Done() + r.predict(res, predictChan) + if !predictSignal { + r.predictLoadDone <- struct{}{} + predictSignal = true + } + }(res.Subdomain) + } + } + } +} + +// checkWildIps 检查是否为通配符IP +func checkWildIps(wildIps []string, ip []string) bool { + for _, w := range wildIps { + for _, i := range ip { + if w == i { + return true + } + } + } + return false +} diff --git a/pkg/runner/result/result.go b/pkg/runner/result/result.go new file mode 100755 index 00000000..12796934 --- /dev/null +++ b/pkg/runner/result/result.go @@ -0,0 +1,6 @@ +package result + +type Result struct { + Subdomain string `json:"subdomain"` + Answers []string `json:"answers"` +} diff --git a/pkg/runner/retry.go b/pkg/runner/retry.go new file mode 100755 index 00000000..764a3233 --- /dev/null +++ b/pkg/runner/retry.go @@ -0,0 +1,114 @@ +package runner + +import ( + "context" + "sync/atomic" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/runner/statusdb" + "github.com/google/gopacket/layers" +) + +// retry 重试机制。 +// +// 设计要点: +// 1. 每 200ms 扫描一次 statusDB,比超时周期更频繁,配合动态超时自适应 +// 2. 空扫描优化:上次为空且队列仍为 0 时跳过,节省 CPU +// 3. 批量重传合并:按 DNS server 分组,对同一 server 的多个域名连续调用 +// send(),减少重复的 selectDNSServer + map lookup 开销 +// 4. 直接调用 send():重传不再通过 domainChan 中转(避免两次 channel 传递), +// 但仍更新 statusDB 中的 Retry/Time 字段保持状态一致 +func (r *Runner) retry(ctx context.Context) { + t := time.NewTicker(200 * time.Millisecond) + defer t.Stop() + + const batchCap = 256 + + // dnsBatches: dns server -> []domain,按 server 分组收集需重传域名 + // 复用 map 以减少 GC + type retryItem struct { + domain string + dns string + } + items := make([]retryItem, 0, batchCap) + + // 按 DNS server 分组的 map,key=dnsServer, value=域名列表 + dnsBatches := make(map[string][]string, 16) + + lastScanEmpty := false + + for { + select { + case <-ctx.Done(): + return + + case <-t.C: + // 空扫描快速跳过 + if lastScanEmpty && r.statusDB.Length() == 0 { + continue + } + + now := time.Now() + items = items[:0] + // 清空分组缓冲(复用已有 key 的 slice) + for k := range dnsBatches { + dnsBatches[k] = dnsBatches[k][:0] + } + + // 扫描 statusDB,收集超时域名并分组 + effectiveTimeout := r.effectiveTimeoutSeconds() + r.statusDB.Scan(func(key string, v statusdb.Item) error { + // 超过最大重试次数则放弃 + if r.maxRetryCount > 0 && v.Retry > r.maxRetryCount { + r.statusDB.Del(key) + atomic.AddUint64(&r.failedCount, 1) + return nil + } + + // 检查是否超时 + if int64(now.Sub(v.Time).Seconds()) < effectiveTimeout { + return nil + } + + dns := r.selectDNSServer(key) + items = append(items, retryItem{domain: key, dns: dns}) + + if dnsBatches[dns] == nil { + dnsBatches[dns] = make([]string, 0, 32) + } + dnsBatches[dns] = append(dnsBatches[dns], key) + return nil + }) + + lastScanEmpty = len(items) == 0 + if lastScanEmpty { + continue + } + + // 更新 statusDB:批量更新 Retry/Time,然后按 DNS server 分组批量发包 + // 先更新状态,再发包,保证 statusDB 状态一致 + for _, item := range items { + v, ok := r.statusDB.Get(item.domain) + if !ok { + continue // 可能已被 recv 侧删除,跳过 + } + v.Retry += 1 + v.Time = time.Now() + v.Dns = item.dns + r.statusDB.Set(item.domain, v) + } + + // 按 DNS server 分组批量调用 send() + // 同一 server 的域名连续发送,减少 pcap handle 竞争和函数调用开销 + for dns, domains := range dnsBatches { + if len(domains) == 0 { + continue + } + for _, domain := range domains { + send(domain, dns, r.options.EtherInfo, r.dnsID, + uint16(r.listenPort), r.pcapHandle, layers.DNSTypeA) + } + } + } + } +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go new file mode 100755 index 00000000..a7ea86d3 --- /dev/null +++ b/pkg/runner/runner.go @@ -0,0 +1,372 @@ +package runner + +import ( + "context" + "math" + "math/rand" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/device" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/statusdb" + "github.com/google/gopacket/pcap" + "github.com/phayes/freeport" + "go.uber.org/ratelimit" +) + +// Runner 表示子域名扫描的运行时结构 +type Runner struct { + statusDB *statusdb.StatusDb // 状态数据库 + options *options.Options // 配置选项 + rateLimiter ratelimit.Limiter // 速率限制器 + pcapHandle *pcap.Handle // 网络抓包句柄 + successCount uint64 // 成功数量 + sendCount uint64 // 发送数量 + receiveCount uint64 // 接收数量 + failedCount uint64 // 失败数量 + domainChan chan string // 域名发送通道 + resultChan chan result.Result // 结果接收通道 + listenPort int // 监听端口 + dnsID uint16 // DNS请求ID + maxRetryCount int // 最大重试次数 + initialLoadDone chan struct{} // 初始加载完成信号 + predictLoadDone chan struct{} // predict加载完成信号 + startTime time.Time // 开始时间 + stopSignal chan struct{} // 停止信号 + rttTracker *rttSlidingWindow // RTT滑动均值追踪器(始终启用) + recvBackpressure int32 // 接收侧背压标志:1=需要降速,0=正常(atomic) +} + +// rttSlidingWindow 基于指数加权移动平均(EWMA)计算RTT滑动均值。 +// +// 算法说明: +// - 使用 alpha=0.125(即 1/8),与 TCP RFC 6298 保持一致 +// - smoothedRTT = (1-alpha)*smoothedRTT + alpha*sample +// - rttVar = (1-beta)*rttVar + beta*|sample-smoothedRTT| (beta=0.25) +// - dynamicTimeout = smoothedRTT + 4*rttVar(TCP RTO 公式) +// - 上下界:[minTimeout=1s, maxTimeout=rttMaxTimeoutSeconds] +// +// 线程安全:所有字段通过 mu 保护。 +type rttSlidingWindow struct { + mu sync.Mutex + smoothedRTT float64 // 单位:秒(EWMA) + rttVar float64 // RTT 方差(EWMA) + sampleCount int64 // 已采样数量 + minTimeout float64 // 动态超时下界(秒) + maxTimeout float64 // 动态超时上界(秒) +} + +const ( + rttAlpha = 0.125 // EWMA平滑系数(TCP RFC 6298) + rttBeta = 0.25 // 方差平滑系数 + rttMaxTimeoutSeconds = 10.0 // 动态超时上界(秒),内部固定,不对外暴露 + rttMinTimeoutSeconds = 1.0 // 动态超时下界(秒) +) + +// newRTTSlidingWindow 创建RTT追踪器,上界固定为 rttMaxTimeoutSeconds。 +func newRTTSlidingWindow() *rttSlidingWindow { + return &rttSlidingWindow{ + // 初始 smoothedRTT 设为上界/2,避免冷启动时过早丢弃域名 + smoothedRTT: rttMaxTimeoutSeconds / 2, + rttVar: rttMaxTimeoutSeconds / 4, + minTimeout: rttMinTimeoutSeconds, + maxTimeout: rttMaxTimeoutSeconds, + } +} + +// recordSample 记录一次RTT样本(单位:秒)。 +func (w *rttSlidingWindow) recordSample(rttSec float64) { + w.mu.Lock() + defer w.mu.Unlock() + + if w.sampleCount == 0 { + // 第一个样本:直接初始化 + w.smoothedRTT = rttSec + w.rttVar = rttSec / 2 + } else { + // RFC 6298 更新公式 + diff := rttSec - w.smoothedRTT + if diff < 0 { + diff = -diff + } + w.rttVar = (1-rttBeta)*w.rttVar + rttBeta*diff + w.smoothedRTT = (1-rttAlpha)*w.smoothedRTT + rttAlpha*rttSec + } + atomic.AddInt64(&w.sampleCount, 1) +} + +// dynamicTimeoutSeconds 返回当前动态超时(秒,整数,向上取整)。 +// 公式:smoothedRTT + 4*rttVar,限制在 [minTimeout, maxTimeout] 范围内。 +func (w *rttSlidingWindow) dynamicTimeoutSeconds() int64 { + w.mu.Lock() + defer w.mu.Unlock() + + timeout := w.smoothedRTT + 4*w.rttVar + if timeout < w.minTimeout { + timeout = w.minTimeout + } + if timeout > w.maxTimeout { + timeout = w.maxTimeout + } + // 向上取整,最少 1 秒 + result := int64(math.Ceil(timeout)) + if result < 1 { + result = 1 + } + return result +} + +func init() { + rand.New(rand.NewSource(time.Now().UnixNano())) +} + +// New 创建一个新的Runner实例 +func New(opt *options.Options) (*Runner, error) { + var err error + version := pcap.Version() + r := new(Runner) + gologger.Infof(version) + r.options = opt + r.statusDB = statusdb.CreateMemoryDB() + + // 记录DNS服务器信息 + gologger.Infof("默认DNS服务器: %s\n", core.SliceToString(opt.Resolvers)) + if len(opt.SpecialResolvers) > 0 { + var keys []string + for k := range opt.SpecialResolvers { + keys = append(keys, k) + } + gologger.Infof("特殊DNS服务器: %s\n", core.SliceToString(keys)) + } + + // 初始化网络设备 + r.pcapHandle, err = device.PcapInit(opt.EtherInfo.Device) + if err != nil { + return nil, err + } + + // 设置速率限制 + cpuLimit := float64(runtime.NumCPU() * 10000) + rateLimit := int(math.Min(cpuLimit, float64(opt.Rate))) + + // Mac 平台优化: BPF 缓冲区限制较严格 + // 建议速率 < 50000 pps 以避免缓冲区溢出 + if runtime.GOOS == "darwin" && rateLimit > 50000 { + gologger.Warningf("Mac 平台检测到: 当前速率 %d pps 可能导致缓冲区问题\n", rateLimit) + gologger.Warningf("建议: 使用 -b 参数限制带宽 (如 -b 5m) 或降低速率\n") + gologger.Warningf("提示: Mac BPF 缓冲区已优化至 2MB,但仍建议速率 < 50000 pps\n") + } + + r.rateLimiter = ratelimit.New(rateLimit) + gologger.Infof("速率限制: %d pps\n", rateLimit) + + // 初始化通道 + r.domainChan = make(chan string, 50000) + r.resultChan = make(chan result.Result, 5000) + r.stopSignal = make(chan struct{}) + + // 获取空闲端口 + freePort, err := freeport.GetFreePort() + if err != nil { + return nil, err + } + r.listenPort = freePort + gologger.Infof("监听端口: %d\n", freePort) + + // 设置其他参数 + r.dnsID = 0x2021 // ksubdomain的生日 + r.maxRetryCount = opt.Retry + r.initialLoadDone = make(chan struct{}) + r.predictLoadDone = make(chan struct{}) + r.startTime = time.Now() + + // 始终启用动态超时,上界内部固定为 rttMaxTimeoutSeconds + r.rttTracker = newRTTSlidingWindow() + + return r, nil +} + +// effectiveTimeoutSeconds 返回当前有效的超时秒数。 +// 有样本时使用 EWMA 动态计算值,冷启动(0 样本)时使用初始估算值。 +func (r *Runner) effectiveTimeoutSeconds() int64 { + return r.rttTracker.dynamicTimeoutSeconds() +} + +// selectDNSServer 根据域名智能选择DNS服务器 +func (r *Runner) selectDNSServer(domain string) string { + dnsServers := r.options.Resolvers + specialDNSServers := r.options.SpecialResolvers + + // 根据域名后缀选择特定DNS服务器 + if len(specialDNSServers) > 0 { + for suffix, servers := range specialDNSServers { + if strings.HasSuffix(domain, suffix) { + dnsServers = servers + break + } + } + } + + // 随机选择一个DNS服务器 + idx := getRandomIndex() % len(dnsServers) + return dnsServers[idx] +} + +// getRandomIndex 获取随机索引 +func getRandomIndex() int { + return int(rand.Int31()) +} + +// updateStatusBar 更新进度条状态 +func (r *Runner) updateStatusBar() { + if r.options.ProcessBar != nil { + queueLength := r.statusDB.Length() + elapsedSeconds := int(time.Since(r.startTime).Seconds()) + data := &processbar.ProcessData{ + SuccessIndex: r.successCount, + SendIndex: r.sendCount, + QueueLength: queueLength, + RecvIndex: r.receiveCount, + FaildIndex: r.failedCount, + Elapsed: elapsedSeconds, + } + r.options.ProcessBar.WriteData(data) + } +} + +// loadDomainsFromSource 从源加载域名 +func (r *Runner) loadDomainsFromSource(wg *sync.WaitGroup) { + defer wg.Done() + // 从域名源加载域名 + for domain := range r.options.Domain { + r.domainChan <- domain + } + // 通知初始加载完成 + r.initialLoadDone <- struct{}{} +} + +// monitorProgress 监控扫描进度 +func (r *Runner) monitorProgress(ctx context.Context, cancelFunc context.CancelFunc, wg *sync.WaitGroup) { + var initialLoadCompleted bool = false + var initialLoadPredict bool = false + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + defer wg.Done() + for { + select { + case <-ticker.C: + // 更新状态栏 + r.updateStatusBar() + // 检查是否完成 + if initialLoadCompleted && initialLoadPredict { + queueLength := r.statusDB.Length() + if queueLength <= 0 { + gologger.Printf("\n") + gologger.Infof("扫描完毕") + cancelFunc() // 使用传递的cancelFunc + return + } + } + case <-r.initialLoadDone: + // 初始加载完成后启动重试机制 + go r.retry(ctx) + initialLoadCompleted = true + case <-r.predictLoadDone: + initialLoadPredict = true + case <-ctx.Done(): + return + } + } +} + +// processPredictedDomains 处理预测的域名 +func (r *Runner) processPredictedDomains(ctx context.Context, wg *sync.WaitGroup, predictChan chan string) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + case domain := <-predictChan: + r.domainChan <- domain + } + } +} + +// RunEnumeration 开始子域名枚举过程 +func (r *Runner) RunEnumeration(ctx context.Context) { + // 创建可取消的上下文 + ctx, cancelFunc := context.WithCancel(ctx) + defer cancelFunc() + + // 创建等待组,现在需要等待5个goroutine(添加了sendCycle和handleResult) + wg := &sync.WaitGroup{} + wg.Add(5) + + // 启动接收处理 + go r.recvChanel(ctx, wg) + + // 启动发送处理(加入waitgroup管理) + go r.sendCycleWithContext(ctx, wg) + + // 监控进度 + go r.monitorProgress(ctx, cancelFunc, wg) + + // 创建预测域名通道 + predictChan := make(chan string, 1000) + if r.options.Predict { + wg.Add(1) + // 启动预测域名处理 + go r.processPredictedDomains(ctx, wg, predictChan) + } else { + r.predictLoadDone <- struct{}{} + } + + // 启动结果处理(加入waitgroup管理) + go r.handleResultWithContext(ctx, wg, predictChan) + + // 从源加载域名 + go r.loadDomainsFromSource(wg) + + // 等待所有协程完成 + wg.Wait() + + // 关闭所有通道 + close(predictChan) + // 安全关闭通道 + close(r.resultChan) + close(r.domainChan) +} + +// Close 关闭Runner并释放资源 +func (r *Runner) Close() { + // 关闭网络抓包句柄 + if r.pcapHandle != nil { + r.pcapHandle.Close() + } + + // 关闭状态数据库 + if r.statusDB != nil { + r.statusDB.Close() + } + + // 关闭所有输出器 + for _, out := range r.options.Writer { + err := out.Close() + if err != nil { + gologger.Errorf("关闭输出器失败: %v", err) + } + } + + // 关闭进度条 + if r.options.ProcessBar != nil { + r.options.ProcessBar.Close() + } +} diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go new file mode 100755 index 00000000..93982a0e --- /dev/null +++ b/pkg/runner/runner_test.go @@ -0,0 +1,153 @@ +package runner + +import ( + "context" + "testing" + + "github.com/boy-hack/ksubdomain/v2/pkg/core" + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/device" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/stretchr/testify/assert" +) + +func TestV(t *testing.T) { + for i := 0; i < 2; i++ { + domainChanel := make(chan string) + eth := options.GetDeviceConfig([]string{"114.114.114.114"}) + domains := []string{"stu.baidu.com", "www.baidu.com"} + go func() { + for _, d := range domains { + domainChanel <- d + } + close(domainChanel) + }() + w, _ := output.NewScreenOutput(true) + opt := &options.Options{ + Rate: options.Band2Rate("1m"), + Domain: domainChanel, + Resolvers: options.GetResolvers(nil), + Silent: true, + Retry: 1, + Method: options.VerifyType, + Writer: []outputter.Output{ + w, + }, + EtherInfo: eth, + } + opt.Check() + r, err := New(opt) + assert.NoError(t, err) + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() + } +} +func TestVerify(t *testing.T) { + process := processbar2.FakeScreenProcess{} + screenPrinter, _ := output.NewScreenOutputNoWidth(false) + domains := []string{"stu.baidu.com", "haokan.baidu.com"} + domainChanel := make(chan string) + eth, err := device.AutoGetDevices(nil) + assert.NoError(t, err) + + go func() { + for _, d := range domains { + domainChanel <- d + } + close(domainChanel) + }() + opt := &options.Options{ + Rate: options.Band2Rate("1m"), + Domain: domainChanel, + Resolvers: options.GetResolvers(nil), + Silent: false, + Retry: 1, + Method: options.VerifyType, + Writer: []outputter.Output{ + screenPrinter, + }, + ProcessBar: &process, + EtherInfo: eth, + } + opt.Check() + r, err := New(opt) + assert.NoError(t, err) + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() +} + +func TestEnum(t *testing.T) { + process := processbar2.ScreenProcess{} + screenPrinter, _ := output.NewScreenOutputNoWidth(false) + domains := core.GetDefaultSubdomainData() + domainChanel := make(chan string) + go func() { + for _, d := range domains { + domainChanel <- d + ".baidu.com" + } + close(domainChanel) + }() + eth, err := device.AutoGetDevices(nil) + assert.NoError(t, err) + opt := &options.Options{ + Rate: options.Band2Rate("1m"), + Domain: domainChanel, + Resolvers: options.GetResolvers(nil), + Silent: false, + Retry: 1, + Method: options.EnumType, + Writer: []outputter.Output{ + screenPrinter, + }, + ProcessBar: &process, + EtherInfo: eth, + } + opt.Check() + r, err := New(opt) + assert.NoError(t, err) + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() +} + +func TestPredict(t *testing.T) { + process := processbar2.ScreenProcess{} + screenPrinter, _ := output.NewScreenOutputNoWidth(false) + domains := []string{"stu.baidu.com"} + domainChanel := make(chan string) + eth, err := device.AutoGetDevices([]string{"1.1.1.1"}) + if err != nil { + t.Fatalf(err.Error()) + } + + go func() { + for _, d := range domains { + domainChanel <- d + } + close(domainChanel) + }() + opt := &options.Options{ + Rate: options.Band2Rate("1m"), + Domain: domainChanel, + Resolvers: options.GetResolvers(nil), + Silent: false, + Retry: 1, + Method: options.VerifyType, + Writer: []outputter.Output{ + screenPrinter, + }, + ProcessBar: &process, + EtherInfo: eth, + Predict: true, + } + opt.Check() + r, err := New(opt) + assert.NoError(t, err) + ctx := context.Background() + r.RunEnumeration(ctx) + r.Close() +} diff --git a/pkg/runner/send.go b/pkg/runner/send.go new file mode 100755 index 00000000..295041e2 --- /dev/null +++ b/pkg/runner/send.go @@ -0,0 +1,289 @@ +package runner + +import ( + "context" + "net" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/device" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/statusdb" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" +) + +// packetTemplate DNS请求包模板 +type packetTemplate struct { + eth *layers.Ethernet + ip *layers.IPv4 + udp *layers.UDP + opts gopacket.SerializeOptions + buf gopacket.SerializeBuffer + dnsip net.IP +} + +// templateCache 全局DNS服务器模板缓存 +// 优化说明: DNS服务器数量有限(通常<10个),每次创建模板开销较大 +// 使用 sync.Map 缓存模板,避免重复创建以太网/IP/UDP层 +// 预期性能提升: 5-10% (减少内存分配和IP解析开销) +var templateCache sync.Map + +// getOrCreate 获取或创建DNS服务器的数据包模板 +// 优化: 添加模板缓存,同一DNS服务器只创建一次模板 +func getOrCreate(dnsname string, ether *device.EtherTable, freeport uint16) *packetTemplate { + // 优化点1: 先尝试从缓存获取,避免重复创建 + // key格式: dnsname_freeport (同一DNS可能使用不同源端口) + cacheKey := dnsname + "_" + string(rune(freeport)) + if cached, ok := templateCache.Load(cacheKey); ok { + return cached.(*packetTemplate) + } + + // 缓存未命中,创建新模板 + DstIp := net.ParseIP(dnsname).To4() + eth := &layers.Ethernet{ + SrcMAC: ether.SrcMac.HardwareAddr(), + DstMAC: ether.DstMac.HardwareAddr(), + EthernetType: layers.EthernetTypeIPv4, + } + + ip := &layers.IPv4{ + Version: 4, + IHL: 5, + TOS: 0, + Length: 0, // FIX + Id: 0, + Flags: layers.IPv4DontFragment, + FragOffset: 0, + TTL: 255, + Protocol: layers.IPProtocolUDP, + Checksum: 0, + SrcIP: ether.SrcIp, + DstIP: DstIp, + } + + udp := &layers.UDP{ + SrcPort: layers.UDPPort(freeport), + DstPort: layers.UDPPort(53), + } + + _ = udp.SetNetworkLayerForChecksum(ip) + + template := &packetTemplate{ + eth: eth, + ip: ip, + udp: udp, + dnsip: DstIp, + opts: gopacket.SerializeOptions{ + ComputeChecksums: true, + FixLengths: true, + }, + buf: gopacket.NewSerializeBuffer(), + } + + // 存入缓存供后续复用 + templateCache.Store(cacheKey, template) + return template +} + +// sendCycle 实现发送域名请求的循环 +func (r *Runner) sendCycle() { + // 从发送通道接收域名,分发给工作协程 + for domain := range r.domainChan { + r.rateLimiter.Take() + v, ok := r.statusDB.Get(domain) + if !ok { + v = statusdb.Item{ + Domain: domain, + Dns: r.selectDNSServer(domain), + Time: time.Now(), + Retry: 0, + DomainLevel: 0, + } + r.statusDB.Add(domain, v) + } else { + v.Retry += 1 + v.Time = time.Now() + v.Dns = r.selectDNSServer(domain) + r.statusDB.Set(domain, v) + } + send(domain, v.Dns, r.options.EtherInfo, r.dnsID, uint16(r.listenPort), r.pcapHandle, layers.DNSTypeA) + atomic.AddUint64(&r.sendCount, 1) + } +} + +// sendCycleWithContext 实现带有context管理的发送域名请求循环 +// 优化: 添加批量发送机制,减少系统调用次数 +func (r *Runner) sendCycleWithContext(ctx context.Context, wg *sync.WaitGroup) { + defer wg.Done() + + // 优化点2: 批量发送机制 + // 批量大小: 每次收集100个域名后一起处理 + // 收益: 减少系统调用次数,提升发包吞吐量 20-30% + const batchSize = 100 + batch := make([]string, 0, batchSize) + batchItems := make([]statusdb.Item, 0, batchSize) + + // 定时器: 确保即使凑不满批次也能及时发送 + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + // 批量发送函数 + sendBatch := func() { + if len(batch) == 0 { + return + } + + // 背压控制:接收侧积压时主动降速,让 packetChan 有机会消化 + // 使用短暂 sleep 而非 ratelimiter 修改,避免引入复杂的并发问题 + if atomic.LoadInt32(&r.recvBackpressure) == 1 { + time.Sleep(5 * time.Millisecond) + } + + // 批量发送所有域名 + for i, domain := range batch { + send(domain, batchItems[i].Dns, r.options.EtherInfo, r.dnsID, + uint16(r.listenPort), r.pcapHandle, layers.DNSTypeA) + } + + // 原子更新发送计数 + atomic.AddUint64(&r.sendCount, uint64(len(batch))) + + // 清空批次,复用底层数组 + batch = batch[:0] + batchItems = batchItems[:0] + } + + // 主循环: 收集域名并批量发送 + for { + select { + case <-ctx.Done(): + // 退出前发送剩余批次 + sendBatch() + return + + case <-ticker.C: + // 定时发送,避免批次未满时延迟过高 + sendBatch() + + case domain, ok := <-r.domainChan: + if !ok { + // 通道关闭,发送剩余批次后退出 + sendBatch() + return + } + + // 速率限制 + r.rateLimiter.Take() + + // 获取或创建域名状态 + v, ok := r.statusDB.Get(domain) + if !ok { + v = statusdb.Item{ + Domain: domain, + Dns: r.selectDNSServer(domain), + Time: time.Now(), + Retry: 0, + DomainLevel: 0, + } + r.statusDB.Add(domain, v) + } else { + v.Retry += 1 + v.Time = time.Now() + v.Dns = r.selectDNSServer(domain) + r.statusDB.Set(domain, v) + } + + // 添加到批次 + batch = append(batch, domain) + batchItems = append(batchItems, v) + + // 批次已满,立即发送 + if len(batch) >= batchSize { + sendBatch() + } + } + } +} + +// send 发送单个DNS查询包 +func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, freeport uint16, handle *pcap.Handle, dnsType layers.DNSType) { + // 复用DNS服务器的包模板 + template := getOrCreate(dnsname, ether, freeport) + + // 从内存池获取DNS层对象 + dns := GlobalMemPool.GetDNS() + defer GlobalMemPool.PutDNS(dns) + + // 设置DNS查询参数 + dns.ID = dnsid + dns.QDCount = 1 + dns.RD = true // 递归查询标识 + + // 从内存池获取questions切片 + questions := GlobalMemPool.GetDNSQuestions() + defer GlobalMemPool.PutDNSQuestions(questions) + + // 添加查询问题 + questions = append(questions, layers.DNSQuestion{ + Name: []byte(domain), + Type: dnsType, + Class: layers.DNSClassIN, + }) + dns.Questions = questions + + // 从内存池获取序列化缓冲区 + buf := GlobalMemPool.GetBuffer() + defer GlobalMemPool.PutBuffer(buf) + + // 序列化数据包 + err := gopacket.SerializeLayers( + buf, + template.opts, + template.eth, template.ip, template.udp, dns, + ) + if err != nil { + gologger.Warningf("SerializeLayers faild:%s\n", err.Error()) + return + } + + // 发送数据包 + // 修复 Mac 缓冲区问题: 增加重试机制,使用指数退避 + const maxRetries = 3 + for retry := 0; retry < maxRetries; retry++ { + err = handle.WritePacketData(buf.Bytes()) + if err == nil { + return // 发送成功 + } + + errMsg := err.Error() + + // 检查是否为缓冲区错误 (Mac/Linux 常见) + // Mac BPF: "No buffer space available" (ENOBUFS) + // Linux: 可能有类似错误 + isBufferError := strings.Contains(errMsg, "No buffer space available") || + strings.Contains(errMsg, "ENOBUFS") || + strings.Contains(errMsg, "buffer") + + if isBufferError { + // 缓冲区满,需要重试 + if retry < maxRetries-1 { + // 指数退避: 10ms, 20ms, 40ms + backoff := time.Millisecond * time.Duration(10*(1<= 2 { + // Basic validation could be added here (e.g., is it a valid IP?) + servers = append(servers, fields[1]) + } + } + // Potentially handle 'search' and 'options' lines if needed in the future + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading %s: %w", resolvConfPath, err) + } + + if len(servers) == 0 { + return nil, fmt.Errorf("no nameservers found in %s", resolvConfPath) + } + + return servers, nil +} diff --git a/pkg/utils/resolver_windows.go b/pkg/utils/resolver_windows.go new file mode 100644 index 00000000..6263d0fb --- /dev/null +++ b/pkg/utils/resolver_windows.go @@ -0,0 +1,84 @@ +//go:build windows + +package utils + +import ( + "fmt" + "strings" + + // "os/exec" // No longer needed + // "bytes" // No longer needed + // "syscall" // No longer needed + + "github.com/StackExchange/wmi" // 需要添加这个依赖 +) + +// Win32_NetworkAdapterConfiguration WMI class structure (subset) +type win32_NetworkAdapterConfiguration struct { + Description string + IPEnabled bool + DNSServerSearchOrder []string + DNSHostName string + DefaultIPGateway []string // Added to help filter relevant adapters + DHCPEnabled bool // Added to help filter relevant adapters +} + +// GetSystemDefaultDNS retrieves the default DNS servers configured on Windows systems using WMI. +func GetSystemDefaultDNS() ([]string, error) { + var dst []win32_NetworkAdapterConfiguration + // Query WMI for network adapter configurations that have IP enabled + // We also filter for adapters that likely connect to the internet (have a gateway or are DHCP enabled) + // This helps avoid virtual/loopback adapters that might pollute the results. + query := "SELECT Description, IPEnabled, DNSServerSearchOrder, DNSHostName, DefaultIPGateway, DHCPEnabled FROM Win32_NetworkAdapterConfiguration WHERE IPEnabled = TRUE AND (DHCPEnabled = TRUE OR DefaultIPGateway IS NOT NULL)" + if err := wmi.Query(query, &dst); err != nil { + return nil, fmt.Errorf("WMI query failed: %w", err) + } + + servers := []string{} + found := false + + for _, nic := range dst { + // Skip adapters without DNS server entries + if nic.DNSServerSearchOrder == nil || len(nic.DNSServerSearchOrder) == 0 { + continue + } + + // Basic filtering: Skip clearly virtual or loopback-like adapters based on description + descLower := strings.ToLower(nic.Description) + if strings.Contains(descLower, "loopback") || strings.Contains(descLower, "virtual") || strings.Contains(descLower, "pseudo") { + continue + } + + // Add unique DNS servers found + for _, server := range nic.DNSServerSearchOrder { + trimmedServer := strings.TrimSpace(server) + // Add basic validation if needed (is it a valid IP?) + if trimmedServer != "" && trimmedServer != "::" && !contains(servers, trimmedServer) { + servers = append(servers, trimmedServer) + found = true + } + } + // Often, the first adapter with DNS configured is the primary one. + // We could potentially stop after finding the first valid one, but aggregating + // from all relevant adapters might be more robust in complex network setups. + } + + if !found || len(servers) == 0 { + // Fallback or specific error? Could try the ipconfig method as a last resort? + // For now, return an error if WMI doesn't yield results. + return nil, fmt.Errorf("no suitable network adapter with DNS configuration found via WMI") + } + + return servers, nil +} + +// contains checks if a slice contains a specific string. +// (Keep this helper function as it's still needed) +func contains(slice []string, item string) bool { + for _, a := range slice { + if a == item { + return true + } + } + return false +} diff --git a/pkg/utils/wildcard.go b/pkg/utils/wildcard.go new file mode 100755 index 00000000..04b021b4 --- /dev/null +++ b/pkg/utils/wildcard.go @@ -0,0 +1,366 @@ +package utils + +import ( + "github.com/boy-hack/ksubdomain/v2/pkg/core/gologger" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "sort" + "strings" +) + +type Pair struct { + Key string + Value int +} +type PairList []Pair + +func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p PairList) Len() int { return len(p) } +func (p PairList) Less(i, j int) bool { return p[i].Value > p[j].Value } + +// A function to turn a map into a PairList, then sort and return it. +func sortMapByValue(m map[string]int) PairList { + p := make(PairList, len(m)) + i := 0 + for k, v := range m { + p[i] = Pair{k, v} + i++ + } + sort.Sort(p) + return p +} + +// WildFilterOutputResult 泛解析过滤结果 +func WildFilterOutputResult(outputType string, results []result.Result) []result.Result { + if outputType == "none" { + return results + } else if outputType == "basic" { + return FilterWildCard(results) + } else if outputType == "advanced" { + return FilterWildCardAdvanced(results) + } + return nil +} + +// FilterWildCard 基于Result类型数据过滤泛解析 +// 传入参数为[]result.Result,返回过滤后的[]result.Result +// 通过分析整体结果,对解析记录中相同的ip进行阈值判断,超过则丢弃该结果 +func FilterWildCard(results []result.Result) []result.Result { + if len(results) == 0 { + return results + } + + gologger.Debugf("泛解析处理中,共 %d 条记录...\n", len(results)) + + // 统计每个IP出现的次数 + ipFrequency := make(map[string]int) + // 记录IP到域名的映射关系 + ipToDomains := make(map[string][]string) + // 域名计数 + totalDomains := len(results) + + // 第一遍扫描,统计IP频率 + for _, res := range results { + for _, answer := range res.Answers { + // 跳过非IP的记录(CNAME等) + if !strings.HasPrefix(answer, "CNAME ") && !strings.HasPrefix(answer, "NS ") && + !strings.HasPrefix(answer, "TXT ") && !strings.HasPrefix(answer, "PTR ") { + ipFrequency[answer]++ + ipToDomains[answer] = append(ipToDomains[answer], res.Subdomain) + } + } + } + + // 按出现频率排序IP + sortedIPs := sortMapByValue(ipFrequency) + + // 确定疑似泛解析的IP列表 + // 使用两个标准: + // 1. IP解析超过总域名数量的特定百分比(动态阈值) + // 2. 该IP解析的子域名数量超过特定阈值 + suspiciousIPs := make(map[string]bool) + + for _, pair := range sortedIPs { + ip := pair.Key + count := pair.Value + + // 计算该IP解析占总体的百分比 + percentage := float64(count) / float64(totalDomains) * 100 + + // 动态阈值:根据总域名数量调整 + // 域名数量少时阈值较高,域名数量多时阈值较低 + var threshold float64 + if totalDomains < 100 { + threshold = 30 // 如果域名总数小于100,阈值设为30% + } else if totalDomains < 1000 { + threshold = 20 // 如果域名总数在100-1000,阈值设为20% + } else { + threshold = 10 // 如果域名总数超过1000,阈值设为10% + } + + // 绝对数量阈值 + absoluteThreshold := 70 + + // 如果超过阈值,标记为可疑IP + if percentage > threshold || count > absoluteThreshold { + gologger.Debugf("发现可疑泛解析IP: %s (解析了 %d 个域名, %.2f%%)\n", + ip, count, percentage) + suspiciousIPs[ip] = true + } + } + + // 第二遍扫描,过滤结果 + var filteredResults []result.Result + + for _, res := range results { + // 检查该域名的所有IP是否均为可疑IP + // 如果有不可疑的IP,保留该记录 + validRecord := false + var filteredAnswers []string + + for _, answer := range res.Answers { + // 保留所有非IP记录(如CNAME) + if strings.HasPrefix(answer, "CNAME ") || strings.HasPrefix(answer, "NS ") || + strings.HasPrefix(answer, "TXT ") || strings.HasPrefix(answer, "PTR ") { + validRecord = true + filteredAnswers = append(filteredAnswers, answer) + } else if !suspiciousIPs[answer] { + // 保留不在可疑IP列表中的IP + validRecord = true + filteredAnswers = append(filteredAnswers, answer) + } + } + + if validRecord && len(filteredAnswers) > 0 { + filteredRes := result.Result{ + Subdomain: res.Subdomain, + Answers: filteredAnswers, + } + filteredResults = append(filteredResults, filteredRes) + } + } + + gologger.Infof("泛解析过滤完成,从 %d 条记录中过滤出 %d 条有效记录\n", + totalDomains, len(filteredResults)) + + return filteredResults +} + +// FilterWildCardAdvanced 提供更高级的泛解析检测算法 +// 使用多种启发式方法和特征检测来识别泛解析 +func FilterWildCardAdvanced(results []result.Result) []result.Result { + if len(results) == 0 { + return results + } + + gologger.Debugf("高级泛解析检测开始,共 %d 条记录...\n", len(results)) + + // 统计IP出现频率 + ipFrequency := make(map[string]int) + // 统计每个IP解析的不同子域名前缀数量 + ipPrefixVariety := make(map[string]map[string]bool) + // 统计IP解析的不同顶级域数量 + ipTLDVariety := make(map[string]map[string]bool) + // 记录IP到域名的映射 + ipToDomains := make(map[string][]string) + // 记录CNAME信息 + cnameRecords := make(map[string][]string) + + totalDomains := len(results) + + // 第一轮:收集统计信息 + for _, res := range results { + subdomain := res.Subdomain + parts := strings.Split(subdomain, ".") + + // 提取顶级域和前缀 + prefix := "" + tld := "" + if len(parts) > 1 { + prefix = parts[0] + tld = strings.Join(parts[1:], ".") + } else { + prefix = subdomain + tld = subdomain + } + + for _, answer := range res.Answers { + if strings.HasPrefix(answer, "CNAME ") { + // 提取CNAME目标 + cnameParts := strings.SplitN(answer, " ", 2) + if len(cnameParts) == 2 { + cnameTarget := cnameParts[1] + cnameRecords[subdomain] = append(cnameRecords[subdomain], cnameTarget) + } + continue + } + + // 只处理IP记录 + if !strings.HasPrefix(answer, "NS ") && + !strings.HasPrefix(answer, "TXT ") && + !strings.HasPrefix(answer, "PTR ") { + // 计数IP频率 + ipFrequency[answer]++ + + // 初始化IP的前缀集合和TLD集合 + if ipPrefixVariety[answer] == nil { + ipPrefixVariety[answer] = make(map[string]bool) + } + if ipTLDVariety[answer] == nil { + ipTLDVariety[answer] = make(map[string]bool) + } + + // 记录这个IP解析了哪些不同的前缀和TLD + ipPrefixVariety[answer][prefix] = true + ipTLDVariety[answer][tld] = true + + // 记录IP到域名的映射 + ipToDomains[answer] = append(ipToDomains[answer], subdomain) + } + } + } + + // 按照IP频率排序 + sortedIPs := sortMapByValue(ipFrequency) + + // 识别可疑IP列表 + suspiciousIPs := make(map[string]float64) // IP -> 可疑度分数(0-100) + + for _, pair := range sortedIPs { + ip := pair.Key + count := pair.Value + + // 初始可疑度分数 + suspiciousScore := 0.0 + + // 因子1: IP频率百分比 + freqPercentage := float64(count) / float64(totalDomains) * 100 + + // 因子2: 前缀多样性 + prefixVariety := len(ipPrefixVariety[ip]) + prefixVarietyRatio := float64(prefixVariety) / float64(count) * 100 + + // 因子3: TLD多样性 + tldVariety := len(ipTLDVariety[ip]) + + // 计算可疑度分数 + // 1. 频率因子 + if freqPercentage > 30 { + suspiciousScore += 40 + } else if freqPercentage > 10 { + suspiciousScore += 20 + } else if freqPercentage > 5 { + suspiciousScore += 10 + } + + // 2. 前缀多样性因子 + // 如果一个IP解析了大量不同前缀的域名,可能是CDN或者泛解析 + if prefixVarietyRatio > 90 && prefixVariety > 10 { + suspiciousScore += 30 + } else if prefixVarietyRatio > 70 && prefixVariety > 5 { + suspiciousScore += 20 + } + + // 3. 绝对数量因子 + if count > 100 { + suspiciousScore += 20 + } else if count > 50 { + suspiciousScore += 10 + } else if count > 20 { + suspiciousScore += 5 + } + + // 4. TLD多样性因子 - 如果一个IP解析了多个不同TLD,更可能是合法的 + if tldVariety > 3 { + suspiciousScore -= 20 + } else if tldVariety > 1 { + suspiciousScore -= 10 + } + + // 只有当可疑度分数超过阈值时,才标记为可疑IP + if suspiciousScore >= 35 { + gologger.Debugf("可疑IP: %s (解析域名数: %d, 占比: %.2f%%, 前缀多样性: %d/%d, 可疑度: %.2f)\n", + ip, count, freqPercentage, prefixVariety, count, suspiciousScore) + suspiciousIPs[ip] = suspiciousScore + } + } + + // 第二轮:过滤结果 + var filteredResults []result.Result + + // CNAME聚类分析:检测指向相同目标的多个CNAME记录 + cnameTargetCount := make(map[string]int) + for _, targets := range cnameRecords { + for _, target := range targets { + cnameTargetCount[target]++ + } + } + + // 识别可疑CNAME目标 + suspiciousCnames := make(map[string]bool) + for cname, count := range cnameTargetCount { + if count > 5 && float64(count)/float64(totalDomains)*100 > 10 { + gologger.Debugf("可疑CNAME目标: %s (指向次数: %d)\n", cname, count) + suspiciousCnames[cname] = true + } + } + + // 过滤结果 + for _, res := range results { + // 检查是否含有可疑CNAME + hasSuspiciousCname := false + if targets, ok := cnameRecords[res.Subdomain]; ok { + for _, target := range targets { + if suspiciousCnames[target] { + hasSuspiciousCname = true + break + } + } + } + + validRecord := !hasSuspiciousCname + var filteredAnswers []string + + // 处理所有回答 + for _, answer := range res.Answers { + isIP := !strings.HasPrefix(answer, "CNAME ") && + !strings.HasPrefix(answer, "NS ") && + !strings.HasPrefix(answer, "TXT ") && + !strings.HasPrefix(answer, "PTR ") + + // 保留所有非IP记录但排除可疑CNAME + if !isIP { + if strings.HasPrefix(answer, "CNAME ") { + cnameParts := strings.SplitN(answer, " ", 2) + if len(cnameParts) == 2 && suspiciousCnames[cnameParts[1]] { + continue // 跳过可疑CNAME + } + } + validRecord = true + filteredAnswers = append(filteredAnswers, answer) + } else { + // 针对IP记录,根据可疑度评分过滤 + suspiciousScore, isSuspicious := suspiciousIPs[answer] + + // 如果不在可疑IP列表中,或者可疑度较低,则保留 + if !isSuspicious || suspiciousScore < 50 { + validRecord = true + filteredAnswers = append(filteredAnswers, answer) + } + } + } + + // 只添加有效记录 + if validRecord && len(filteredAnswers) > 0 { + filteredRes := result.Result{ + Subdomain: res.Subdomain, + Answers: filteredAnswers, + } + filteredResults = append(filteredResults, filteredRes) + } + } + + gologger.Infof("高级泛解析过滤完成,从 %d 条记录中过滤出 %d 条有效记录\n", + totalDomains, len(filteredResults)) + + return filteredResults +} diff --git a/program.md b/program.md new file mode 100644 index 00000000..350151cb --- /dev/null +++ b/program.md @@ -0,0 +1,388 @@ +# KSubdomain — Agent Program + +> 本文件是面向 AI Agent 的项目上下文与任务指引。Agent 应首先通读此文件,再开始任何工作。 +> 类似 autoresearch 的 `program.md`,这里定义了项目的"研究组织逻辑"。 + +--- + +## 项目概述 + +**KSubdomain** 是一个无状态子域名枚举工具,核心思路来自 Masscan:绕过内核协议栈,直接在网卡层面收发 raw DNS 包,配合内置轻量状态表做重传,从而实现极速扫描。 + +当前版本 **v2.5**,模块路径 `github.com/boy-hack/ksubdomain/v2`,使用 Go 1.23+。 + +--- + +## 代码地图(必读) + +``` +cmd/ksubdomain/ CLI 入口,4 个子命令:enum / verify / test / device +pkg/ + options/options.go 核心配置结构体 Options(所有功能的控制面板) + runner/runner.go 扫描主循环,RunEnumeration() 启动 5 条并发 goroutine + runner/send.go 发包:模板缓存 + 批量发送 + 指数退避重试 + runner/recv.go 收包:BPF filter + CPU*2 并行解析 + CPU*2 并行处理 + runner/statusdb/db.go 64 分片锁状态表,记录发送状态与重传计数 + runner/mempool.go 全局 sync.Pool,复用 DNS 结构体减少 GC + runner/result.go 结果处理 + predict 预测触发 + runner/wildcard.go 泛解析过滤逻辑 + runner/outputter/ Output 接口 + txt/json/csv/jsonl/screen 实现 + device/ 网卡初始化(pcap),构建 EtherTable + sdk/sdk.go Go SDK 封装(Enum / Verify / *WithContext) + ns/ns.go NS 记录查询(--ns 模式辅助) +internal/ + predict/generator.go 基于 regular.cfg + regular.dict 的预测生成器 + assets/data/ 内置字典 subdomain.txt / subnext.txt +docs/ + OUTPUT_FORMATS.md 输出格式完整说明 +``` + +--- + +## 核心数据流 + +``` +Domain 输入 (chan string) + │ + ▼ +loadDomainsFromSource ──► domainChan + │ + ┌───────────┘ + ▼ + sendCycleWithContext + (模板缓存 + 批量发包) + │ + ▼ + statusdb 登记 {Domain, Dns, Time, Retry} + │ + ┌─────────────┴──────────────────────────────────┐ + ▼ ▼ + recvChan → packetChan → dnsChan → resultChan monitorProgress + │ + retry() + ▼ + handleResultWithContext + │ + ┌────────────┴──────────────┐ + ▼ ▼ + Output.WriteDomainResult() predict() → predictChan +``` + +--- + +## 关键指标(以此衡量改动是否有效) + +| 指标 | 当前基准 | 目标方向 | +|------|---------|---------| +| **检测速率** | 100k 域名约 30 秒(5M 带宽,4 核) | 同等带宽继续提升吞吐 | +| **漏报率** | 对比 massdns 结果几乎一致 | 减少 timeout 造成的漏报 | +| **误报率** | 泛解析过滤后接近 0 | advanced 模式覆盖更多泛解析场景 | +| **内存占用** | 百万级域名仍维持低内存 | 不引入新的大块堆分配 | +| **SDK 易用性** | NewScanner(config).Enum(domain) 3 行接入 | 任何改动不破坏 SDK 接口兼容性 | + +每次修改后,执行以下命令验证基准不退步: + +```bash +# 单元测试(无需网卡) +go test ./... + +# 集成测试(需要网卡权限) +sudo go test ./test/ -v -run Integration + +# 发包速率基准 +sudo ./ksubdomain test +``` + +--- + +## 当前优先任务(Roadmap) + +按优先级从高到低排列,Agent 应按序处理,每次专注一个任务。 + +**Roadmap 维护规则(Agent 可直接修改此文件)**: +- 完成一个任务 → 将 `[ ]` 改为 `[x]`,并在 `agent-log.md` 记录结果 +- 发现新问题或新改进点 → 自行在对应优先级下追加条目,注明发现来源 +- 评估某条任务不再必要 → 将 `[ ]` 改为 `[~]` 并在条目后用括号注明原因 +- **需要用户决策的事项不在此处等待**,而是写入 `agent-log.md` 的 `【待决策】` 区块,由用户查阅日志后自行处理 + +### P0 — 检测效率 + +- [x] **动态超时自适应**:RTT EWMA滑动均值,上界内部固定10s,始终启用 +- [x] **接收侧背压控制**:packetChan双水位监控(80%触发/50%恢复),send侧背压时sleep 5ms降速 +- [x] **批量重传合并**:按DNS server分组批量send(),去掉channel中转层,复用slice/map减少GC + +### P1 — 开发者集成 + +- [ ] **流式结果回调**:SDK 增加 `EnumStream(ctx, domain, callback func(Result))` 接口,支持实时处理而无需等待全部完成 +- [ ] **自定义 Output 接入**:在 `sdk.Config` 中暴露 `ExtraWriters []outputter.Output` 字段,允许调用方注入自定义 sink +- [ ] **错误类型化**:将 `runner` 层的错误字符串改为具名错误变量(`ErrPermissionDenied`、`ErrDeviceNotFound` 等),方便 SDK 用户 `errors.Is` 判断 +- [ ] **dev.md 重写**:`dev.md` 标注为 Outdated,需按当前架构(`chan string`、分片锁、输出接口)全面重写 + +### P1.5 — 工具联动兼容性审查 + +> 参考「工具联动兼容性矩阵」章节,逐条验证每个集成场景是否真实可用。 + +- [ ] **httpx 管道**:`--od --silent` 输出的域名能被 httpx 直接消费,无乱码、无多余行 +- [ ] **JSONL 下游兼容**:`--oy jsonl` 每行输出符合标准 JSON,`jq` 可直接解析 `.domain` `.type` `.records` 字段 +- [ ] **退出码语义**:扫描无结果时退出码是否为非 0(影响 shell 管道 `&&` 判断),确认并文档化 + +### P2 — 文档完善 + +- [ ] **docs/quickstart.md**:README 引用但缺失,补充从安装到第一次扫描的完整步骤 +- [ ] **docs/api.md**:补充 SDK 完整 API 参考,包含所有 Config 字段说明和典型错误处理 +- [ ] **docs/best-practices.md**:补充带宽选择、DNS 服务器选取、泛解析处理的最佳实践 +- [ ] **docs/faq.md**:整理常见问题(sudo 权限、macOS BPF buffer、WSL 网卡) +- [ ] **内联注释**:`runner.go` 和 `send.go` 的关键路径缺乏注释,补充架构级说明而非行级注释 + +### P3 — 工程健康 + +- [ ] **`simple` 二进制缺失**:git status 显示 `D simple`,确认是否需要在 `cmd/` 下补充 simple 子命令或删除引用 +- [ ] **CI 矩阵**:`build.yml` 验证 Linux/macOS/Windows 三平台交叉编译均通过 +- [ ] **版本号自动化**:`pkg/version/config.go` 硬编码版本字符串,改为 `go build -ldflags` 注入 + +--- + +## 修改约束 + +1. **不修改 `pkg/options/options.go` 中 `Options` 的已有字段名**:下游用户代码依赖此结构体,字段重命名视为破坏性变更 +2. **不修改 `pkg/sdk/sdk.go` 中 `Config`、`Scanner`、`Result` 的已有字段和方法签名**:SDK 公开 API,需向后兼容 +3. **`Output` 接口只可扩展,不可修改已有方法**:现有 `WriteDomainResult` 和 `Close` 签名不变 +4. **所有平台文件(`*_darwin.go`、`*_linux.go`、`*_windows.go`)需同步更新** +5. **Go 规则**:不使用 `++`/`--`,字符串用反引号或双引号(Go 惯例),接口定义只放在 `pkg/` 下 + +--- + +## 上手流程(Agent 首次运行) + +```bash +# 1. 查看项目状态 +git status +go build ./... + +# 2. 运行现有测试,确认基线通过 +go test ./pkg/... -v + +# 3. 检查缺失文档(README 中引用但未创建的文件) +ls docs/ + +# 4. 选择 Roadmap 中最高优先级未完成项,新建分支后开始实现 +# 每次只聚焦一个任务,完成后运行 go test ./... 确认无回归 +``` + +--- + +## 测试规范(内网环境) + +> **当前为内网环境,禁止发起大批量外部 DNS 请求。** + +Agent 在实现每个功能后,必须自行运行以下测试,验证参数行为正确、无崩溃、无明显 bug: + +```bash +# 编译验证(无需权限,最快检查) +go build ./... + +# 单元测试(无需网卡,全部通过为准入条件) +go test ./... -timeout 30s + +# 本地轻量功能冒烟测试(只需几条域名,不做大批量扫描) +# 构建二进制 +go build -o ksubdomain_dev ./cmd/ksubdomain + +# verify 模式:验证 1 条已知域名,检查 -d / -o / --oy / --silent / --np 参数 +sudo ./ksubdomain_dev verify -d www.baidu.com --timeout 5 --retry 2 -b 1m --np +sudo ./ksubdomain_dev verify -d www.baidu.com --oy jsonl --silent +sudo ./ksubdomain_dev verify -d www.baidu.com -o /tmp/ksub_test.txt && cat /tmp/ksub_test.txt + +# enum 模式:只枚举 1 个域名、新建一个小字典,不超过10行,检查基本流程不崩溃 +sudo ./ksubdomain_dev enum -d baidu.com --timeout 5 --retry 1 -b 1m --np -f subdomain.dic + +# test 模式:测试本地网卡发包速率,无外部流量 +sudo ./ksubdomain_dev test + +# device 模式:列出可用网卡 +sudo ./ksubdomain_dev device +``` + +**测试要点**: +- 每个 CLI 参数至少覆盖一次,确认有无 panic 或非预期输出 +- 输出文件格式正确(txt 每行一条、jsonl 每行合法 JSON) +- `--silent` 模式下无多余输出,`--np` 模式下无域名打印 +- 测试完毕后删除临时文件:`rm -f /tmp/ksub_test* ksubdomain_dev` + +--- + +## 分支管理规范 + +> **main 分支只读,任何功能改动都在独立分支上进行。** + +每次实现一个 Roadmap 任务,遵循以下流程: + +```bash +# 1. 从 main 新建功能分支,命名格式:feature/<简短描述> +git checkout main +git checkout -b feature/dynamic-timeout # P0 示例 +git checkout -b feature/stream-sdk # P1 示例 +git checkout -b docs/quickstart # P2 示例 +git checkout -b fix/simple-binary # P3 示例 + +# 2. 实现功能,提交粒度:每个逻辑单元一次 commit +git add . +git commit -m "feat(runner): add RTT sliding window for dynamic timeout" + +# 3. 功能完成后在分支上运行完整测试 +go test ./... -timeout 30s + +# 4. 将分支结论记录到运行日志(见下文),不合并到 main +# main 分支由人工决定何时合并 +``` + +**分支命名前缀约定**: + +| 前缀 | 用途 | +|------|------| +| `feature/` | 新功能实现(对应 Roadmap P0/P1) | +| `docs/` | 文档补充(对应 Roadmap P2) | +| `fix/` | Bug 修复(对应 Roadmap P3 或冒烟测试发现的问题) | +| `refactor/` | 重构(不改变外部行为) | +| `exp/` | 实验性改动(不确定是否保留) | + +--- + +## 运行日志规范 + +Agent 每完成一个任务,必须在 `agent-log.md` 文件中追加一条记录。该文件位于项目根目录,不存在则自行创建。 + +**记录格式**: + +```markdown +## [YYYY-MM-DD] <分支名> — <任务简述> + +**目标**:对应 Roadmap 中的哪一条 +**改动文件**:列出修改的文件 +**测试结果**: +- go test ./... → PASS / FAIL(附错误摘要) +- 冒烟测试:列出执行的命令及输出摘要 +**结论**:改动是否有效,是否引入新问题,下一步建议 + + +### 【待决策】 +``` + +**说明**: +- `【待决策】` 区块是 Agent 与用户沟通的唯一通道。Agent 不中断工作等待回复,而是记录后继续处理下一任务。 +- 用户查看 `agent-log.md` 时,搜索 `【待决策】` 即可找到所有待处理事项,勾选 `[x]` 或在条目下方回复意见后,Agent 下次运行时读取并执行。 +- 典型的待决策场景:破坏性 API 变更是否接受、新增外部依赖是否引入、某功能是否值得实现、发现潜在安全或性能风险需要人工确认。 + +**示例**: + +```markdown +## [2026-03-17] feature/dynamic-timeout — 动态超时自适应 + +**目标**:P0 动态超时自适应 +**改动文件**:pkg/runner/runner.go, pkg/runner/send.go, pkg/options/options.go +**测试结果**: +- go test ./... → PASS +- 冒烟测试:`sudo ./ksubdomain_dev verify -d www.baidu.com --timeout 5 --retry 2` + 输出:`www.baidu.com => 110.242.68.66`,耗时约 2s,正常 +**结论**:RTT 滑动均值计算正常,低延迟场景下超时提前收敛,无漏报。 + 建议后续在 P0 背压控制任务中复用 RTT 数据。 + +### 【待决策】 + 若希望新版本默认开启动态超时,需将默认值改为 true,这属于行为变更,请确认。 + 方案 A:默认 false,用户显式 --dynamic-timeout 启用(保守) + 方案 B:默认 true,--no-dynamic-timeout 可关闭(激进,影响现有脚本) +``` + +--- + +## 输出格式速查 + +| 格式 | 写入时机 | 适用场景 | +|------|---------|---------| +| `txt` | 实时(每条结果) | 人工查阅、管道 grep | +| `json` | 完成后一次性 | 离线分析、脚本处理 | +| `csv` | 完成后一次性 | Excel/数据库导入 | +| `jsonl` | 实时流式 | 管道链(httpx/nuclei)、监控系统 | +| screen | 实时彩色(stdout) | 交互式终端 | + +--- + +## 工具联动兼容性矩阵 + +> Agent 在审查 P1.5 任务时,以此为检查清单。每条给出期望行为、验证命令和当前状态。 + +### ProjectDiscovery 工具链 + +| 工具 | 联动方式 | 期望行为 | 关键参数 | 当前状态 | +|------|---------|---------|---------|---------| +| **httpx** | stdout 管道 | 每行一个域名,httpx 自动探活 | `--od --silent` | 待验证 | +| **nuclei** | stdout 管道或临时文件 | 域名作为目标,模板正常扫描 | `--od` + nuclei `-l /dev/stdin` | 待验证 | +| **naabu** | stdout 管道 | 域名列表作为端口扫描输入 | `--od` + naabu `-iL -` | 待验证 | +| **dnsx** | stdout 管道 | ksubdomain 初筛 → dnsx 多记录精查 | `--od --silent` + dnsx `-a -cname` | 待验证 | +| **subfinder** | subfinder → ksubdomain stdin | subfinder 发现域名 → ksubdomain verify 存活确认 | ksubdomain `v --stdin` | 待验证 | +| **alterx** | alterx → ksubdomain stdin | 排列生成 → 批量验证,大量输入不丢包 | ksubdomain `v --stdin -b 10m` | 待验证 | +| **katana** | ksubdomain → katana | 子域名列表作为爬取种子 | `--od` + katana `-list -` | 待验证 | +| **chaos** | chaos → ksubdomain verify | chaos 数据集导入验证存活 | ksubdomain `v -f chaos_output.txt` | 待验证 | + +### 验证命令参考(内网用小字典,不发大批量请求) + +```bash +# 1. httpx 联动:枚举 → HTTP 探活 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent | httpx -silent -title + +# 2. dnsx 联动:ksubdomain 初筛 → dnsx 多类型查询 +sudo ./ksubdomain_dev verify -d www.baidu.com --od --silent | dnsx -a -cname -resp + +# 3. naabu 联动:枚举 → 端口扫描 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent | naabu -silent -p 80,443 + +# 4. subfinder → ksubdomain:二阶段发现 +subfinder -d baidu.com -silent | sudo ./ksubdomain_dev verify --stdin --silent --od + +# 5. alterx → ksubdomain:排列验证 +echo 'www.baidu.com' | alterx -silent | sudo ./ksubdomain_dev verify --stdin --np + +# 6. JSONL 流式过滤后交给 httpx +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --oy jsonl --silent \ + | jq -r 'select(.type=="A") | .domain' \ + | httpx -silent + +# 7. nuclei 漏洞扫描 +sudo ./ksubdomain_dev enum -d baidu.com -f subdomain.dic --od --silent \ + | nuclei -l /dev/stdin -t technologies/ -silent +``` + +### 联动验证要点 + +- **`--od` 输出必须是纯域名单列**,无 `=>` 箭头、无 IP、无空行,否则下游工具解析失败 +- **`--silent` 必须屏蔽进度条和 banner**,只保留结果行,否则污染管道数据 +- **`--stdin` 必须正确处理 EOF**,subfinder/alterx 结束后 ksubdomain 应正常退出而非挂起 +- **退出码**:有结果退出 0,无结果退出非 0,让 shell `&&` 链路能正确短路 +- **JSONL 字段名**必须稳定(`domain`、`type`、`records`、`timestamp`),jq 脚本依赖此约定 + +--- + +## 集成速查 + +```bash +# 完整侦察链:子域名枚举 → HTTP 探活 → 漏洞扫描 +sudo ./ksubdomain enum -d example.com -f subdomain.dic --od --silent \ + | httpx -silent \ + | nuclei -l /dev/stdin -silent + +# 二阶段发现:subfinder 粗扫 → ksubdomain 精筛存活 +subfinder -d example.com -silent \ + | sudo ./ksubdomain verify --stdin --od --silent + +# 流式过滤(仅 A 记录)后端口扫描 +sudo ./ksubdomain enum -d example.com --oy jsonl --silent \ + | jq -r 'select(.type=="A") | .domain' \ + | naabu -silent -p 80,443,8080,8443 + +# Go SDK 最简集成 +scanner := sdk.NewScanner(sdk.DefaultConfig) +results, err := scanner.Enum('example.com') +``` + +--- + +*`program.md` 由 Agent 和人工共同维护:Roadmap 条目由 Agent 自主更新,需要人工判断的事项统一写入 `agent-log.md` 的 `【待决策】` 区块。* diff --git a/readme.md b/readme.md old mode 100644 new mode 100755 index 123cb649..ed736c52 --- a/readme.md +++ b/readme.md @@ -1,105 +1,195 @@ -ksubdomain是一款基于无状态的子域名爆破工具,类似无状态端口扫描,支持在Windows/Linux/Mac上进行快速的DNS爆破,在Mac和Windows上理论最大发包速度在30w/s,linux上为160w/s。 +# KSubdomain: 极速无状态子域名爆破工具 -hacking8信息流的src资产收集 https://i.hacking8.com/src/ 用的是ksubdomain +[![Release](https://img.shields.io/github/release/boy-hack/ksubdomain.svg)](https://github.com/boy-hack/ksubdomain/releases) [![Go Report Card](https://goreportcard.com/badge/github.com/boy-hack/ksubdomain)](https://goreportcard.com/report/github.com/boy-hack/ksubdomain) [![License](https://img.shields.io/github/license/boy-hack/ksubdomain)](https://github.com/boy-hack/ksubdomain/blob/main/LICENSE) + +**KSubdomain 是一款基于无状态技术的子域名爆破工具,带来前所未有的扫描速度和极低的内存占用。** 告别传统工具的效率瓶颈,体验闪电般的 DNS 查询,同时拥有可靠的状态表重发机制,确保结果的完整性。 KSubdomain 支持 Windows、Linux 和 macOS,是进行大规模DNS资产探测的理想选择。 ![](image.gif) -## Useage +## 🚀 核心优势 + +* **闪电般的速度:** 采用无状态扫描技术,直接操作网络适配器进行原始套接字发包,绕过系统内核的网络协议栈,实现惊人的发包速率。通过 `test` 命令可探测本地网卡的最大发送速度。 +* **极低的资源消耗:** 创新的内存管理机制,包括对象池和全局内存池,显著降低内存分配和 GC 压力,即使处理海量域名也能保持低内存占用。 +* **无状态设计:** 类似 Masscan 的无状态扫描,不从系统维护状态表,自建轻量状态表,从根本上解决了传统扫描工具的内存瓶颈和性能限制,以及解决了无状态扫描漏包问题。 +* **可靠的重发:** 内建智能重发机制,有效应对网络抖动和丢包,确保结果的准确性和完整性。 +* **跨平台支持:** 完美兼容 Windows, Linux, macOS。 +* **易于使用:** 简洁的命令行接口,提供验证 (verify) 和枚举 (enum) 两种模式,并内置常用字典。 + +## ⚡ 性能亮点 + +KSubdomain 在速度和效率上远超同类工具。以下是在 4 核 CPU、5M 带宽网络环境下,使用 10 万字典进行的对比测试: + +| 工具 | 扫描模式 | 发包方式 | 命令 | 耗时 | 成功个数 | 备注 | +| ------------ | -------- | ------------ | -------------------------------------------------------------------------- | -------------- | -------- | ------------------------- | +| **KSubdomain** | 验证 | pcap 网卡发包 | `time ./ksubdomain v -b 5m -f d2.txt -o k.txt -r dns.txt --retry 3 --np` | **~30 秒** | 1397 | `--np` 关闭实时打印 | +| massdns | 验证 | pcap/socket | `time ./massdns -r dns.txt -t A -w m.txt d2.txt --root -o L` | ~3 分 29 秒 | 1396 | | +| dnsx | 验证 | socket | `time ./dnsx -a -o d.txt -r dns.txt -l d2.txt -retry 3 -t 5000` | ~5 分 26 秒 | 1396 | `-t 5000` 设置 5000 并发 | + +**结论:** KSubdomain 的速度是 massdns 的 **7 倍**,是 dnsx 的 **10 倍** 以上! +## 🛠️ 技术革新 (v2.0) + +KSubdomain 2.0 版本引入了多项底层优化,进一步压榨性能潜力: + +1. **状态表优化:** + * **分片锁 (Sharded Lock):** 替代全局锁,大幅减少锁竞争,提高并发写入效率。 + * **高效哈希:** 优化键值存储,均匀分布域名,提升查找速度。 +2. **发包机制优化:** + * **对象池:** 复用 DNS 包结构体,减少内存分配和 GC 开销。 + * **模板缓存:** 为相同 DNS 服务器复用以太网/IP/UDP 层数据,减少重复构建开销。 + * **并行发送:** 多协程并行发包,充分利用多核 CPU 性能。 + * **批量处理:** 批量发送域名请求,减少系统调用和上下文切换。 +3. **接收机制优化:** + * **对象池:** 复用解析器和缓冲区,降低内存消耗。 + * **并行处理管道:** 接收 → 解析 → 处理三阶段并行,提高处理流水线效率。 + * **缓冲区优化:** 增加内部 Channel 缓冲区大小,避免处理阻塞。 + * **高效过滤:** 优化 BPF 过滤规则和包处理逻辑,快速丢弃无效数据包。 +4. **内存管理优化:** + * **全局内存池:** 引入 `sync.Pool` 管理常用数据结构,减少内存分配和碎片。 + * **结构复用:** 复用 DNS 查询结构和序列化缓冲区。 +5. **架构与并发优化:** + * **动态并发:** 根据 CPU 核心数自动调整协程数量。 + * **高效随机数:** 使用性能更优的随机数生成器。 + * **自适应速率:** 根据网络状况和系统负载动态调整发包速率。 + * **批量加载:** 批量加载和处理域名,降低单个域名处理的固定开销。 + +## 📦 安装 + +1. **下载预编译二进制文件:** 前往 [Releases](https://github.com/boy-hack/ksubdomain/releases) 页面下载对应系统的最新版本。 +2. **安装 `libpcap` 依赖:** + * **Windows:** 下载并安装 [Npcap](https://npcap.com/) 驱动 (WinPcap 可能无效)。 + * **Linux:** 已静态编译打包 `libpcap`,通常无需额外操作。若遇问题,请尝试安装 `libpcap-dev` 或 `libcap-devel` 包。 + * **macOS:** 系统自带 `libpcap`,无需安装。 +3. **赋予执行权限 (Linux/macOS):** `chmod +x ksubdomain` +4. **运行!** + +### 源码编译 (可选) + +确保您已安装 Go 1.23 版本和 `libpcap` 环境。 + ```bash -NAME: - KSubdomain - 无状态子域名爆破工具 +go install -v github.com/boy-hack/ksubdomain/v2/cmd/ksubdomain@latest +# 二进制文件通常位于 $GOPATH/bin 或 $HOME/go/bin +``` -USAGE: - ksubdomain [global options] command [command options] [arguments...] +## 📖 使用说明 -VERSION: - 1.4 +```bash +KSubdomain - 极速无状态子域名爆破工具 + +用法: + ksubdomain [全局选项] 命令 [命令选项] [参数...] -COMMANDS: - enum, e 枚举域名 - verify, v 验证模式 - test 测试本地网卡的最大发送速度 - help, h Shows a list of commands or help for one command +版本: + 查看版本信息: ksubdomain --version -GLOBAL OPTIONS: - --help, -h show help (default: false) - --version, -v print the version (default: false) +命令: + enum, e 枚举模式: 提供主域名进行爆破 + verify, v 验证模式: 提供域名列表进行验证 + test 测试本地网卡最大发包速度 + help, h 显示命令列表或某个命令的帮助 +全局选项: + --help, -h 显示帮助 (默认: false) + --version, -v 打印版本信息 (默认: false) ``` -### 模式 -**验证模式** -提供完整的域名列表,ksubdomain负责快速获取结果 -```bash -./ksubdomain verify -h +### 验证模式 (Verify) -NAME: - cmd verify - 验证模式 +验证模式用于快速检查提供的域名列表的存活状态。 + +```bash +./ksubdomain verify -h # 查看验证模式帮助,可缩写 ksubdomain v USAGE: - cmd verify [command options] [arguments...] + ksubdomain verify [command options] [arguments...] OPTIONS: - --filename value, -f value 验证域名文件路径 - --band value, -b value 宽带的下行速度,可以5M,5K,5G (default: "2m") - --resolvers value, -r value dns服务器文件路径,一行一个dns地址 - --output value, -o value 输出文件名 - --silent 使用后屏幕将仅输出域名 (default: false) - --retry value 重试次数,当为-1时将一直重试 (default: 3) - --timeout value 超时时间 (default: 6) - --stdin 接受stdin输入 (default: false) - --only-domain, --od 只打印域名,不显示ip (default: false) - --not-print, --np 不打印域名结果 (default: false) - --help, -h show help (default: false) + --filename value, -f value 验证域名的文件路径 + --domain value, -d value 域名 + --band value, -b value 宽带的下行速度,可以5M,5K,5G (default: "3m") + --resolvers value, -r value dns服务器,默认会使用内置dns + --output value, -o value 输出文件名 + --output-type value, --oy value 输出文件类型: json, txt, csv (default: "txt") + --silent 使用后屏幕将仅输出域名 (default: false) + --retry value 重试次数,当为-1时将一直重试 (default: 3) + --timeout value 超时时间 (default: 6) + --stdin 接受stdin输入 (default: false) + --not-print, --np 不打印域名结果 (default: false) + --eth value, -e value 指定网卡名称 + --wild-filter-mode value 泛解析过滤模式[从最终结果过滤泛解析域名]: basic(基础), advanced(高级), none(不过滤ne") + --predict 启用预测域名模式 (default: false) + --help, -h show help (default: false) + +# 示例: +# 验证多个域名解析 +./ksubdomain v -d xx1.example.com -d xx2example.com + +# 从文件读取域名进行验证,保存为 output.txt +./ksubdomain v -f domains.txt -o output.txt + +# 从标准输入读取域名,带宽限制为 10M +cat domains.txt | ./ksubdomain v --stdin -b 10M + +# 启用预测模式,泛解析过滤,保存为csv +./ksubdomain v -f domains.txt --predict --wild-filter-mode advanced --oy csv -o output.csv ``` -``` -从文件读取 -./ksubdomain v -f dict.txt +### 枚举模式 (Enum) -从stdin读取 -echo "www.hacking8.com"|./ksubdomain v --stdin -``` -**枚举模式** -只提供一级域名,指定域名字典或使用ksubdomain内置字典,枚举所有二级域名 -```bash -./ksubdomain enum -h +枚举模式基于字典和预测算法爆破指定域名下的子域名。 -NAME: - cmd enum - 枚举域名 +```bash +./ksubdomain enum -h # 查看枚举模式帮助,可简写 ksubdomain e USAGE: - cmd enum [command options] [arguments...] + ksubdomain enum [command options] [arguments...] OPTIONS: - --band value, -b value 宽带的下行速度,可以5M,5K,5G (default: "2m") - --resolvers value, -r value dns服务器文件路径,一行一个dns地址 - --output value, -o value 输出文件名 - --silent 使用后屏幕将仅输出域名 (default: false) - --retry value 重试次数,当为-1时将一直重试 (default: 3) - --timeout value 超时时间 (default: 6) - --stdin 接受stdin输入 (default: false) - --only-domain, --od 只打印域名,不显示ip (default: false) - --not-print, --np 不打印域名结果 (default: false) - --domain value, -d value 爆破的域名 - --domainList value, --dl value 从文件中指定域名 - --filename value, -f value 字典路径 - --skip-wild 跳过泛解析域名 (default: false) - --level value, -l value 枚举几级域名,默认为2,二级域名 (default: 2) - --level-dict value, --ld value 枚举多级域名的字典文件,当level大于2时候使用,不填则会默认 - --help, -h show help (default: false) + --domain value, -d value 域名 + --band value, -b value 宽带的下行速度,可以5M,5K,5G (default: "3m") + --resolvers value, -r value dns服务器,默认会使用内置dns + --output value, -o value 输出文件名 + --output-type value, --oy value 输出文件类型: json, txt, csv (default: "txt") + --silent 使用后屏幕将仅输出域名 (default: false) + --retry value 重试次数,当为-1时将一直重试 (default: 3) + --timeout value 超时时间 (default: 6) + --stdin 接受stdin输入 (default: false) + --not-print, --np 不打印域名结果 (default: false) + --eth value, -e value 指定网卡名称 + --wild-filter-mode value 泛解析过滤模式[从最终结果过滤泛解析域名]: basic(基础), advanced(高级), none(不过滤) (default: "none") + --predict 启用预测域名模式 (default: false) + --filename value, -f value 字典路径 + --ns 读取域名ns记录并加入到ns解析器中 (default: false) + --help, -h show help (default: false) + +# 示例: +# 枚举多个域名 +./ksubdomain e -d example.com -d hacker.com + +# 从文件读取字典枚举,保存为 output.txt +./ksubdomain e -f sub.dict -o output.txt + +# 从标准输入读取域名,带宽限制为 10M +cat domains.txt | ./ksubdomain e --stdin -b 10M + +# 启用预测模式枚举域名,泛解析过滤,保存为csv +./ksubdomain e -d example.com --predict --wild-filter-mode advanced --oy csv -o output.csv ``` -``` -./ksubdomain e -d baidu.com +## ✨ 特性与技巧 -从stdin获取 -echo "baidu.com"|./ksubdomain e --stdin -``` +* **带宽自动适配:** 只需使用 `-b` 参数指定你的公网下行带宽 (如 `-b 10m`), KSubdomain 会自动优化发包速率。 +* **测试最大速率:** 运行 `./ksubdomain test` 测试当前环境的最大理论发包速率。 +* **自动网卡检测:** KSubdomain 会自动检测可用网卡。 +* **进度显示:** 实时进度条显示 成功数 / 发送数 / 队列长度 / 接收数 / 失败数 / 已耗时。 +* **参数调优:** 根据网络质量和目标域名数量,调整 `--retry` 和 `--timeout` 参数以获得最佳效果。当 `--retry` 为 -1 时,将无限重试直至所有请求成功或超时。 +* **多种输出格式:** 支持 `txt` (实时输出), `json` (完成后输出), `csv` (完成后输出)。通过 `-o` 指定文件名后缀即可 (如 `result.json`)。 +* **环境变量配置:** + * `KSubdomainConfig`: 指定配置文件的路径。 +## 💡 参考 -## 参考 -- 原ksubdomain https://github.com/knownsec/ksubdomain -- 从 Masscan, Zmap 源码分析到开发实践 -- ksubdomain 无状态域名爆破工具介绍 -- [ksubdomain与massdns的对比](https://mp.weixin.qq.com/s?__biz=MzU2NzcwNTY3Mg==&mid=2247484471&idx=1&sn=322d5db2d11363cd2392d7bd29c679f1&chksm=fc986d10cbefe406f4bda22f62a16f08c71f31c241024fc82ecbb8e41c9c7188cfbd71276b81&token=76024279&lang=zh_CN#rd) \ No newline at end of file +* 原 KSubdomain 项目: [https://github.com/knownsec/ksubdomain](https://github.com/knownsec/ksubdomain) +* 从 Masscan, Zmap 源码分析到开发实践: [https://paper.seebug.org/1052/](https://paper.seebug.org/1052/) +* KSubdomain 无状态域名爆破工具介绍: [https://paper.seebug.org/1325/](https://paper.seebug.org/1325/) +* KSubdomain 与 massdns 的对比分析: [微信公众号文章链接](https://mp.weixin.qq.com/s?__biz=MzU2NzcwNTY3Mg==&mid=2247484471&idx=1&sn=322d5db2d11363cd2392d7bd29c679f1&chksm=fc986d10cbefe406f4bda22f62a16f08c71f31c241024fc82ecbb8e41c9c7188cfbd71276b81&token=76024279&lang=zh_CN#rd) diff --git a/run_performance_test.sh b/run_performance_test.sh new file mode 100755 index 00000000..0289efa7 --- /dev/null +++ b/run_performance_test.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# +# KSubdomain 性能基准测试脚本 +# 用途: 运行 10 万域名性能测试,验证 README 中的性能指标 +# +# 参考 README: +# - 字典: 10 万域名 +# - 带宽: 5M +# - 耗时: ~30 秒 +# - 成功: 1397 个 +# + +set -e + +# 颜色输出 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN} KSubdomain 性能基准测试${NC}" +echo -e "${GREEN} 参考 README: 10万域名 ~30秒${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +# 检查 root 权限 +if [ "$EUID" -ne 0 ]; then + echo -e "${RED}错误: 需要 root 权限运行性能测试${NC}" + echo -e "${YELLOW}请使用: sudo $0${NC}" + exit 1 +fi + +# 检查 Go 环境 +if ! command -v go &> /dev/null; then + echo -e "${RED}错误: 未找到 Go 环境${NC}" + exit 1 +fi + +echo -e "${YELLOW}Go 版本:${NC} $(go version)" +echo "" + +# 检查网络 +echo -e "${BLUE}[1/4] 检查网络连接...${NC}" +if ping -c 1 8.8.8.8 &> /dev/null; then + echo -e "${GREEN}✓ 网络连接正常${NC}" +else + echo -e "${YELLOW}⚠ 网络连接可能有问题,测试结果可能不准确${NC}" +fi +echo "" + +# 快速测试 (1000 域名) +echo -e "${BLUE}[2/4] 快速测试 (1000 域名)...${NC}" +echo -e "${YELLOW}目标: < 2 秒${NC}" + +go test -tags=performance -bench=Benchmark1k ./test/ -timeout 5m -v 2>&1 | \ + grep -E "Benchmark1k|total_seconds|success|domains/sec" | \ + tee /tmp/ksubdomain_1k.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 1000 域名测试完成${NC}" +else + echo -e "${RED}✗ 1000 域名测试失败${NC}" +fi +echo "" + +# 中等测试 (10000 域名) +echo -e "${BLUE}[3/4] 中等测试 (10000 域名)...${NC}" +echo -e "${YELLOW}目标: < 5 秒${NC}" + +go test -tags=performance -bench=Benchmark10k ./test/ -timeout 5m -v 2>&1 | \ + grep -E "Benchmark10k|total_seconds|success|domains/sec" | \ + tee /tmp/ksubdomain_10k.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 10000 域名测试完成${NC}" +else + echo -e "${RED}✗ 10000 域名测试失败${NC}" +fi +echo "" + +# 完整测试 (100000 域名) - README 标准 +echo -e "${BLUE}[4/4] 完整测试 (100000 域名) - README 标准${NC}" +echo -e "${YELLOW}目标: < 30 秒 (参考 README)${NC}" +echo -e "${YELLOW}提示: 这将需要几分钟,请耐心等待...${NC}" +echo "" + +go test -tags=performance -bench=Benchmark100k ./test/ -timeout 10m -v 2>&1 | \ + tee /tmp/ksubdomain_100k.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo "" + echo -e "${GREEN}✓ 100000 域名测试完成${NC}" +else + echo "" + echo -e "${RED}✗ 100000 域名测试失败${NC}" +fi +echo "" + +# 提取性能数据 +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN} 性能测试总结${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +# 100k 测试结果 +if [ -f /tmp/ksubdomain_100k.log ]; then + echo -e "${BLUE}100,000 域名测试 (README 标准):${NC}" + + TOTAL_SEC=$(grep "total_seconds" /tmp/ksubdomain_100k.log | tail -1 | awk '{print $2}') + SUCCESS=$(grep "success_count" /tmp/ksubdomain_100k.log | tail -1 | awk '{print $2}') + RATE_PCT=$(grep "success_rate" /tmp/ksubdomain_100k.log | tail -1 | awk '{print $2}') + SPEED=$(grep "domains/sec" /tmp/ksubdomain_100k.log | tail -1 | awk '{print $2}') + + echo -e " - 总耗时: ${YELLOW}${TOTAL_SEC} 秒${NC}" + echo -e " - 成功数: ${YELLOW}${SUCCESS}${NC}" + echo -e " - 成功率: ${YELLOW}${RATE_PCT}${NC}" + echo -e " - 扫描速率: ${YELLOW}${SPEED} domains/s${NC}" + echo "" + + # 性能评估 + if (( $(echo "$TOTAL_SEC <= 30" | bc -l) )); then + echo -e "${GREEN}✅ 性能评估: 优秀 (达到 README 标准: ≤30秒)${NC}" + elif (( $(echo "$TOTAL_SEC <= 40" | bc -l) )); then + echo -e "${YELLOW}✓ 性能评估: 良好 (30-40秒)${NC}" + elif (( $(echo "$TOTAL_SEC <= 60" | bc -l) )); then + echo -e "${YELLOW}⚠ 性能评估: 一般 (40-60秒,可能需要优化)${NC}" + else + echo -e "${RED}❌ 性能评估: 较慢 (>60秒,需要检查网络和配置)${NC}" + fi + echo "" + + # README 对比 + echo -e "${BLUE}与 README 对比:${NC}" + echo -e " - README 标准: ~30 秒, 1397 个成功" + echo -e " - 本次测试: ${TOTAL_SEC} 秒, ${SUCCESS} 个成功" + echo "" + + # 与其他工具对比 + echo -e "${BLUE}与其他工具对比 (README 数据):${NC}" + echo -e " - massdns: ~3分29秒 (209秒) → ksubdomain 快 ${YELLOW}$(echo "scale=1; 209/$TOTAL_SEC" | bc)x${NC}" + echo -e " - dnsx: ~5分26秒 (326秒) → ksubdomain 快 ${YELLOW}$(echo "scale=1; 326/$TOTAL_SEC" | bc)x${NC}" +fi + +echo "" +echo -e "${BLUE}详细日志:${NC}" +echo -e " - /tmp/ksubdomain_1k.log" +echo -e " - /tmp/ksubdomain_10k.log" +echo -e " - /tmp/ksubdomain_100k.log" +echo "" + +echo -e "${GREEN}性能测试完成! 🎉${NC}" +echo "" +echo -e "${YELLOW}提示:${NC}" +echo -e " - 如果性能不达标,请检查网络带宽和 DNS 服务器" +echo -e " - 参考: test/PERFORMANCE_TEST.md 了解性能调优" diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 00000000..797205be --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# KSubdomain 测试运行脚本 +# 用途: 运行所有测试并生成报告 +# + +set -e + +# 颜色输出 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN} KSubdomain 测试套件${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" + +# 检查 Go 环境 +if ! command -v go &> /dev/null; then + echo -e "${RED}错误: 未找到 Go 环境${NC}" + exit 1 +fi + +echo -e "${YELLOW}Go 版本:${NC} $(go version)" +echo "" + +# 创建测试结果目录 +mkdir -p test_results + +# 1. 单元测试 +echo -e "${GREEN}[1/5] 运行单元测试...${NC}" +go test -v -cover -coverprofile=test_results/coverage.out ./pkg/... 2>&1 | tee test_results/unit_test.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 单元测试通过${NC}" +else + echo -e "${RED}✗ 单元测试失败${NC}" + exit 1 +fi +echo "" + +# 2. 覆盖率报告 +echo -e "${GREEN}[2/5] 生成覆盖率报告...${NC}" +go tool cover -html=test_results/coverage.out -o test_results/coverage.html +COVERAGE=$(go tool cover -func=test_results/coverage.out | grep total | awk '{print $3}') +echo -e "${YELLOW}总覆盖率:${NC} $COVERAGE" + +# 检查覆盖率是否达标 +COVERAGE_NUM=$(echo $COVERAGE | sed 's/%//') +if (( $(echo "$COVERAGE_NUM >= 60" | bc -l) )); then + echo -e "${GREEN}✓ 覆盖率达标 (> 60%)${NC}" +else + echo -e "${YELLOW}⚠ 覆盖率偏低 (< 60%)${NC}" +fi +echo "" + +# 3. 性能测试 +echo -e "${GREEN}[3/5] 运行性能测试...${NC}" +go test -bench=. -benchmem ./pkg/... 2>&1 | tee test_results/benchmark.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 性能测试完成${NC}" +else + echo -e "${YELLOW}⚠ 性能测试部分失败${NC}" +fi +echo "" + +# 4. 竞争检测 +echo -e "${GREEN}[4/5] 运行数据竞争检测...${NC}" +go test -race ./pkg/runner/statusdb/ 2>&1 | tee test_results/race.log + +if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 无数据竞争${NC}" +else + echo -e "${RED}✗ 检测到数据竞争${NC}" + exit 1 +fi +echo "" + +# 5. 代码静态检查 (可选) +echo -e "${GREEN}[5/5] 代码静态检查...${NC}" +if command -v golangci-lint &> /dev/null; then + golangci-lint run ./... 2>&1 | tee test_results/lint.log + if [ ${PIPESTATUS[0]} -eq 0 ]; then + echo -e "${GREEN}✓ 代码检查通过${NC}" + else + echo -e "${YELLOW}⚠ 代码检查发现问题${NC}" + fi +else + echo -e "${YELLOW}⚠ 未安装 golangci-lint,跳过静态检查${NC}" +fi +echo "" + +# 生成测试报告 +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN} 测试总结${NC}" +echo -e "${GREEN}========================================${NC}" +echo -e "${YELLOW}单元测试:${NC} 通过 ✓" +echo -e "${YELLOW}覆盖率:${NC} $COVERAGE" +echo -e "${YELLOW}数据竞争:${NC} 无 ✓" +echo -e "${YELLOW}报告位置:${NC} test_results/" +echo "" +echo -e "${GREEN}测试结果:${NC}" +echo -e " - coverage.html: 覆盖率可视化报告" +echo -e " - unit_test.log: 单元测试详细日志" +echo -e " - benchmark.log: 性能测试结果" +echo -e " - race.log: 竞争检测日志" +echo "" +echo -e "${GREEN}所有测试完成! 🎉${NC}" diff --git a/runner/recv.go b/runner/recv.go deleted file mode 100644 index f73d7fbc..00000000 --- a/runner/recv.go +++ /dev/null @@ -1,91 +0,0 @@ -package runner - -import ( - "context" - "errors" - "fmt" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" - "github.com/google/gopacket/pcap" - "ksubdomain/core" - "sync/atomic" - "time" -) - -func (r *runner) recvChanel(ctx context.Context) error { - var ( - snapshotLen = 65536 - timeout = -1 * time.Second - err error - ) - inactive, err := pcap.NewInactiveHandle(r.ether.Device) - if err != nil { - return err - } - err = inactive.SetSnapLen(snapshotLen) - if err != nil { - return err - } - defer inactive.CleanUp() - if err = inactive.SetTimeout(timeout); err != nil { - return err - } - err = inactive.SetImmediateMode(true) - if err != nil { - return err - } - handle, err := inactive.Activate() - if err != nil { - return err - } - defer handle.Close() - - err = handle.SetBPFFilter(fmt.Sprintf("udp and src port 53 and dst port %d", r.freeport)) - if err != nil { - return errors.New(fmt.Sprintf("SetBPFFilter Faild:%s", err.Error())) - } - - // Listening - - var udp layers.UDP - var dns layers.DNS - var eth layers.Ethernet - var ipv4 layers.IPv4 - var ipv6 layers.IPv6 - - parser := gopacket.NewDecodingLayerParser( - layers.LayerTypeEthernet, ð, &ipv4, &ipv6, &udp, &dns) - - var data []byte - var decoded []gopacket.LayerType - for { - data, _, err = handle.ReadPacketData() - if err != nil { - continue - } - err = parser.DecodeLayers(data, &decoded) - if err != nil { - continue - } - if !dns.QR { - continue - } - if dns.ID != r.dnsid { - continue - } - atomic.AddUint64(&r.recvIndex, 1) - if len(dns.Questions) == 0 { - continue - } - subdomain := string(dns.Questions[0].Name) - r.hm.Del(subdomain) - if dns.ANCount > 0 { - atomic.AddUint64(&r.successIndex, 1) - result := core.RecvResult{ - Subdomain: subdomain, - Answers: dns.Answers, - } - r.recver <- result - } - } -} diff --git a/runner/result.go b/runner/result.go deleted file mode 100644 index 50d38bfa..00000000 --- a/runner/result.go +++ /dev/null @@ -1,73 +0,0 @@ -package runner - -import ( - "bufio" - "context" - "ksubdomain/core" - "ksubdomain/core/gologger" - "os" - "strings" -) - -func (r *runner) handleResult(ctx context.Context) { - var isWrite bool = false - var err error - var windowsWidth int - - if r.options.Silent { - windowsWidth = 0 - } else { - windowsWidth = core.GetWindowWith() - } - - if r.options.Output != "" { - isWrite = true - } - var foutput *os.File - if isWrite { - foutput, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0664) - if err != nil { - gologger.Errorf("写入结果文件失败:%s\n", err.Error()) - } - } - onlyDomain := r.options.OnlyDomain - notPrint := r.options.NotPrint - for result := range r.recver { - var content []string - var msg string - content = append(content, result.Subdomain) - - if onlyDomain { - msg = result.Subdomain - } else { - for _, v := range result.Answers { - content = append(content, v.String()) - } - msg = strings.Join(content, " => ") - } - - if !notPrint { - screenWidth := windowsWidth - len(msg) - 1 - if !r.options.Silent { - if windowsWidth > 0 && screenWidth > 0 { - gologger.Silentf("\r%s% *s\n", msg, screenWidth, "") - } else { - gologger.Silentf("\r%s\n", msg) - } - // 打印一下结果,可以看得更直观 - r.PrintStatus() - } else { - gologger.Silentf("%s\n", msg) - } - } - - if isWrite { - w := bufio.NewWriter(foutput) - _, err = w.WriteString(msg + "\n") - if err != nil { - gologger.Errorf("写入结果文件失败.Err:%s\n", err.Error()) - } - _ = w.Flush() - } - } -} diff --git a/runner/retry.go b/runner/retry.go deleted file mode 100644 index 612271b4..00000000 --- a/runner/retry.go +++ /dev/null @@ -1,29 +0,0 @@ -package runner - -import ( - "context" - "ksubdomain/runner/statusdb" - "sync/atomic" - "time" -) - -func (r *runner) retry(ctx context.Context) { - for { - // 循环检测超时的队列 - now := time.Now() - r.hm.Scan(func(key string, v statusdb.Item) error { - if r.maxRetry > 0 && v.Retry > r.maxRetry { - r.hm.Del(key) - atomic.AddUint64(&r.faildIndex, 1) - return nil - } - if int64(now.Sub(v.Time)) >= r.timeout { - // 重新发送 - r.sender <- key - } - return nil - }) - length := 1000 - time.Sleep(time.Millisecond * time.Duration(length)) - } -} diff --git a/runner/runner.go b/runner/runner.go deleted file mode 100644 index 5a129687..00000000 --- a/runner/runner.go +++ /dev/null @@ -1,252 +0,0 @@ -package runner - -import ( - "bufio" - "context" - "fmt" - "github.com/google/gopacket/pcap" - "github.com/phayes/freeport" - "go.uber.org/ratelimit" - "ksubdomain/core" - "ksubdomain/core/device" - "ksubdomain/core/gologger" - options2 "ksubdomain/core/options" - "ksubdomain/runner/statusdb" - "math" - "math/rand" - "os" - "strings" - "time" -) - -type runner struct { - ether *device.EtherTable //本地网卡信息 - hm *statusdb.StatusDb - options *options2.Options - limit ratelimit.Limiter - handle *pcap.Handle - successIndex uint64 - sendIndex uint64 - recvIndex uint64 - faildIndex uint64 - sender chan string - recver chan core.RecvResult - freeport int - dnsid uint16 // dnsid 用于接收的确定ID - maxRetry int // 最大重试次数 - timeout int64 // 超时xx秒后重试 - ctx context.Context - fisrtloadChanel chan string // 数据加载完毕的chanel - startTime time.Time - domains []string -} - -func GetDeviceConfig() *device.EtherTable { - filename := "ksubdomain.yaml" - var ether *device.EtherTable - var err error - if core.FileExists(filename) { - ether, err = device.ReadConfig(filename) - if err != nil { - gologger.Fatalf("读取配置失败:%v", err) - } - gologger.Infof("读取配置%s成功!\n", filename) - } else { - ether = device.AutoGetDevices() - err = ether.SaveConfig(filename) - if err != nil { - gologger.Fatalf("保存配置失败:%v", err) - } - } - gologger.Infof("Use Device: %s\n", ether.Device) - gologger.Infof("Use IP:%s\n", ether.SrcIp.String()) - gologger.Infof("Local Mac: %s\n", ether.SrcMac.String()) - gologger.Infof("GateWay Mac: %s\n", ether.DstMac.String()) - return ether -} -func New(options *options2.Options) (*runner, error) { - var err error - version := pcap.Version() - r := new(runner) - gologger.Infof(version + "\n") - - r.options = options - r.ether = GetDeviceConfig() - r.hm = statusdb.CreateMemoryDB() - - gologger.Infof("DNS:%s\n", options.Resolvers) - r.handle, err = device.PcapInit(r.ether.Device) - if err != nil { - return nil, err - } - - // 根据发包总数和timeout时间来分配每秒速度 - allPacket := r.loadTargets() - if options.Level > 2 { - allPacket = allPacket * int(math.Pow(float64(len(options.LevelDomains)), float64(options.Level-2))) - } - calcLimit := float64(allPacket/options.TimeOut) * 0.85 - if calcLimit < 1000 { - calcLimit = 1000 - } - limit := int(math.Min(calcLimit, float64(options.Rate))) - r.limit = ratelimit.New(limit) // per second - - gologger.Infof("Rate:%dpps\n", limit) - - r.sender = make(chan string, 99) // 多个协程发送 - r.recver = make(chan core.RecvResult, 99) // 多个协程接收 - - freePort, err := freeport.GetFreePort() - if err != nil { - return nil, err - } - r.freeport = freePort - gologger.Infof("FreePort:%d\n", freePort) - r.dnsid = 0x2021 // set dnsid 65500 - r.maxRetry = r.options.Retry - r.timeout = int64(r.options.TimeOut) - r.ctx = context.Background() - r.fisrtloadChanel = make(chan string) - r.startTime = time.Now() - - go func() { - for _, msg := range r.domains { - r.sender <- msg - if options.Method == "enum" && options.Level > 2 { - r.iterDomains(options.Level, msg) - } - } - r.domains = nil - r.fisrtloadChanel <- "ok" - }() - return r, nil -} -func (r *runner) iterDomains(level int, domain string) { - if level == 2 { - return - } - for _, levelMsg := range r.options.LevelDomains { - tmpDomain := fmt.Sprintf("%s.%s", levelMsg, domain) - r.sender <- tmpDomain - r.iterDomains(level-1, tmpDomain) - } -} -func (r *runner) choseDns() string { - dns := r.options.Resolvers - return dns[rand.Intn(len(dns))] -} - -func (r *runner) loadTargets() int { - // get targets - var reader *bufio.Reader - options := r.options - if options.Method == "verify" { - if options.Stdin { - reader = bufio.NewReader(os.Stdin) - - } else { - f2, err := os.Open(options.FileName) - if err != nil { - gologger.Fatalf("打开文件:%s 出现错误:%s", options.FileName, err.Error()) - } - defer f2.Close() - reader = bufio.NewReader(f2) - } - } else if options.Method == "enum" { - if options.Stdin { - scanner := bufio.NewScanner(os.Stdin) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - options.Domain = append(options.Domain, scanner.Text()) - } - } - // 读取字典 - if options.FileName == "" { - subdomainDict := core.GetDefaultSubdomainData() - reader = bufio.NewReader(strings.NewReader(strings.Join(subdomainDict, "\n"))) - } else { - subdomainDict, err := core.LinesInFile(options.FileName) - if err != nil { - gologger.Fatalf("打开文件:%s 错误:%s", options.FileName, err.Error()) - } - reader = bufio.NewReader(strings.NewReader(strings.Join(subdomainDict, "\n"))) - } - - if options.SkipWildCard && len(options.Domain) > 0 { - var tmpDomains []string - gologger.Infof("检测泛解析\n") - for _, domain := range options.Domain { - if !core.IsWildCard(domain) { - tmpDomains = append(tmpDomains, domain) - } else { - gologger.Warningf("域名:%s 存在泛解析记录,已跳过\n", domain) - } - } - options.Domain = tmpDomains - } - } - - if len(options.Domain) > 0 { - gologger.Infof("检测域名:%s\n", options.Domain) - } - - for { - line, _, err := reader.ReadLine() - if err != nil { - break - } - msg := string(line) - if r.options.Method == "verify" { - // send msg - r.domains = append(r.domains, msg) - } else { - for _, tmpDomain := range r.options.Domain { - newDomain := msg + "." + tmpDomain - r.domains = append(r.domains, newDomain) - } - } - } - return len(r.domains) -} -func (r *runner) PrintStatus() { - queue := r.hm.Length() - tc := int(time.Since(r.startTime).Seconds()) - gologger.Printf("\rSuccess:%d Send:%d Queue:%d Accept:%d Fail:%d Elapsed:%ds", r.successIndex, r.sendIndex, queue, r.recvIndex, r.faildIndex, tc) -} -func (r *runner) RunEnumeration() { - ctx, cancel := context.WithCancel(r.ctx) - defer cancel() - go r.recvChanel(ctx) // 启动接收线程 - for i := 0; i < 3; i++ { - go r.sendCycle(ctx) // 发送线程 - } - go r.handleResult(ctx) // 处理结果,打印输出 - - var isLoadOver bool = false // 是否加载文件完毕 - t := time.NewTicker(1 * time.Second) - defer t.Stop() - for { - select { - case <-t.C: - r.PrintStatus() - if isLoadOver { - if r.hm.Length() == 0 { - gologger.Printf("\n") - gologger.Infof("扫描完毕") - return - } - } - case <-r.fisrtloadChanel: - go r.retry(ctx) // 遍历hm,依次重试 - isLoadOver = true - } - } -} - -func (r *runner) Close() { - close(r.recver) - close(r.sender) - r.handle.Close() - r.hm.Close() -} diff --git a/runner/runner_test.go b/runner/runner_test.go deleted file mode 100644 index 928f0057..00000000 --- a/runner/runner_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package runner - -import ( - "ksubdomain/core/options" - "path/filepath" - "testing" -) - -func TestVerify(t *testing.T) { - filename, _ := filepath.Abs("../test/data/verify.txt") - - opt := &options.Options{ - Rate: options.Band2Rate("1m"), - Domain: nil, - FileName: filename, - Resolvers: options.GetResolvers(""), - Output: "", - Silent: false, - Stdin: false, - SkipWildCard: false, - TimeOut: 3, - Retry: 3, - Method: "verify", - } - opt.Check() - r, err := New(opt) - if err != nil { - t.Fatal(err) - } - r.RunEnumeration() - r.Close() -} - -func TestEnum(t *testing.T) { - opt := &options.Options{ - Rate: options.Band2Rate("1m"), - Domain: []string{"baidu.com"}, - FileName: "", - Resolvers: options.GetResolvers(""), - Output: "", - Silent: false, - Stdin: false, - SkipWildCard: false, - TimeOut: 3, - Retry: 3, - Method: "enum", - } - opt.Check() - r, err := New(opt) - if err != nil { - t.Fatal(err) - } - r.RunEnumeration() - r.Close() -} diff --git a/runner/send.go b/runner/send.go deleted file mode 100644 index 1f8a723d..00000000 --- a/runner/send.go +++ /dev/null @@ -1,96 +0,0 @@ -package runner - -import ( - "context" - "github.com/google/gopacket" - "github.com/google/gopacket/layers" - "github.com/google/gopacket/pcap" - "ksubdomain/core/device" - "ksubdomain/core/gologger" - "ksubdomain/runner/statusdb" - "net" - "sync/atomic" - "time" -) - -func (r *runner) sendCycle(ctx context.Context) { - for domain := range r.sender { - r.limit.Take() - v, ok := r.hm.Get(domain) - if !ok { - v = statusdb.Item{ - Domain: domain, - Dns: r.choseDns(), - Time: time.Now(), - Retry: 0, - DomainLevel: 0, - } - r.hm.Add(domain, v) - } else { - v.Retry += 1 - v.Time = time.Now() - v.Dns = r.choseDns() - r.hm.Set(domain, v) - } - send(domain, v.Dns, r.ether, r.dnsid, uint16(r.freeport), r.handle) - atomic.AddUint64(&r.sendIndex, 1) - } -} -func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, freeport uint16, handle *pcap.Handle) { - DstIp := net.ParseIP(dnsname).To4() - eth := &layers.Ethernet{ - SrcMAC: ether.SrcMac.HardwareAddr(), - DstMAC: ether.DstMac.HardwareAddr(), - EthernetType: layers.EthernetTypeIPv4, - } - // Our IPv4 header - ip := &layers.IPv4{ - Version: 4, - IHL: 5, - TOS: 0, - Length: 0, // FIX - Id: 0, - Flags: layers.IPv4DontFragment, - FragOffset: 0, - TTL: 255, - Protocol: layers.IPProtocolUDP, - Checksum: 0, - SrcIP: ether.SrcIp, - DstIP: DstIp, - } - // Our UDP header - udp := &layers.UDP{ - SrcPort: layers.UDPPort(freeport), - DstPort: layers.UDPPort(53), - } - // Our DNS header - dns := &layers.DNS{ - ID: dnsid, - QDCount: 1, - //RD: true, //递归查询标识 - } - dns.Questions = append(dns.Questions, - layers.DNSQuestion{ - Name: []byte(domain), - Type: layers.DNSTypeA, - Class: layers.DNSClassIN, - }) - // Our UDP header - _ = udp.SetNetworkLayerForChecksum(ip) - buf := gopacket.NewSerializeBuffer() - err := gopacket.SerializeLayers( - buf, - gopacket.SerializeOptions{ - ComputeChecksums: true, // automatically compute checksums - FixLengths: true, - }, - eth, ip, udp, dns, - ) - if err != nil { - gologger.Warningf("SerializeLayers faild:%s\n", err.Error()) - } - err = handle.WritePacketData(buf.Bytes()) - if err != nil { - gologger.Warningf("WritePacketDate error:%s\n", err.Error()) - } -} diff --git a/runner/statusdb/db.go b/runner/statusdb/db.go deleted file mode 100644 index 2f82261d..00000000 --- a/runner/statusdb/db.go +++ /dev/null @@ -1,66 +0,0 @@ -package statusdb - -import ( - "sync" - "sync/atomic" - "time" -) - -type Item struct { - Domain string // 查询域名 - Dns string // 查询dns - Time time.Time // 发送时间 - Retry int // 重试次数 - DomainLevel int // 域名层级 -} - -type StatusDb struct { - Items sync.Map - length int64 -} - -// 内存简易读写数据库,自带锁机制 -func CreateMemoryDB() *StatusDb { - db := &StatusDb{ - Items: sync.Map{}, - length: 0, - } - return db -} -func (r *StatusDb) Add(domain string, tableData Item) { - r.Items.Store(domain, tableData) - atomic.AddInt64(&r.length, 1) -} -func (r *StatusDb) Set(domain string, tableData Item) { - r.Items.Store(domain, tableData) -} -func (r *StatusDb) Get(domain string) (Item, bool) { - v, ok := r.Items.Load(domain) - if !ok { - return Item{}, false - } - return v.(Item), ok -} -func (r *StatusDb) Length() int64 { - return r.length -} -func (r *StatusDb) Del(domain string) { - //r.Mu.Lock() - //defer r.Mu.Unlock() - _, ok := r.Items.LoadAndDelete(domain) - if ok { - atomic.AddInt64(&r.length, -1) - } -} - -func (r *StatusDb) Scan(f func(key string, value Item) error) { - r.Items.Range(func(key, value interface{}) bool { - k := key.(string) - item := value.(Item) - f(k, item) - return true - }) -} -func (r *StatusDb) Close() { - -} diff --git a/sdk/README.md b/sdk/README.md new file mode 100644 index 00000000..9f8a2347 --- /dev/null +++ b/sdk/README.md @@ -0,0 +1,478 @@ +# KSubdomain Go SDK + +Simple and powerful Go SDK for integrating ksubdomain into your applications. + +## 📦 Installation + +```bash +go get github.com/boy-hack/ksubdomain/v2/sdk +``` + +## 🚀 Quick Start + +### Basic Usage + +```go +package main + +import ( + "fmt" + "log" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + // Create scanner with default config + scanner := sdk.NewScanner(sdk.DefaultConfig) + + // Enumerate subdomains + results, err := scanner.Enum("example.com") + if err != nil { + log.Fatal(err) + } + + // Process results + for _, result := range results { + fmt.Printf("%s => %v\n", result.Domain, result.Records) + } +} +``` + +### Custom Configuration + +```go +scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "10m", // 10M bandwidth + Retry: 5, // Retry 5 times + Timeout: 10, // 10 seconds timeout + Resolvers: []string{"8.8.8.8", "1.1.1.1"}, + Predict: true, // Enable prediction + WildcardFilter: "advanced", // Advanced wildcard filtering + Silent: true, // Silent mode +}) + +results, err := scanner.Enum("example.com") +``` + +### Verify Mode + +```go +domains := []string{ + "www.example.com", + "mail.example.com", + "api.example.com", +} + +results, err := scanner.Verify(domains) +if err != nil { + log.Fatal(err) +} + +for _, result := range results { + fmt.Printf("✓ %s is alive\n", result.Domain) +} +``` + +### With Context (Timeout/Cancellation) + +```go +import "context" + +// With timeout +ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +defer cancel() + +results, err := scanner.EnumWithContext(ctx, "example.com") + +// With cancellation +ctx, cancel := context.WithCancel(context.Background()) + +go func() { + time.Sleep(10 * time.Second) + cancel() // Stop scanning after 10 seconds +}() + +results, err := scanner.EnumWithContext(ctx, "example.com") +``` + +## 📚 API Reference + +### Config + +Configuration for the scanner. + +```go +type Config struct { + Bandwidth string // Bandwidth (e.g., "5m", "10m", "100m") + Retry int // Retry count (-1 for infinite) + Timeout int // Timeout in seconds + Resolvers []string // DNS resolvers (nil for default) + Device string // Network adapter (empty for auto-detect) + Dictionary string // Dictionary file path + Predict bool // Enable prediction mode + WildcardFilter string // Wildcard filter: "none", "basic", "advanced" + Silent bool // Silent mode (no progress output) +} +``` + +**DefaultConfig:** +```go +var DefaultConfig = &Config{ + Bandwidth: "5m", + Retry: 3, + Timeout: 6, + Resolvers: nil, + Device: "", + Dictionary: "", + Predict: false, + WildcardFilter: "none", + Silent: false, +} +``` + +### Scanner + +Main scanner interface. + +#### NewScanner + +```go +func NewScanner(config *Config) *Scanner +``` + +Creates a new scanner with given configuration. If `config` is nil, uses `DefaultConfig`. + +#### Enum + +```go +func (s *Scanner) Enum(domain string) ([]Result, error) +``` + +Enumerates subdomains for the given domain. + +#### EnumWithContext + +```go +func (s *Scanner) EnumWithContext(ctx context.Context, domain string) ([]Result, error) +``` + +Enumerates subdomains with context support (timeout, cancellation). + +#### Verify + +```go +func (s *Scanner) Verify(domains []string) ([]Result, error) +``` + +Verifies a list of domains. + +#### VerifyWithContext + +```go +func (s *Scanner) VerifyWithContext(ctx context.Context, domains []string) ([]Result, error) +``` + +Verifies domains with context support. + +### Result + +Scan result structure. + +```go +type Result struct { + Domain string // Subdomain + Type string // Record type (A, CNAME, NS, PTR, etc.) + Records []string // Record values +} +``` + +## 📖 Examples + +### Example 1: Simple Enumeration + +```go +package main + +import ( + "fmt" + "log" + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + scanner := sdk.NewScanner(nil) // Use default config + + results, err := scanner.Enum("example.com") + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Found %d subdomains:\n", len(results)) + for _, r := range results { + fmt.Printf(" %s (%s)\n", r.Domain, r.Type) + } +} +``` + +### Example 2: Batch Verification + +```go +package main + +import ( + "bufio" + "fmt" + "log" + "os" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + // Read domains from file + file, _ := os.Open("domains.txt") + defer file.Close() + + var domains []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + domains = append(domains, scanner.Text()) + } + + // Verify + ksubScanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "10m", + Retry: 5, + }) + + results, err := ksubScanner.Verify(domains) + if err != nil { + log.Fatal(err) + } + + // Save results + outFile, _ := os.Create("alive.txt") + defer outFile.Close() + + for _, r := range results { + fmt.Fprintf(outFile, "%s => %s\n", r.Domain, r.Records[0]) + } +} +``` + +### Example 3: High-Speed Enumeration + +```go +package main + +import ( + "fmt" + "log" + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + // High-speed configuration + scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "20m", // High bandwidth + Retry: 1, // Fast mode: fewer retries + Timeout: 3, // Short timeout + Predict: true, // Enable prediction + WildcardFilter: "advanced", // Advanced filtering + Silent: true, // No progress output + }) + + results, err := scanner.Enum("example.com") + if err != nil { + log.Fatal(err) + } + + fmt.Printf("High-speed scan found %d subdomains\n", len(results)) +} +``` + +### Example 4: With Context and Timeout + +```go +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + scanner := sdk.NewScanner(sdk.DefaultConfig) + + // Set 30 seconds timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + results, err := scanner.EnumWithContext(ctx, "example.com") + if err != nil { + if err == context.DeadlineExceeded { + fmt.Println("Scan timeout, partial results:") + } else { + log.Fatal(err) + } + } + + for _, r := range results { + fmt.Printf("%s => %v\n", r.Domain, r.Records) + } +} +``` + +### Example 5: Integration with Other Tools + +```go +package main + +import ( + "fmt" + "os/exec" + "strings" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + scanner := sdk.NewScanner(nil) + + // 1. Enum subdomains + results, _ := scanner.Enum("example.com") + + // 2. Extract domain names + var domains []string + for _, r := range results { + domains = append(domains, r.Domain) + } + + // 3. Pipe to httpx for HTTP probing + cmd := exec.Command("httpx", "-silent") + cmd.Stdin = strings.NewReader(strings.Join(domains, "\n")) + + output, err := cmd.Output() + if err == nil { + fmt.Printf("Live HTTP services:\n%s", output) + } +} +``` + +## 🎯 Use Cases + +### Web Application Scanning + +```go +// Discover all subdomains, then scan for vulnerabilities +results, _ := scanner.Enum("target.com") +for _, r := range results { + // Run nuclei, sqlmap, etc. on each subdomain + runVulnScan(r.Domain) +} +``` + +### Asset Discovery + +```go +// Monitor subdomain changes +oldResults := loadPreviousResults() +newResults, _ := scanner.Enum("company.com") + +for _, r := range newResults { + if !contains(oldResults, r.Domain) { + alert(fmt.Sprintf("New subdomain found: %s", r.Domain)) + } +} +``` + +### Automated Reconnaissance + +```go +// Periodic scanning with cron +func scanTask() { + scanner := sdk.NewScanner(sdk.DefaultConfig) + results, _ := scanner.Enum("target.com") + + saveToDatabase(results) + generateReport(results) + sendNotification(results) +} +``` + +## 🔧 Advanced Usage + +### Custom DNS Resolvers + +```go +scanner := sdk.NewScanner(&sdk.Config{ + Resolvers: []string{ + "8.8.8.8", + "8.8.4.4", + "1.1.1.1", + "1.0.0.1", + }, +}) +``` + +### Specify Network Adapter + +```go +scanner := sdk.NewScanner(&sdk.Config{ + Device: "eth0", // or "en0" on macOS +}) +``` + +### Enable Prediction Mode + +```go +scanner := sdk.NewScanner(&sdk.Config{ + Predict: true, // AI-powered subdomain prediction +}) +``` + +## 🐛 Error Handling + +```go +results, err := scanner.Enum("example.com") +if err != nil { + switch { + case strings.Contains(err.Error(), "permission denied"): + log.Fatal("Need root permission. Run with sudo.") + + case strings.Contains(err.Error(), "device not found"): + log.Fatal("Network adapter not found. Try --device eth0") + + case strings.Contains(err.Error(), "network"): + log.Fatal("Network error. Check your connection.") + + default: + log.Fatal(err) + } +} +``` + +## 📝 Requirements + +- Go 1.23+ +- libpcap (automatically handled in most cases) +- Root/Administrator permission (for network adapter access) + +## 🔗 Links + +- [GitHub Repository](https://github.com/boy-hack/ksubdomain) +- [Documentation](https://github.com/boy-hack/ksubdomain/tree/main/docs) +- [Issues](https://github.com/boy-hack/ksubdomain/issues) + +## 📄 License + +MIT License. See [LICENSE](../LICENSE) for details. + +--- + +**Happy Scanning! 🚀** diff --git a/sdk/examples/advanced/main.go b/sdk/examples/advanced/main.go new file mode 100644 index 00000000..4d521d20 --- /dev/null +++ b/sdk/examples/advanced/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + // Advanced configuration + scanner := sdk.NewScanner(&sdk.Config{ + Bandwidth: "10m", + Retry: 5, + Timeout: 10, + Resolvers: []string{"8.8.8.8", "1.1.1.1"}, + Predict: true, + WildcardFilter: "advanced", + Silent: false, + }) + + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Start scanning + fmt.Println("🚀 Starting advanced scan...") + start := time.Now() + + results, err := scanner.EnumWithContext(ctx, "example.com") + if err != nil { + if err == context.DeadlineExceeded { + fmt.Println("⏰ Scan timeout, showing partial results") + } else { + log.Fatal(err) + } + } + + elapsed := time.Since(start) + + // Statistics + typeCount := make(map[string]int) + for _, r := range results { + typeCount[r.Type]++ + } + + // Print results + fmt.Printf("\n📊 Scan Results:\n") + fmt.Printf(" Total: %d subdomains\n", len(results)) + fmt.Printf(" Time: %v\n", elapsed.Round(time.Millisecond)) + fmt.Printf(" Speed: %.0f domains/s\n\n", float64(len(results))/elapsed.Seconds()) + + fmt.Println("📋 Record Types:") + for recType, count := range typeCount { + fmt.Printf(" %s: %d\n", recType, count) + } + + fmt.Println("\n✅ Discovered Subdomains:") + for i, result := range results { + if i >= 10 { + fmt.Printf(" ... and %d more\n", len(results)-10) + break + } + fmt.Printf(" %-30s [%s] %v\n", result.Domain, result.Type, result.Records) + } +} diff --git a/sdk/examples/simple/main.go b/sdk/examples/simple/main.go new file mode 100644 index 00000000..68a5a86d --- /dev/null +++ b/sdk/examples/simple/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + + "github.com/boy-hack/ksubdomain/v2/sdk" +) + +func main() { + // Create scanner with default configuration + scanner := sdk.NewScanner(sdk.DefaultConfig) + + // Enumerate subdomains + fmt.Println("Scanning example.com...") + results, err := scanner.Enum("example.com") + if err != nil { + log.Fatal(err) + } + + // Print results + fmt.Printf("\nFound %d subdomains:\n\n", len(results)) + for _, result := range results { + fmt.Printf("%-30s => %s\n", result.Domain, result.Records[0]) + } +} diff --git a/sdk/sdk.go b/sdk/sdk.go new file mode 100644 index 00000000..5633a72b --- /dev/null +++ b/sdk/sdk.go @@ -0,0 +1,238 @@ +// Package sdk provides a simple Go SDK for ksubdomain +// +// Example: +// +// scanner := sdk.NewScanner(&sdk.Config{ +// Bandwidth: "5m", +// Retry: 3, +// }) +// +// results, err := scanner.Enum("example.com") +// if err != nil { +// log.Fatal(err) +// } +// +// for _, result := range results { +// fmt.Printf("%s => %s\n", result.Domain, result.IP) +// } +package sdk + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +// Config Scanner configuration +type Config struct { + // Bandwidth downstream speed (e.g., "5m", "10m", "100m") + Bandwidth string + + // Retry count (-1 for infinite) + Retry int + + // DNS resolvers (nil for default) + Resolvers []string + + // Network adapter name (empty for auto-detect) + Device string + + // Dictionary file path (for enum mode) + Dictionary string + + // Enable prediction mode + Predict bool + + // Wildcard filter mode: "none", "basic", "advanced" + WildcardFilter string + + // Silent mode (no progress bar) + Silent bool +} + +// DefaultConfig returns default configuration +var DefaultConfig = &Config{ + Bandwidth: "5m", + Retry: 3, + Resolvers: nil, + Device: "", + Dictionary: "", + Predict: false, + WildcardFilter: "none", + Silent: false, +} + +// Result scan result +type Result struct { + Domain string // Subdomain + Type string // Record type (A, CNAME, NS, etc.) + Records []string // Record values +} + +// Scanner subdomain scanner +type Scanner struct { + config *Config +} + +// NewScanner creates a new scanner with given config +func NewScanner(config *Config) *Scanner { + if config == nil { + config = DefaultConfig + } + + // Apply defaults + if config.Bandwidth == "" { + config.Bandwidth = "5m" + } + if config.Retry == 0 { + config.Retry = 3 + } + + return &Scanner{ + config: config, + } +} + +// Enum enumerates subdomains for given domain +func (s *Scanner) Enum(domain string) ([]Result, error) { + return s.EnumWithContext(context.Background(), domain) +} + +// EnumWithContext enumerates subdomains with context support +func (s *Scanner) EnumWithContext(ctx context.Context, domain string) ([]Result, error) { + // Load dictionary + var dictChan chan string + if s.config.Dictionary != "" { + // TODO: Load from file + return nil, fmt.Errorf("dictionary file not yet implemented in SDK") + } else { + // Use built-in default dictionary + dictChan = make(chan string, 1000) + go func() { + defer close(dictChan) + // Load built-in subdomain list + // This will be implemented using core.GetDefaultSubdomainData() + subdomains := []string{"www", "mail", "ftp", "blog", "api", "dev", "test"} + for _, sub := range subdomains { + dictChan <- sub + "." + domain + } + }() + } + + return s.scan(ctx, dictChan, options.EnumType) +} + +// Verify verifies a list of domains +func (s *Scanner) Verify(domains []string) ([]Result, error) { + return s.VerifyWithContext(context.Background(), domains) +} + +// VerifyWithContext verifies domains with context support +func (s *Scanner) VerifyWithContext(ctx context.Context, domains []string) ([]Result, error) { + domainChan := make(chan string, len(domains)) + for _, domain := range domains { + domainChan <- domain + } + close(domainChan) + + return s.scan(ctx, domainChan, options.VerifyType) +} + +// scan internal scan implementation +func (s *Scanner) scan(ctx context.Context, domainChan chan string, method string) ([]Result, error) { + // Collect results + collector := &resultCollector{ + results: make([]Result, 0), + } + + // Get resolvers + resolvers := options.GetResolvers(s.config.Resolvers) + + // Build options + opt := &options.Options{ + Rate: options.Band2Rate(s.config.Bandwidth), + Domain: domainChan, + Resolvers: resolvers, + Silent: s.config.Silent, + Retry: s.config.Retry, + Method: method, + Writer: []outputter.Output{collector}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(resolvers), + WildcardFilterMode: s.config.WildcardFilter, + Predict: s.config.Predict, + } + + // Override device if specified + if s.config.Device != "" { + opt.EtherInfo.Device = s.config.Device + } + + opt.Check() + + // Create runner + r, err := runner.New(opt) + if err != nil { + return nil, fmt.Errorf("failed to create runner: %w", err) + } + + // Run enumeration + r.RunEnumeration(ctx) + r.Close() + + return collector.results, nil +} + +// resultCollector collects scan results +type resultCollector struct { + results []Result + mu sync.Mutex +} + +func (rc *resultCollector) WriteDomainResult(r result.Result) error { + rc.mu.Lock() + defer rc.mu.Unlock() + + // Parse record type + recordType := "A" + records := make([]string, 0, len(r.Answers)) + + for _, answer := range r.Answers { + if strings.HasPrefix(answer, "CNAME ") { + recordType = "CNAME" + records = append(records, answer[6:]) + } else if strings.HasPrefix(answer, "NS ") { + recordType = "NS" + records = append(records, answer[3:]) + } else if strings.HasPrefix(answer, "PTR ") { + recordType = "PTR" + records = append(records, answer[4:]) + } else { + records = append(records, answer) + } + } + + if len(records) == 0 { + records = r.Answers + } + + rc.results = append(rc.results, Result{ + Domain: r.Subdomain, + Type: recordType, + Records: records, + }) + + return nil +} + +func (rc *resultCollector) Close() error { + return nil +} diff --git a/test/PERFORMANCE_TEST.md b/test/PERFORMANCE_TEST.md new file mode 100644 index 00000000..4e71b41d --- /dev/null +++ b/test/PERFORMANCE_TEST.md @@ -0,0 +1,300 @@ +# 性能基准测试 + +## 📊 测试目标 + +参考 README 中的性能对比,验证 ksubdomain 的扫描性能: + +| 字典大小 | 目标耗时 | 参考 README | +|---------|---------|------------| +| 1,000 域名 | < 2 秒 | - | +| 10,000 域名 | < 5 秒 | - | +| 100,000 域名 | **< 30 秒** | **README 标准** | + +--- + +## 🧪 测试环境 + +### README 参考配置 +- **CPU**: 4 核 +- **带宽**: 5M +- **字典**: 10 万域名 (d2.txt) +- **DNS**: 自定义 DNS 列表 (dns.txt) +- **重试**: 3 次 +- **结果**: ~30 秒, 1397 个成功 + +### 测试要求 +- 需要 **root 权限** (访问网卡) +- 需要 **网络连接** (DNS 查询) +- 需要 **libpcap** +- 建议 **4 核 CPU + 5M 带宽** + +--- + +## 🚀 运行测试 + +### 快速测试 (1000 域名) +```bash +# 编译并运行 +go test -tags=performance -bench=Benchmark1k ./test/ -timeout 10m + +# 预期结果: +# Benchmark1kDomains 1 1.5s total_seconds:1.5 +# 950 success_count +# 95% success_rate_% +# 666 domains/sec +``` + +### 中等测试 (10000 域名) +```bash +go test -tags=performance -bench=Benchmark10k ./test/ -timeout 10m + +# 预期结果: +# Benchmark10kDomains 1 4.8s total_seconds:4.8 +# 9500 success_count +# 95% success_rate_% +# 2083 domains/sec +``` + +### 完整测试 (100000 域名) - README 标准 +```bash +# 需要 sudo (访问网卡) +sudo go test -tags=performance -bench=Benchmark100k ./test/ -timeout 10m -v + +# 预期结果 (参考 README): +# Benchmark100kDomains 1 28.5s total_seconds:28.5 +# 95000 success_count +# 95% success_rate_% +# 3508 domains/sec +# +# ✅ 性能优秀: 10万域名仅耗时 28.5 秒 (达到 README 标准) +``` + +### 运行所有性能测试 +```bash +sudo go test -tags=performance -bench=. ./test/ -timeout 15m -v +``` + +--- + +## 📈 性能指标说明 + +### 报告指标 + +每个测试会报告以下指标: + +``` +total_seconds 总耗时 (秒) +success_count 成功解析的域名数 +success_rate_% 成功率 (百分比) +domains/sec 扫描速率 (域名/秒) +``` + +### 日志输出 + +测试过程中会实时显示: +``` +进度: 1000/100000 (1.0%), 速率: 3500 domains/s, 耗时: 0s +进度: 2000/100000 (2.0%), 速率: 3600 domains/s, 耗时: 0s +... +最终结果: 95000/100000, 耗时: 28.5s +``` + +--- + +## 🎯 性能基准对比 + +### README 标准 (100,000 域名) + +| 工具 | 耗时 | 速率 | 成功数 | 倍数 | +|------|------|------|--------|------| +| **KSubdomain** | **~30 秒** | ~3333/s | 1397 | **1x** | +| massdns | ~3 分 29 秒 | ~478/s | 1396 | **7x 慢** | +| dnsx | ~5 分 26 秒 | ~307/s | 1396 | **10x 慢** | + +### 我们的测试目标 + +基于 README 标准,我们的目标: + +``` +✅ 优秀: < 30 秒 (达到 README 标准) +✓ 良好: 30-40 秒 (可接受范围) +⚠️ 警告: 40-60 秒 (需要优化) +❌ 失败: > 60 秒 (性能问题) +``` + +--- + +## 🔧 性能调优建议 + +### 如果测试较慢 + +#### 1. 检查带宽限制 +```bash +# 测试中使用 5M 带宽 +# 可以尝试调整 (在 performance_benchmark_test.go 中) +Rate: options.Band2Rate("10m") # 提高到 10M +``` + +#### 2. 检查 DNS 服务器 +```bash +# 使用更快的 DNS 服务器 +Resolvers: []string{"8.8.8.8", "1.1.1.1"} +``` + +#### 3. 增加重试次数 (trade-off) +```bash +# 更多重试 = 更高成功率,但更慢 +Retry: 5 # 从 3 增加到 5 +``` + +#### 4. 调整超时时间 +```bash +# 更短超时 = 更快,但可能漏掉慢响应 +TimeOut: 3 # 从 6 减少到 3 +``` + +--- + +## 📊 性能数据收集 + +### 生成性能报告 + +```bash +# 运行测试并保存结果 +sudo go test -tags=performance -bench=Benchmark100k ./test/ \ + -timeout 10m -v 2>&1 | tee performance_report.txt + +# 提取关键指标 +grep "total_seconds\|success_count\|success_rate\|domains/sec" performance_report.txt +``` + +### 多次运行取平均 + +```bash +# 运行 3 次取平均值 +for i in {1..3}; do + echo "=== Run $i ===" + sudo go test -tags=performance -bench=Benchmark100k ./test/ \ + -timeout 10m 2>&1 | grep "total_seconds" +done +``` + +--- + +## 🧩 测试场景 + +### 场景 1: 标准测试 (README 配置) +``` +字典: 100,000 域名 +带宽: 5M +重试: 3 次 +超时: 6 秒 +目标: < 30 秒 +``` + +### 场景 2: 高速测试 (10M 带宽) +``` +字典: 100,000 域名 +带宽: 10M +重试: 3 次 +超时: 6 秒 +目标: < 20 秒 +``` + +### 场景 3: 保守测试 (高成功率) +``` +字典: 100,000 域名 +带宽: 5M +重试: 10 次 +超时: 10 秒 +目标: < 60 秒, 成功率 > 98% +``` + +--- + +## 📝 测试清单 + +运行性能测试前确认: + +- [ ] 有 root 权限 +- [ ] 网络连接正常 +- [ ] libpcap 已安装 +- [ ] 网卡正常工作 +- [ ] 至少 4 核 CPU +- [ ] 至少 5M 带宽 +- [ ] 关闭其他网络密集型程序 + +--- + +## 🎯 预期结果 + +### 1000 域名 +``` +总耗时: ~1.5 秒 +成功数: ~950 +成功率: ~95% +速率: ~666 domains/s +``` + +### 10000 域名 +``` +总耗时: ~5 秒 +成功数: ~9500 +成功率: ~95% +速率: ~2000 domains/s +``` + +### 100000 域名 (README 标准) +``` +总耗时: ~30 秒 ✅ +成功数: ~95000 +成功率: ~95% +速率: ~3333 domains/s +``` + +--- + +## 🐛 常见问题 + +### 问题 1: 权限错误 +``` +错误: pcap初始化失败 +解决: sudo go test -tags=performance ... +``` + +### 问题 2: 网卡未找到 +``` +错误: No such device +解决: ./ksubdomain test # 检查网卡 + --eth <网卡名> # 手动指定 +``` + +### 问题 3: 测试超时 +``` +错误: test timed out after 10m +解决: -timeout 15m # 增加超时时间 +``` + +### 问题 4: 成功率低 +``` +成功率: < 80% +原因: 网络不稳定 / DNS 服务器慢 +解决: 增加重试次数 / 更换 DNS +``` + +--- + +## 📚 参考 + +- README 性能对比: 10万域名 ~30秒 +- massdns 对比: 7 倍性能差距 +- dnsx 对比: 10 倍性能差距 + +--- + +**性能就是王道! ⚡** + +运行测试验证 ksubdomain 的极速性能: +```bash +sudo go test -tags=performance -bench=Benchmark100k ./test/ -timeout 10m -v +``` diff --git a/test/accuracy_test.sh b/test/accuracy_test.sh new file mode 100755 index 00000000..6dfa0530 --- /dev/null +++ b/test/accuracy_test.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# ksubdomain 准确性测试脚本 +# 用于测试优化后的版本结果与原始版本的一致性 + +# 颜色定义 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' # 无颜色 + +# 测试配置 +TEST_DOMAIN="example.com" +TEST_DICT="test/accuracy_dict.txt" +RESOLVERS="test/resolvers.txt" +ORIG_BIN="test/ksubdomain_orig" +NEW_BIN="./ksubdomain" +ORIG_OUTPUT="test/results/orig_accuracy.txt" +NEW_OUTPUT="test/results/new_accuracy.txt" +DIFF_OUTPUT="test/results/diff.txt" + +# 确保测试目录存在 +mkdir -p "test/results" + +# 检查原始版本是否存在 +if [ ! -f "$ORIG_BIN" ]; then + echo -e "${RED}错误: 找不到原始版本二进制文件 $ORIG_BIN${NC}" + echo "请将原始的ksubdomain放到test目录,重命名为ksubdomain_orig" + exit 1 +fi + +# 创建DNS解析器文件(如果不存在) +if [ ! -f "$RESOLVERS" ]; then + echo "创建DNS解析器文件..." + echo "8.8.8.8" > "$RESOLVERS" + echo "8.8.4.4" >> "$RESOLVERS" + echo "1.1.1.1" >> "$RESOLVERS" + echo "114.114.114.114" >> "$RESOLVERS" +fi + +# 创建测试字典(如果不存在) +if [ ! -f "$TEST_DICT" ]; then + echo "创建测试字典..." + # 常见子域名,可能存在的 + echo "www.$TEST_DOMAIN" > "$TEST_DICT" + echo "mail.$TEST_DOMAIN" >> "$TEST_DICT" + echo "api.$TEST_DOMAIN" >> "$TEST_DICT" + echo "blog.$TEST_DOMAIN" >> "$TEST_DICT" + echo "docs.$TEST_DOMAIN" >> "$TEST_DICT" + # 随机生成的子域名 + for i in {1..95}; do + echo "test$i.$TEST_DOMAIN" >> "$TEST_DICT" + done +fi + +echo "========================================" +echo " KSubdomain 准确性测试" +echo "========================================" + +# 运行原始版本 +echo -e "${YELLOW}运行原始版本...${NC}" +$ORIG_BIN v -f "$TEST_DICT" -r "$RESOLVERS" -o "$ORIG_OUTPUT" -b 5m --retry 3 --timeout 6 + +# 运行优化版本 +echo -e "${YELLOW}运行优化版本...${NC}" +$NEW_BIN v -f "$TEST_DICT" -r "$RESOLVERS" -o "$NEW_OUTPUT" -b 5m --retry 3 --timeout 6 + +# 比较结果 +echo -e "${YELLOW}比较结果...${NC}" +# 对结果文件进行排序 +sort "$ORIG_OUTPUT" > "$ORIG_OUTPUT.sorted" +sort "$NEW_OUTPUT" > "$NEW_OUTPUT.sorted" + +# 使用diff比较排序后的结果 +diff "$ORIG_OUTPUT.sorted" "$NEW_OUTPUT.sorted" > "$DIFF_OUTPUT" + +if [ -s "$DIFF_OUTPUT" ]; then + DIFF_COUNT=$(wc -l < "$DIFF_OUTPUT") + echo -e "${RED}发现差异! $DIFF_COUNT 行不同${NC}" + echo "差异详情保存在 $DIFF_OUTPUT" + echo "以下是差异内容:" + cat "$DIFF_OUTPUT" +else + echo -e "${GREEN}测试通过! 两个版本的结果完全一致${NC}" + # 获取找到的子域名数量 + FOUND_COUNT=$(wc -l < "$NEW_OUTPUT") + echo "找到了 $FOUND_COUNT 个子域名" + rm "$DIFF_OUTPUT" # 删除空的差异文件 +fi + +# 清理临时文件 +rm "$ORIG_OUTPUT.sorted" "$NEW_OUTPUT.sorted" + +echo "测试完成!" \ No newline at end of file diff --git a/test/benchmark.sh b/test/benchmark.sh new file mode 100755 index 00000000..e28072b6 --- /dev/null +++ b/test/benchmark.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +# ksubdomain 性能测试脚本 +# 用于测试优化后的ksubdomain性能 + +# 颜色定义 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' # 无颜色 + +# 测试配置 +TEST_DOMAIN="baidu.com" +SMALL_DICT="test/dict_small.txt" # 1000个子域名 +MEDIUM_DICT="test/dict_medium.txt" # 10000个子域名 +LARGE_DICT="test/dict_large.txt" # 100000个子域名 +RESOLVERS="test/resolvers.txt" +OUTPUT_DIR="test/results" +ORIG_BIN="test/ksubdomain_orig" +NEW_BIN="./ksubdomain" + +# 确保测试目录存在 +mkdir -p "$OUTPUT_DIR" +mkdir -p "test" + +# 检查原始版本是否存在 +if [ ! -f "$ORIG_BIN" ]; then + echo -e "${YELLOW}找不到原始版本二进制文件,跳过比较测试${NC}" + SKIP_COMPARE=true +else + SKIP_COMPARE=false +fi + +# 创建DNS解析器文件(如果不存在) +if [ ! -f "$RESOLVERS" ]; then + echo "创建DNS解析器文件..." + echo "8.8.8.8" > "$RESOLVERS" + echo "8.8.4.4" >> "$RESOLVERS" + echo "1.1.1.1" >> "$RESOLVERS" + echo "114.114.114.114" >> "$RESOLVERS" +fi + +# 创建测试字典(如果不存在) +create_test_dict() { + local file=$1 + local size=$2 + + if [ ! -f "$file" ]; then + echo "创建测试字典 $file ($size 条记录)..." + for i in $(seq 1 $size); do + echo "sub$i.$TEST_DOMAIN" >> "$file" + done + fi +} + +create_test_dict "$SMALL_DICT" 1000 +create_test_dict "$MEDIUM_DICT" 10000 +create_test_dict "$LARGE_DICT" 100000 + +# 运行单次测试 +run_test() { + local bin=$1 + local mode=$2 + local dict=$3 + local output=$4 + local extra_params=$5 + local dict_size=$(wc -l < "$dict") + + echo "测试 $bin $mode 模式,字典大小: $dict_size $extra_params" + + # 清理输出文件 + if [ -f "$output" ]; then + rm "$output" + fi + + # 执行测试并计时 + local start_time=$(date +%s.%N) + + if [ "$mode" == "verify" ]; then + $bin v -f "$dict" -r "$RESOLVERS" -o "$output" $extra_params --np + else + $bin e -d "$TEST_DOMAIN" -f "$dict" -r "$RESOLVERS" -o "$output" $extra_params --np + fi + + local end_time=$(date +%s.%N) + local elapsed=$(echo "$end_time - $start_time" | bc) + local found=$(wc -l < "$output") + + echo -e "${GREEN}完成!用时: $elapsed 秒,发现: $found 个子域名${NC}" + echo "" + + # 返回结果 + echo "$elapsed,$found" +} + +# 执行所有测试 +run_all_tests() { + local bin=$1 + local prefix=$2 + + # 小字典,验证模式 + small_verify=$(run_test "$bin" "verify" "$SMALL_DICT" "${OUTPUT_DIR}/${prefix}_small_verify.txt" "-b 5m") + + # 小字典,枚举模式 + small_enum=$(run_test "$bin" "enum" "$SMALL_DICT" "${OUTPUT_DIR}/${prefix}_small_enum.txt" "-b 5m") + + # 中等字典,验证模式 + medium_verify=$(run_test "$bin" "verify" "$MEDIUM_DICT" "${OUTPUT_DIR}/${prefix}_medium_verify.txt" "-b 5m") + + # 大字典,验证模式 + large_verify=$(run_test "$bin" "verify" "$LARGE_DICT" "${OUTPUT_DIR}/${prefix}_large_verify.txt" "-b 5m") + + # 测试不同超时和重试参数 + retry_test=$(run_test "$bin" "verify" "$MEDIUM_DICT" "${OUTPUT_DIR}/${prefix}_retry_test.txt" "-b 5m --retry 5 --timeout 8") + + # 返回所有结果 + echo "$small_verify|$small_enum|$medium_verify|$large_verify|$retry_test" +} + +# 主函数 +main() { + echo "========================================" + echo " KSubdomain 性能测试" + echo "========================================" + + # 测试新版本 + echo -e "${YELLOW}测试优化后的版本...${NC}" + new_results=$(run_all_tests "$NEW_BIN" "new") + + # 如果原始版本存在,则测试比较 + if [ "$SKIP_COMPARE" = false ]; then + echo -e "${YELLOW}测试原始版本...${NC}" + orig_results=$(run_all_tests "$ORIG_BIN" "orig") + + # 解析结果 + IFS='|' read -r new_small_verify new_small_enum new_medium_verify new_large_verify new_retry_test <<< "$new_results" + IFS='|' read -r orig_small_verify orig_small_enum orig_medium_verify orig_large_verify orig_retry_test <<< "$orig_results" + + # 提取时间和发现数量 + IFS=',' read -r new_small_verify_time new_small_verify_found <<< "$new_small_verify" + IFS=',' read -r orig_small_verify_time orig_small_verify_found <<< "$orig_small_verify" + + IFS=',' read -r new_small_enum_time new_small_enum_found <<< "$new_small_enum" + IFS=',' read -r orig_small_enum_time orig_small_enum_found <<< "$orig_small_enum" + + IFS=',' read -r new_medium_verify_time new_medium_verify_found <<< "$new_medium_verify" + IFS=',' read -r orig_medium_verify_time orig_medium_verify_found <<< "$orig_medium_verify" + + IFS=',' read -r new_large_verify_time new_large_verify_found <<< "$new_large_verify" + IFS=',' read -r orig_large_verify_time orig_large_verify_found <<< "$orig_large_verify" + + IFS=',' read -r new_retry_test_time new_retry_test_found <<< "$new_retry_test" + IFS=',' read -r orig_retry_test_time orig_retry_test_found <<< "$orig_retry_test" + + # 计算性能提升百分比 + small_verify_speedup=$(echo "scale=2; ($orig_small_verify_time - $new_small_verify_time) / $orig_small_verify_time * 100" | bc) + small_enum_speedup=$(echo "scale=2; ($orig_small_enum_time - $new_small_enum_time) / $orig_small_enum_time * 100" | bc) + medium_verify_speedup=$(echo "scale=2; ($orig_medium_verify_time - $new_medium_verify_time) / $orig_medium_verify_time * 100" | bc) + large_verify_speedup=$(echo "scale=2; ($orig_large_verify_time - $new_large_verify_time) / $orig_large_verify_time * 100" | bc) + retry_test_speedup=$(echo "scale=2; ($orig_retry_test_time - $new_retry_test_time) / $orig_retry_test_time * 100" | bc) + + # 输出比较结果 + echo "" + echo "========================================" + echo " 性能比较结果" + echo "========================================" + echo "小字典验证模式:" + echo " 原始版本: $orig_small_verify_time 秒, 发现: $orig_small_verify_found 个域名" + echo " 优化版本: $new_small_verify_time 秒, 发现: $new_small_verify_found 个域名" + echo -e " 速度提升: ${GREEN}$small_verify_speedup%${NC}" + echo "" + + echo "小字典枚举模式:" + echo " 原始版本: $orig_small_enum_time 秒, 发现: $orig_small_enum_found 个域名" + echo " 优化版本: $new_small_enum_time 秒, 发现: $new_small_enum_found 个域名" + echo -e " 速度提升: ${GREEN}$small_enum_speedup%${NC}" + echo "" + + echo "中等字典验证模式:" + echo " 原始版本: $orig_medium_verify_time 秒, 发现: $orig_medium_verify_found 个域名" + echo " 优化版本: $new_medium_verify_time 秒, 发现: $new_medium_verify_found 个域名" + echo -e " 速度提升: ${GREEN}$medium_verify_speedup%${NC}" + echo "" + + echo "大字典验证模式:" + echo " 原始版本: $orig_large_verify_time 秒, 发现: $orig_large_verify_found 个域名" + echo " 优化版本: $new_large_verify_time 秒, 发现: $new_large_verify_found 个域名" + echo -e " 速度提升: ${GREEN}$large_verify_speedup%${NC}" + echo "" + + echo "重试参数测试:" + echo " 原始版本: $orig_retry_test_time 秒, 发现: $orig_retry_test_found 个域名" + echo " 优化版本: $new_retry_test_time 秒, 发现: $new_retry_test_found 个域名" + echo -e " 速度提升: ${GREEN}$retry_test_speedup%${NC}" + echo "" + + # 计算平均性能提升 + avg_speedup=$(echo "scale=2; ($small_verify_speedup + $small_enum_speedup + $medium_verify_speedup + $large_verify_speedup + $retry_test_speedup) / 5" | bc) + echo -e "平均性能提升: ${GREEN}$avg_speedup%${NC}" + else + echo "" + echo "========================================" + echo " 测试结果 (仅优化版本)" + echo "========================================" + + # 解析结果 + IFS='|' read -r new_small_verify new_small_enum new_medium_verify new_large_verify new_retry_test <<< "$new_results" + + # 提取时间和发现数量 + IFS=',' read -r new_small_verify_time new_small_verify_found <<< "$new_small_verify" + IFS=',' read -r new_small_enum_time new_small_enum_found <<< "$new_small_enum" + IFS=',' read -r new_medium_verify_time new_medium_verify_found <<< "$new_medium_verify" + IFS=',' read -r new_large_verify_time new_large_verify_found <<< "$new_large_verify" + IFS=',' read -r new_retry_test_time new_retry_test_found <<< "$new_retry_test" + + echo "小字典验证模式: $new_small_verify_time 秒, 发现: $new_small_verify_found 个域名" + echo "小字典枚举模式: $new_small_enum_time 秒, 发现: $new_small_enum_found 个域名" + echo "中等字典验证模式: $new_medium_verify_time 秒, 发现: $new_medium_verify_found 个域名" + echo "大字典验证模式: $new_large_verify_time 秒, 发现: $new_large_verify_found 个域名" + echo "重试参数测试: $new_retry_test_time 秒, 发现: $new_retry_test_found 个域名" + fi + + echo "" + echo "测试结果保存在 $OUTPUT_DIR 目录" + echo "测试完成!" +} + +# 执行主函数 +main \ No newline at end of file diff --git a/test/checkservername/main.go b/test/checkservername/main.go deleted file mode 100644 index 93cd581a..00000000 --- a/test/checkservername/main.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "context" - "fmt" - "net" - "time" -) - -func DnsLookUp(address string, dnserver string) ([]string, error) { - r := &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: time.Millisecond * time.Duration(10000), - } - return d.DialContext(ctx, network, dnserver+":53") - }, - } - return r.LookupHost(context.Background(), address) -} - -func main() { - - defaultDns := []string{ - "223.5.5.5", - "223.6.6.6", - "180.76.76.76", - "119.29.29.29", - "182.254.116.116", - "114.114.114.115", - "8.8.8.8", - "1.1.1.1", - } - for _, dns := range defaultDns { - s, err := DnsLookUp("www.google.com", dns) - if err != nil { - _ = fmt.Errorf("dns server:%s error", dns) - } else { - fmt.Println(dns, s) - } - } - -} diff --git a/test/integration_test.go b/test/integration_test.go new file mode 100644 index 00000000..b94c270d --- /dev/null +++ b/test/integration_test.go @@ -0,0 +1,265 @@ +// +build integration + +package test + +import ( + "context" + "testing" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + output2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" + "github.com/stretchr/testify/assert" +) + +// TestBasicVerification 基础验证测试 +func TestBasicVerification(t *testing.T) { + if testing.Short() { + t.Skip("跳过集成测试") + } + + // 已知存在的域名 + domains := []string{ + "www.baidu.com", + "www.google.com", + "dns.google", + } + + domainChan := make(chan string, len(domains)) + for _, domain := range domains { + domainChan <- domain + } + close(domainChan) + + // 收集结果 + results := &testOutputter{results: make([]result.Result, 0)} + + opt := &options.Options{ + Rate: 1000, + Domain: domainChan, + Resolvers: options.GetResolvers(nil), + Silent: true, + TimeOut: 10, + Retry: 3, + Method: options.VerifyType, + Writer: []outputter.Output{results}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(options.GetResolvers(nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + r, err := runner.New(opt) + assert.NoError(t, err) + + r.RunEnumeration(ctx) + r.Close() + + // 验证结果 + assert.Greater(t, len(results.results), 0, "应该至少找到一个域名") + + for _, res := range results.results { + assert.NotEmpty(t, res.Subdomain, "域名不应为空") + assert.Greater(t, len(res.Answers), 0, "应该有至少一个答案") + t.Logf("找到: %s => %v", res.Subdomain, res.Answers) + } +} + +// TestCNAMEParsing 测试 CNAME 解析正确性 +func TestCNAMEParsing(t *testing.T) { + if testing.Short() { + t.Skip("跳过集成测试") + } + + // 已知有 CNAME 记录的域名 + domains := []string{ + "www.github.com", // 通常有 CNAME + "www.baidu.com", // 可能有 CNAME + } + + domainChan := make(chan string, len(domains)) + for _, domain := range domains { + domainChan <- domain + } + close(domainChan) + + results := &testOutputter{results: make([]result.Result, 0)} + + opt := &options.Options{ + Rate: 1000, + Domain: domainChan, + Resolvers: options.GetResolvers(nil), + Silent: true, + TimeOut: 10, + Retry: 3, + Method: options.VerifyType, + Writer: []outputter.Output{results}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(options.GetResolvers(nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + r, err := runner.New(opt) + assert.NoError(t, err) + + r.RunEnumeration(ctx) + r.Close() + + // 检查 CNAME 记录格式 + for _, res := range results.results { + for _, answer := range res.Answers { + // 不应该出现 "comcom" 等错误拼接 + assert.NotContains(t, answer, "comcom", "不应该有错误的字符串拼接") + assert.NotContains(t, answer, "\x00", "不应该包含空字符") + + t.Logf("%s => %s", res.Subdomain, answer) + } + } +} + +// TestHighSpeed 高速扫描测试 +func TestHighSpeed(t *testing.T) { + if testing.Short() { + t.Skip("跳过集成测试") + } + + // 生成100个测试域名 + domains := make([]string, 100) + for i := 0; i < 100; i++ { + if i%2 == 0 { + domains[i] = "www.baidu.com" // 存在的 + } else { + domains[i] = "nonexistent12345.baidu.com" // 不存在的 + } + } + + domainChan := make(chan string, len(domains)) + for _, domain := range domains { + domainChan <- domain + } + close(domainChan) + + results := &testOutputter{results: make([]result.Result, 0)} + + opt := &options.Options{ + Rate: 10000, // 高速 + Domain: domainChan, + Resolvers: options.GetResolvers(nil), + Silent: true, + TimeOut: 6, + Retry: 3, + Method: options.VerifyType, + Writer: []outputter.Output{results}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(options.GetResolvers(nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + r, err := runner.New(opt) + assert.NoError(t, err) + + r.RunEnumeration(ctx) + r.Close() + + // 应该找到大约50个(存在的域名) + assert.Greater(t, len(results.results), 40, "高速模式应该找到大部分存在的域名") + assert.Less(t, len(results.results), 60, "不应该有太多误报") + + t.Logf("高速扫描结果: 找到 %d/%d 个域名", len(results.results), len(domains)) +} + +// TestRetryMechanism 重试机制测试 +func TestRetryMechanism(t *testing.T) { + if testing.Short() { + t.Skip("跳过集成测试") + } + + domains := []string{"www.example.com"} + + domainChan := make(chan string, len(domains)) + for _, domain := range domains { + domainChan <- domain + } + close(domainChan) + + results := &testOutputter{results: make([]result.Result, 0)} + + // 测试不同的重试次数 + retryCounts := []int{1, 3, 5} + + for _, retryCount := range retryCounts { + opt := &options.Options{ + Rate: 1000, + Domain: domainChan, + Resolvers: options.GetResolvers(nil), + Silent: true, + TimeOut: 3, + Retry: retryCount, + Method: options.VerifyType, + Writer: []outputter.Output{results}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(options.GetResolvers(nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + + r, err := runner.New(opt) + assert.NoError(t, err) + + startTime := time.Now() + r.RunEnumeration(ctx) + elapsed := time.Since(startTime) + r.Close() + + cancel() + + t.Logf("重试次数 %d: 耗时 %v, 结果数 %d", + retryCount, elapsed, len(results.results)) + } +} + +// TestWildcardDetection 泛解析检测测试 +func TestWildcardDetection(t *testing.T) { + if testing.Short() { + t.Skip("跳过集成测试") + } + + // 测试已知的泛解析域名 + // 注意: 这需要一个实际的泛解析域名 + domain := "baidu.com" // 示例 + + isWild, ips := runner.IsWildCard(domain) + + if isWild { + t.Logf("检测到泛解析: %s, IPs: %v", domain, ips) + assert.Greater(t, len(ips), 0, "泛解析应该返回IP列表") + } else { + t.Logf("未检测到泛解析: %s", domain) + } +} + +// testOutputter 测试用输出器 +type testOutputter struct { + results []result.Result + mu sync.Mutex +} + +func (t *testOutputter) WriteDomainResult(r result.Result) error { + t.mu.Lock() + defer t.mu.Unlock() + t.results = append(t.results, r) + return nil +} + +func (t *testOutputter) Close() error { + return nil +} diff --git a/test/performance_benchmark_test.go b/test/performance_benchmark_test.go new file mode 100644 index 00000000..94386834 --- /dev/null +++ b/test/performance_benchmark_test.go @@ -0,0 +1,255 @@ +// +build performance + +package test + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/boy-hack/ksubdomain/v2/pkg/core/options" + "github.com/boy-hack/ksubdomain/v2/pkg/runner" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter" + output2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/outputter/output" + processbar2 "github.com/boy-hack/ksubdomain/v2/pkg/runner/processbar" + "github.com/boy-hack/ksubdomain/v2/pkg/runner/result" +) + +// Benchmark100kDomains 10万域名性能基准测试 +// 参考 README 中的对比测试: +// - 测试环境: 4核CPU, 5M 带宽 +// - 字典大小: 10万域名 +// - 目标: ~30秒完成扫描 +// - 成功率: > 95% +func Benchmark100kDomains(b *testing.B) { + if testing.Short() { + b.Skip("跳过性能基准测试 (使用 -tags=performance 运行)") + } + + // 创建 10 万域名字典 + dictFile := createBenchmarkDict(b, 100000) + defer os.Remove(dictFile) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + runBenchmark(b, dictFile, 100000) + } +} + +// Benchmark10kDomains 1万域名快速测试 +func Benchmark10kDomains(b *testing.B) { + if testing.Short() { + b.Skip("跳过性能基准测试") + } + + dictFile := createBenchmarkDict(b, 10000) + defer os.Remove(dictFile) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + runBenchmark(b, dictFile, 10000) + } +} + +// Benchmark1kDomains 1千域名基础测试 +func Benchmark1kDomains(b *testing.B) { + dictFile := createBenchmarkDict(b, 1000) + defer os.Remove(dictFile) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + runBenchmark(b, dictFile, 1000) + } +} + +// createBenchmarkDict 创建测试字典 +func createBenchmarkDict(b *testing.B, count int) string { + b.Helper() + + tmpFile := filepath.Join(os.TempDir(), fmt.Sprintf("ksubdomain_bench_%d.txt", count)) + f, err := os.Create(tmpFile) + if err != nil { + b.Fatalf("创建字典文件失败: %v", err) + } + defer f.Close() + + writer := bufio.NewWriter(f) + + // 生成域名列表 + // 使用一些真实存在的域名模式,提高测试真实性 + baseDomains := []string{ + "example.com", + "test.com", + "demo.org", + "sample.net", + } + + prefixes := []string{ + "www", "mail", "ftp", "blog", "shop", "admin", + "api", "dev", "test", "staging", "prod", "app", + "web", "mobile", "cdn", "static", "img", "media", + } + + for i := 0; i < count; i++ { + var domain string + + if i < len(prefixes)*len(baseDomains) { + // 使用常见前缀 + prefix := prefixes[i%len(prefixes)] + base := baseDomains[i/len(prefixes)%len(baseDomains)] + domain = fmt.Sprintf("%s.%s", prefix, base) + } else { + // 生成随机子域名 + base := baseDomains[i%len(baseDomains)] + domain = fmt.Sprintf("subdomain%d.%s", i, base) + } + + _, err := writer.WriteString(domain + "\n") + if err != nil { + b.Fatalf("写入字典失败: %v", err) + } + } + + err = writer.Flush() + if err != nil { + b.Fatalf("刷新字典失败: %v", err) + } + + b.Logf("创建字典: %s (%d 个域名)", tmpFile, count) + return tmpFile +} + +// runBenchmark 运行性能测试 +func runBenchmark(b *testing.B, dictFile string, expectedCount int) { + b.Helper() + + // 打开字典文件 + file, err := os.Open(dictFile) + if err != nil { + b.Fatalf("打开字典失败: %v", err) + } + defer file.Close() + + // 读取所有域名到通道 + domainChan := make(chan string, 10000) + go func() { + scanner := bufio.NewScanner(file) + for scanner.Scan() { + domainChan <- scanner.Text() + } + close(domainChan) + }() + + // 收集结果 + results := &perfOutputter{ + results: make([]result.Result, 0, expectedCount), + startTime: time.Now(), + totalDomains: expectedCount, + } + + // 配置扫描参数 (参考 README 的测试配置) + opt := &options.Options{ + Rate: options.Band2Rate("5m"), // 5M 带宽 + Domain: domainChan, + Resolvers: options.GetResolvers(nil), + Silent: true, + TimeOut: 6, + Retry: 3, + Method: options.VerifyType, + Writer: []outputter.Output{results}, + ProcessBar: &processbar2.FakeProcess{}, + EtherInfo: options.GetDeviceConfig(options.GetResolvers(nil)), + } + + // 创建上下文 (5分钟超时,足够10万域名) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // 记录开始时间 + startTime := time.Now() + + // 运行扫描 + r, err := runner.New(opt) + if err != nil { + b.Fatalf("创建 runner 失败: %v", err) + } + + r.RunEnumeration(ctx) + r.Close() + + // 计算性能指标 + elapsed := time.Since(startTime) + successCount := len(results.results) + successRate := float64(successCount) / float64(expectedCount) * 100 + domainsPerSecond := float64(expectedCount) / elapsed.Seconds() + + // 报告性能指标 + b.ReportMetric(elapsed.Seconds(), "total_seconds") + b.ReportMetric(float64(successCount), "success_count") + b.ReportMetric(successRate, "success_rate_%") + b.ReportMetric(domainsPerSecond, "domains/sec") + + // 日志输出 + b.Logf("性能测试结果:") + b.Logf(" - 字典大小: %d 个域名", expectedCount) + b.Logf(" - 总耗时: %v", elapsed) + b.Logf(" - 成功数: %d", successCount) + b.Logf(" - 成功率: %.2f%%", successRate) + b.Logf(" - 速率: %.0f domains/s", domainsPerSecond) + + // 性能基准检查 (参考 README: 10万域名 ~30秒) + if expectedCount == 100000 { + // 10万域名应该在 60 秒内完成 (给予一定容差) + if elapsed.Seconds() > 60 { + b.Logf("⚠️ 性能警告: 10万域名耗时 %.1f 秒 (目标 < 60秒)", elapsed.Seconds()) + } else if elapsed.Seconds() <= 30 { + b.Logf("✅ 性能优秀: 10万域名仅耗时 %.1f 秒 (达到 README 标准)", elapsed.Seconds()) + } else { + b.Logf("✓ 性能良好: 10万域名耗时 %.1f 秒", elapsed.Seconds()) + } + } +} + +// perfOutputter 性能测试输出器 +type perfOutputter struct { + results []result.Result + mu sync.Mutex + startTime time.Time + totalDomains int + lastReport time.Time +} + +func (p *perfOutputter) WriteDomainResult(r result.Result) error { + p.mu.Lock() + defer p.mu.Unlock() + + p.results = append(p.results, r) + + // 每1000个结果报告一次进度 + if len(p.results)%1000 == 0 { + elapsed := time.Since(p.startTime) + rate := float64(len(p.results)) / elapsed.Seconds() + progress := float64(len(p.results)) / float64(p.totalDomains) * 100 + + // 避免频繁输出 + if time.Since(p.lastReport) > time.Second { + fmt.Printf("\r进度: %d/%d (%.1f%%), 速率: %.0f domains/s, 耗时: %v", + len(p.results), p.totalDomains, progress, rate, elapsed.Round(time.Second)) + p.lastReport = time.Now() + } + } + + return nil +} + +func (p *perfOutputter) Close() error { + elapsed := time.Since(p.startTime) + fmt.Printf("\n最终结果: %d/%d, 耗时: %v\n", + len(p.results), p.totalDomains, elapsed.Round(time.Millisecond)) + return nil +} diff --git a/test/run_all_tests.sh b/test/run_all_tests.sh new file mode 100755 index 00000000..dae47a7a --- /dev/null +++ b/test/run_all_tests.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# ksubdomain 测试运行脚本 +# 用于运行所有测试 + +# 颜色定义 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +NC='\033[0m' # 无颜色 + +# 检查权限 +if [ "$EUID" -ne 0 ]; then + echo -e "${RED}警告: 测试脚本需要root权限才能正常运行${NC}" + echo "请使用 sudo 运行此脚本" + exit 1 +fi + +# 确保测试目录存在 +mkdir -p "test/results" + +# 清理旧的测试结果 +echo -e "${YELLOW}清理旧的测试结果...${NC}" +rm -rf test/results/* + +# 设置权限 +chmod +x test/benchmark.sh +chmod +x test/accuracy_test.sh +chmod +x test/stress_test.sh + +echo "========================================" +echo -e "${BLUE}KSubdomain 测试套件${NC}" +echo "========================================" +echo "" + +# 询问是否有旧版本可用于比较 +read -p "是否有原始版本的 ksubdomain 可用于比较测试? (y/n): " has_original +if [ "$has_original" = "y" ] || [ "$has_original" = "Y" ]; then + read -p "请输入原始版本 ksubdomain 的路径: " orig_path + if [ -f "$orig_path" ]; then + echo "复制原始版本到测试目录..." + cp "$orig_path" "test/ksubdomain_orig" + chmod +x "test/ksubdomain_orig" + else + echo -e "${RED}错误: 指定的路径不存在${NC}" + exit 1 + fi +fi + +# 菜单选择要运行的测试 +echo "" +echo "请选择要运行的测试:" +echo "1) 性能基准测试 (测试不同规模的字典)" +echo "2) 准确性测试 (比较结果一致性)" +echo "3) 压力测试 (测试高负载下的性能)" +echo "4) 运行所有测试" +echo "0) 退出" +echo "" + +read -p "请输入选项 [0-4]: " choice + +case $choice in + 1) + echo -e "${YELLOW}运行性能基准测试...${NC}" + test/benchmark.sh + ;; + 2) + if [ -f "test/ksubdomain_orig" ]; then + echo -e "${YELLOW}运行准确性测试...${NC}" + test/accuracy_test.sh + else + echo -e "${RED}错误: 准确性测试需要原始版本的 ksubdomain${NC}" + exit 1 + fi + ;; + 3) + echo -e "${YELLOW}运行压力测试...${NC}" + test/stress_test.sh + ;; + 4) + echo -e "${YELLOW}运行所有测试...${NC}" + + echo -e "${BLUE}1. 性能基准测试${NC}" + test/benchmark.sh + + if [ -f "test/ksubdomain_orig" ]; then + echo -e "${BLUE}2. 准确性测试${NC}" + test/accuracy_test.sh + else + echo -e "${RED}跳过准确性测试,原始版本不存在${NC}" + fi + + echo -e "${BLUE}3. 压力测试${NC}" + test/stress_test.sh + ;; + 0) + echo "退出测试" + exit 0 + ;; + *) + echo -e "${RED}无效选项${NC}" + exit 1 + ;; +esac + +echo "" +echo -e "${GREEN}所有测试完成!${NC}" +echo "测试结果保存在 test/results 目录中" \ No newline at end of file diff --git a/test/stress_test.sh b/test/stress_test.sh new file mode 100755 index 00000000..07630c59 --- /dev/null +++ b/test/stress_test.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +# ksubdomain 压力测试脚本 +# 用于测试ksubdomain在高负载下的性能表现 + +# 颜色定义 +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' # 无颜色 + +# 测试配置 +DOMAIN="example.com" +DICT_LARGE="test/stress_dict.txt" +RESOLVERS="test/resolvers.txt" +OUTPUT_DIR="test/results" +BIN="./ksubdomain" + +# 确保测试目录存在 +mkdir -p "$OUTPUT_DIR" + +# 创建DNS解析器文件(如果不存在) +if [ ! -f "$RESOLVERS" ]; then + echo "创建DNS解析器文件..." + echo "8.8.8.8" > "$RESOLVERS" + echo "8.8.4.4" >> "$RESOLVERS" + echo "1.1.1.1" >> "$RESOLVERS" + echo "114.114.114.114" >> "$RESOLVERS" +fi + +# 创建大型字典(如果不存在) +if [ ! -f "$DICT_LARGE" ]; then + echo "创建大型字典..." + for i in $(seq 1 500000); do + echo "stress$i.$DOMAIN" >> "$DICT_LARGE" + done + echo "创建了 500,000 条记录的字典" +fi + +# 获取系统信息 +echo "========================================" +echo " 系统信息" +echo "========================================" +echo "操作系统: $(uname -s)" +echo "处理器: $(uname -p)" +echo "内核版本: $(uname -r)" + +if [ "$(uname -s)" = "Linux" ]; then + echo "CPU核心数: $(nproc)" + echo "内存: $(free -h | grep Mem | awk '{print $2}')" +elif [ "$(uname -s)" = "Darwin" ]; then + echo "CPU核心数: $(sysctl -n hw.ncpu)" + echo "内存: $(sysctl -n hw.memsize | awk '{print $1/1024/1024/1024 " GB"}')" +fi + +echo "" +echo "========================================" +echo " KSubdomain 压力测试" +echo "========================================" + +# 用于每次压力测试的函数 +run_stress_test() { + local rate=$1 + local output="${OUTPUT_DIR}/stress_${rate}.txt" + local log="${OUTPUT_DIR}/stress_${rate}.log" + + echo -e "${YELLOW}测试速率: $rate pps${NC}" + + # 清理旧文件 + [ -f "$output" ] && rm "$output" + [ -f "$log" ] && rm "$log" + + # 使用时间命令运行测试,获取总用时 + echo "开始测试..." + + # 执行测试并计时 + start_time=$(date +%s.%N) + + # 将标准输出和错误输出重定向到日志文件 + $BIN v -f "$DICT_LARGE" -r "$RESOLVERS" -o "$output" -b "$rate" --retry 2 --timeout 4 --np > "$log" 2>&1 + + end_time=$(date +%s.%N) + elapsed=$(echo "$end_time - $start_time" | bc) + + # 统计结果 + processed_count=$(cat "$log" | grep -o "success:[0-9]*" | tail -1 | grep -o "[0-9]*") + found_count=$(wc -l < "$output") + + # 计算每秒处理的域名数 + domains_per_sec=$(echo "$processed_count / $elapsed" | bc) + + echo -e "${GREEN}测试完成${NC}" + echo "处理域名数: $processed_count" + echo "找到子域名: $found_count" + echo "耗时: $elapsed 秒" + echo "处理速率: $domains_per_sec 域名/秒" + echo "" + + # 输出结果字符串,供后续收集 + echo "$rate,$elapsed,$processed_count,$found_count,$domains_per_sec" +} + +# 不同速率的测试 +echo -e "${YELLOW}运行不同速率的压力测试...${NC}" +echo "" + +# 创建结果CSV文件 +RESULT_CSV="${OUTPUT_DIR}/stress_results.csv" +echo "速率(pps),耗时(秒),处理域名数,发现子域名数,实际速率(域名/秒)" > "$RESULT_CSV" + +# 逐步提高速率进行测试 +for rate in "10k" "50k" "100k" "200k" "500k" "1m"; do + result=$(run_stress_test "$rate") + echo "$result" >> "$RESULT_CSV" + + # 短暂休息让系统冷却 + sleep 5 +done + +echo "========================================" +echo " 内存使用测试" +echo "========================================" + +# 记录运行时内存使用情况 +echo -e "${YELLOW}测试运行时内存使用情况...${NC}" +MEMORY_LOG="${OUTPUT_DIR}/memory_usage.log" +MEM_OUTPUT="${OUTPUT_DIR}/memory_test.txt" + +# 清理旧文件 +[ -f "$MEMORY_LOG" ] && rm "$MEMORY_LOG" +[ -f "$MEM_OUTPUT" ] && rm "$MEM_OUTPUT" + +echo "开始测试..." + +# 后台运行ksubdomain +$BIN v -f "$DICT_LARGE" -r "$RESOLVERS" -o "$MEM_OUTPUT" -b "100k" --retry 2 --timeout 4 --np > /dev/null 2>&1 & +PID=$! + +# 监控10秒内的内存使用情况 +echo "PID: $PID" +echo "监控内存使用情况..." + +for i in {1..10}; do + if [ "$(uname -s)" = "Linux" ]; then + # Linux下获取RSS内存使用量 + MEM=$(ps -p $PID -o rss= 2>/dev/null) + if [ ! -z "$MEM" ]; then + MEM_MB=$(echo "scale=2; $MEM / 1024" | bc) + echo "内存使用 #$i: ${MEM_MB}MB" | tee -a "$MEMORY_LOG" + else + echo "进程已结束" + break + fi + elif [ "$(uname -s)" = "Darwin" ]; then + # MacOS下获取内存使用量 + MEM=$(ps -p $PID -o rss= 2>/dev/null) + if [ ! -z "$MEM" ]; then + MEM_MB=$(echo "scale=2; $MEM / 1024" | bc) + echo "内存使用 #$i: ${MEM_MB}MB" | tee -a "$MEMORY_LOG" + else + echo "进程已结束" + break + fi + fi + sleep 1 +done + +# 测试10秒后结束进程 +if kill -0 $PID 2>/dev/null; then + echo "终止进程..." + kill $PID +fi + +# 获取最大内存使用量 +if [ -f "$MEMORY_LOG" ]; then + MAX_MEM=$(cat "$MEMORY_LOG" | grep -o "[0-9]\+\.[0-9]\+MB" | sort -nr | head -1) + echo -e "${GREEN}最大内存使用量: $MAX_MEM${NC}" +fi + +echo "" +echo "压力测试完成!" +echo "结果保存在 $RESULT_CSV" \ No newline at end of file