diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a0fa2e9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.github +.venv +__pycache__ +*.pyc +reports/* +!reports/.gitkeep diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..3a9e810 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,33 @@ +name: tests + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install the packaged CLI + run: python -m pip install . + - name: Check dependency consistency + run: python -m pip check + - name: Verify installed command + run: appsec-recon --version + - name: Run unit and local integration tests + run: python -B -m unittest discover -s tests -v + - name: Compile Python sources + run: PYTHONPYCACHEPREFIX=/tmp/appsec_recon_pycache python -m compileall -q appsec_framework examples diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fdcd74c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN useradd --create-home --uid 10001 scanner + +COPY pyproject.toml README_EN.md LICENSE ./ +COPY appsec_framework ./appsec_framework +COPY examples ./examples +COPY wordlists ./wordlists +RUN python -m pip install --no-cache-dir . + +RUN mkdir -p /app/reports && chown -R scanner:scanner /app +USER scanner + +ENTRYPOINT ["appsec-recon"] +CMD ["--help"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0fd3721 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Denis Yakushanov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 97c450e..b755d3c 100644 --- a/README.md +++ b/README.md @@ -1,225 +1,188 @@ # AppSec Recon Framework -AppSec Recon Framework — учебный, но практически ориентированный инструмент для первичной разведки веб-сервисов. Проект объединяет TCP connect scanning, базовые AppSec-проверки, smart fuzzing скрытых путей и генерацию отчетов в JSON/Markdown. - -Цель проекта — показать не только знание уязвимостей, но и инженерный подход к security automation: CLI, модульная структура, безопасные лимиты, отчеты для pipeline и демонстрационный локальный стенд. - -> Важно: инструмент предназначен только для систем, которыми вы владеете, локальных лабораторий или целей, на тестирование которых у вас есть явное разрешение. - -## Что делает инструмент - -- Сканирует TCP-порты у IP, доменов и небольших CIDR-диапазонов. -- Определяет web-порты и запускает поверх них AppSec-проверки. -- Ищет типовые misconfiguration: открытые `.env`, `.git/config`, `phpinfo.php`, `backup.sql`, `server-status`, Spring Actuator `/actuator/env`. -- Проверяет CORS на отражение недоверенного `Origin`. -- Проверяет базовые security headers: CSP, HSTS, clickjacking protection, `X-Content-Type-Options`. -- Делает smart fuzzing скрытых путей по wordlist. -- Ищет секретоподобные строки в HTML и same-origin JavaScript, не записывая полный секрет в отчет. -- Формирует JSON, Markdown и список URL для дальнейшего запуска Nuclei. -- Поддерживает bash-оркестрацию с `subfinder` и Telegram summary. - -## Архитектура - -```text -appsec_framework/ - cli.py # CLI, запуск модулей и сбор итогового результата - network.py # TCP connect scanner без root-прав - web.py # CORS, headers, sensitive paths, JS secrets, smart fuzzer - targets.py # разбор targets, CIDR и port ranges - models.py # dataclass-модели findings и scan result - reporting.py # JSON, Markdown и Nuclei target list - -scripts/ - run_scan.sh # быстрый запуск с готовыми артефактами - demo_local.sh # локальная уязвимая demo-цель - pipeline_subfinder.sh # subfinder -> scanner -> reports - send_telegram.sh # краткое Telegram-уведомление по JSON-отчету - -examples/ - vulnerable_demo_server.py # намеренно уязвимый локальный HTTP-сервис - -wordlists/ - common-web.txt # небольшой стартовый wordlist для fuzzing - -tests/ - test_targets.py # тесты парсинга портов и CIDR safety limit -``` +[![tests](https://github.com/fant3k/AppSec-Recon-Framework/actions/workflows/tests.yml/badge.svg)](https://github.com/fant3k/AppSec-Recon-Framework/actions/workflows/tests.yml) +![Python](https://img.shields.io/badge/Python-3.9%2B-3776ab) +![License](https://img.shields.io/badge/license-MIT-22c55e) +![Mode](https://img.shields.io/badge/mode-authorized_scans_only-f59e0b) + +CLI-инструмент для контролируемой первичной разведки веб-сервисов. Он +объединяет bounded TCP connect scan, определение HTTP(S) на стандартных и +нестандартных портах, несколько точечных AppSec-проверок, path discovery и +структурированные отчёты. + +> Используйте scanner только для localhost, собственных систем или целей с +> явным разрешением на тестирование. Для любой non-loopback цели требуется +> флаг `--acknowledge-authorization`. + +## Возможности + +- TCP connect scan без root-прав; +- IP, hostname, файл целей и небольшие CIDR-диапазоны; +- protocol probing HTTP/HTTPS на любом найденном порту; +- проверка CSP, HSTS, clickjacking protection и `nosniff`; +- проверка CORS с контролируемым недоверенным `Origin`; +- подтверждение открытых `.env`, `.git/config`, backup и debug endpoints по + сигнатурам; +- поиск секретоподобных значений в HTML и same-origin JavaScript; +- редактирование найденных значений перед записью evidence; +- path discovery с baseline для фильтрации soft-404; +- JSON, Markdown и Nuclei target list; +- локальная уязвимая demo-цель; +- pipeline с Subfinder и необязательной Telegram-сводкой. ## Быстрый старт ```bash +git clone https://github.com/fant3k/AppSec-Recon-Framework.git +cd AppSec-Recon-Framework python3 -m venv .venv source .venv/bin/activate -pip install -r requirements.txt +python -m pip install --upgrade pip +pip install . +appsec-recon --version ``` -Проверить, что проект запускается: +Установка создаёт команду `appsec-recon` и включает встроенный стартовый +wordlist в Python package. -```bash -python3 appsec_scan.py --help -python3 -B -m unittest discover -s tests -``` - -## Локальная демонстрация - -Самый безопасный способ посмотреть работу проекта — запустить локальный demo target: +## Безопасная локальная демонстрация ```bash scripts/demo_local.sh ``` -Demo-сервер поднимается на `127.0.0.1:8088` и специально отдает несколько небезопасных ответов: fake `.env`, fake `.git/config`, permissive CORS, JS-файл с fake API key и `/admin` для fuzzing. +Скрипт временно запускает цель на `127.0.0.1:8088`, выполняет scan и завершает +demo-сервер. Проверяются: + +- permissive credentialed CORS; +- открытые `.env` и `.git/config`; +- три отсутствующих security header; +- редактируемый секретоподобный JavaScript token; +- путь `/admin`, найденный через baseline-aware discovery. -Пример вывода: +Ожидаемый итог: ```text -[+] 127.0.0.1 tcp/8088 radan-http - -[*] Starting web AppSec checks on 1 service(s)... -[web] discovered 200 http://127.0.0.1:8088/admin -[LOW] Missing Content-Security-Policy: http://127.0.0.1:8088 -[HIGH] Permissive CORS policy: http://127.0.0.1:8088 -[HIGH] Exposed Git repository metadata: http://127.0.0.1:8088/.git/config -[CRITICAL] Exposed environment file: http://127.0.0.1:8088/.env -[HIGH] Potential secret in client-side JavaScript: http://127.0.0.1:8088/static/app.js - -[*] Scan summary - Open ports: 1 - Issues: 7 - Discovered paths: 1 - Severity: critical=1, high=3, low=3 +Open ports: 1 +Web services: 1 +Issues: 7 +Discovered paths: 1 +Severity: critical=1, high=3, low=3 ``` -Отчеты сохраняются в `reports/`. +## Использование -## Примеры использования - -Скан одного домена: +Только TCP scan локальной цели: ```bash -python3 appsec_scan.py example.com --ports 80,443,8080 +appsec-recon 127.0.0.1 --ports 1-1000 --no-web ``` -Скан с JSON и Markdown отчетами: +Авторизованная внешняя проверка с отчётами: ```bash -python3 appsec_scan.py example.com \ - --ports 80,443,8080,8443 \ +appsec-recon example.com \ + --ports 80,443,7000,8080,9443 \ + --acknowledge-authorization \ --json-out reports/example.json \ - --markdown-out reports/example.md + --markdown-out reports/example.md \ + --nuclei-targets reports/example-urls.txt ``` -Скан небольшой локальной сети: +Небольшой CIDR: ```bash -python3 appsec_scan.py 192.168.1.0/24 \ - --ports 22,80,443,8080 \ - --max-hosts 256 +appsec-recon 192.168.1.0/24 \ + --ports 80,443,8080 \ + --max-hosts 256 \ + --acknowledge-authorization ``` -Только TCP-сканирование без web-модулей: +## Safety controls -```bash -python3 appsec_scan.py 10.0.0.5 --ports 1-1000 --no-web -``` +Scanner ограничивает собственную активность: -Свой wordlist для fuzzing: +- `--max-hosts` ограничивает раскрытие каждого CIDR; +- `--max-scan-jobs` ограничивает произведение targets × ports; +- `--threads` допускает только 1–512 workers; +- TCP jobs отправляются ограниченными batch, а не складываются целиком в RAM; +- `--timeout`, `--fuzz-limit` и `--request-delay` валидируются; +- каждый HTTP response ограничен 512 KiB; +- JavaScript response ограничен 200 KiB; +- автоматические redirects не пересекают подтверждённый scope; +- внешняя цель требует `--acknowledge-authorization`; +- scanner использует идентифицируемый `User-Agent`. -```bash -python3 appsec_scan.py example.com \ - --ports 80,443 \ - --fuzz-wordlist wordlists/common-web.txt \ - --fuzz-limit 100 -``` +## Точность findings -Запуск через wrapper с готовыми артефактами: +HTTP 200 сам по себе не считается подтверждением открытого файла. Finding для +`.env`, `.git/config` и других чувствительных путей создаётся только при +совпадении ожидаемой сигнатуры. Baseline path позволяет path discovery +отфильтровать приложения, которые возвращают одинаковую soft-404 страницу с +кодом 200. -```bash -scripts/run_scan.sh example.com --ports 80,443,8080 -``` - -## Pipeline с subfinder +Это reconnaissance, а не доказательство exploitability. Каждый finding нужно +подтверждать вручную в рамках согласованного scope. -Если установлен `subfinder`, можно собрать простой recon pipeline: +## Архитектура -```bash -scripts/pipeline_subfinder.sh example.com --ports 80,443,8080,8443 +```text +appsec_framework/ + cli.py CLI, валидация scope и orchestration + targets.py target/CIDR/port parsing + network.py bounded TCP connect scan + web.py protocol probing и web checks + models.py типизированная схема результата + reporting.py JSON, Markdown и Nuclei targets + data/ встроенный стартовый wordlist + +examples/ локальная demo-цель +scripts/ demo, Subfinder pipeline и Telegram summary +tests/ unit и local integration tests ``` -Pipeline делает следующее: +## Проверка -1. Собирает поддомены через `subfinder`. -2. Сохраняет список целей в `reports/`. -3. Передает список в Python-сканер. -4. Создает JSON и Markdown отчеты. -5. Создает target list для Nuclei. -6. Отправляет Telegram summary, если заданы `TELEGRAM_BOT_TOKEN` и `TELEGRAM_CHAT_ID`. +```bash +scripts/test.sh +``` -## Формат отчета +Тесты запускают реальные localhost TCP/HTTP-сервисы и проверяют: -JSON-отчет содержит четыре основные секции: +- target и port parsing; +- ресурсные лимиты и authorization gate; +- bounded TCP scan; +- HTTP probing на случайном нестандартном порту; +- все семь demo findings; +- soft-200 filtering; +- response-size limits; +- redaction секретов; +- JSON, Markdown и Nuclei output. -- `open_ports` — найденные TCP-сервисы. -- `issues` — AppSec findings с severity, evidence и recommendation. -- `discovered_paths` — пути, найденные smart fuzzer. -- `errors` — нефатальные ошибки отдельных проверок. +GitHub Actions повторяет проверку на Python 3.9, 3.11 и 3.13, устанавливая +проект именно как пакет и вызывая установленную CLI-команду. -Пример finding: +## Docker -```json -{ - "id": "cors_misconfiguration", - "title": "Permissive CORS policy", - "severity": "high", - "target": "127.0.0.1", - "url": "http://127.0.0.1:8088", - "evidence": "Origin https://evil.example received Access-Control-Allow-Origin: https://evil.example; credentials=true.", - "recommendation": "Restrict CORS to trusted origins and avoid reflecting arbitrary Origin values." -} +```bash +docker compose up -d demo +docker compose --profile tools run --rm scanner +docker compose down ``` -## Что я изучил в процессе - -- Как работает TCP connect scanning и почему он удобен для инструмента без root-прав. -- Как проектировать CLI так, чтобы инструмент было удобно использовать вручную и в automation pipeline. -- Почему для web fuzzing нужен baseline-запрос, иначе кастомные 200-страницы дают много false positives. -- Как проверять CORS misconfiguration через controlled `Origin`. -- Почему evidence в отчетах должен быть полезным, но не должен размножать секреты. -- Как разделять scanner core, reporting и orchestration scripts. - -## Мой личный вклад и улучшения - -- Собрал модульную структуру вместо одного большого скрипта. -- Добавил safety controls: `--max-hosts`, timeout, `--fuzz-limit`, запуск web-проверок только по web-портам. -- Реализовал JSON/Markdown reporting и Nuclei target list. -- Добавил локальный уязвимый стенд для безопасной демонстрации проекта. -- Добавил bash-скрипты для практического сценария: recon -> scan -> report -> notification. -- Отделил machine-readable output от human-readable Markdown. +Demo публикуется только на `127.0.0.1:8088`. Scanner и demo взаимодействуют во +внутренней Docker network. ## Ограничения -- Это не замена Nmap, Nuclei, Burp Suite или полноценному DAST. -- Сканер не делает fingerprinting сервисов по banner grabbing. -- Smart fuzzer использует небольшой starter wordlist. -- JS secret detection основан на regex и может давать false positives. -- Web checks покрывают только базовые misconfiguration и не выполняют глубокую бизнес-логику. - -## Планы по развитию - -- Добавить banner grabbing и более точную идентификацию web-сервисов. -- Добавить HTML-отчет с фильтрами по severity. -- Добавить SARIF output для GitHub Security tab. -- Добавить опциональный запуск Nuclei по созданному target list. -- Добавить screenshots найденных web-сервисов через Playwright. -- Добавить Dockerfile и docker-compose для воспроизводимого demo. - -## Проверка проекта - -```bash -python3 -B -m unittest discover -s tests -PYTHONPYCACHEPREFIX=/tmp/appsec_pycache python3 -m compileall appsec_framework -``` +- это не замена Nmap, Nuclei, Burp Suite или полноценному DAST; +- service fingerprinting ограничен HTTP(S) probing и server header; +- секреты ищутся эвристически и требуют ручной проверки; +- path discovery использует небольшой встроенный wordlist; +- scanner не анализирует бизнес-логику и authenticated workflows. -## Лицензия и этика +## Документы -Проект создан для обучения, портфолио и авторизованных проверок. Не используйте его для сканирования чужой инфраструктуры без разрешения. +- [English README](README_EN.md) +- [Security and authorized-use policy](SECURITY.md) +- [MIT License](LICENSE) diff --git a/README_EN.md b/README_EN.md index a7ba041..e1c9511 100644 --- a/README_EN.md +++ b/README_EN.md @@ -1,226 +1,105 @@ # AppSec Recon Framework -AppSec Recon Framework is a lightweight reconnaissance and application security automation tool built for portfolio and training purposes. It combines TCP connect scanning, basic web misconfiguration checks, smart content fuzzing, and JSON/Markdown reporting. +[![tests](https://github.com/fant3k/AppSec-Recon-Framework/actions/workflows/tests.yml/badge.svg)](https://github.com/fant3k/AppSec-Recon-Framework/actions/workflows/tests.yml) +![Python](https://img.shields.io/badge/Python-3.9%2B-3776ab) +![License](https://img.shields.io/badge/license-MIT-22c55e) +![Mode](https://img.shields.io/badge/mode-authorized_scans_only-f59e0b) -The goal of this project is to demonstrate practical AppSec engineering skills: modular Python code, CLI design, safe defaults, structured findings, shell orchestration, and reporting that can be used by both humans and automation pipelines. +AppSec Recon Framework is a bounded command-line scanner for initial web +service reconnaissance. It combines TCP connect scanning, HTTP(S) protocol +probing on standard and non-standard ports, focused AppSec checks, path +discovery and structured reporting. -> Use this tool only against systems you own, local labs, or targets where you have explicit permission to test. +> Use it only against localhost, systems you own, or targets for which you have +> explicit authorization. Every non-loopback scan requires the +> `--acknowledge-authorization` flag. ## Capabilities -- TCP port scanning for IP addresses, domains, and small CIDR ranges. -- Web checks on common HTTP/HTTPS ports. -- Detection of common exposed resources: `.env`, `.git/config`, `phpinfo.php`, `backup.sql`, `server-status`, Spring Actuator `/actuator/env`. -- CORS misconfiguration check using a controlled untrusted `Origin`. -- Basic security header review: CSP, HSTS, clickjacking protection, `X-Content-Type-Options`. -- Smart path fuzzing with a configurable wordlist. -- Client-side secret pattern detection in HTML and same-origin JavaScript, with redacted evidence. -- JSON and Markdown reports. -- Nuclei target list generation. -- Bash orchestration for `subfinder`, scanner execution, reporting, and Telegram summaries. - -## Architecture - -```text -appsec_framework/ - cli.py # CLI parsing, scan orchestration, report writing - network.py # TCP connect scanner that does not require root privileges - web.py # CORS, headers, sensitive paths, JS secrets, smart fuzzer - targets.py # target, CIDR, and port range parsing - models.py # dataclass models for findings and scan results - reporting.py # JSON, Markdown, and Nuclei target output - -scripts/ - run_scan.sh # one-command scan wrapper - demo_local.sh # local intentionally vulnerable demo target - pipeline_subfinder.sh # subfinder -> scanner -> reports - send_telegram.sh # short Telegram notification from a JSON report - -examples/ - vulnerable_demo_server.py # intentionally vulnerable local HTTP service - -wordlists/ - common-web.txt # starter wordlist for web content fuzzing - -tests/ - test_targets.py # tests for port parsing and CIDR safety limits -``` +- non-root TCP connect scanning; +- hostnames, IPs, target files and bounded CIDR expansion; +- HTTP/HTTPS detection on every discovered port; +- CSP, HSTS, clickjacking and `nosniff` review; +- controlled untrusted-Origin CORS checks; +- signature-confirmed exposed file and debug endpoint checks; +- redacted secret-pattern detection in HTML and same-origin JavaScript; +- baseline-aware path discovery; +- JSON, Markdown and Nuclei target output; +- a deterministic localhost demo target; +- optional Subfinder and Telegram orchestration. -## Installation +## Install ```bash +git clone https://github.com/fant3k/AppSec-Recon-Framework.git +cd AppSec-Recon-Framework python3 -m venv .venv source .venv/bin/activate -pip install -r requirements.txt -``` - -Verify the setup: - -```bash -python3 appsec_scan.py --help -python3 -B -m unittest discover -s tests +python -m pip install --upgrade pip +pip install . +appsec-recon --version ``` -## Local Demo +The packaged CLI includes its default wordlist. -The safest way to try the scanner is the local demo target: +## Local demo ```bash scripts/demo_local.sh ``` -The demo service listens on `127.0.0.1:8088` and intentionally exposes fake findings: `.env`, `.git/config`, permissive CORS, a fake JavaScript API key, and an `/admin` path for the fuzzer. - -Example output: - -```text -[+] 127.0.0.1 tcp/8088 radan-http - -[*] Starting web AppSec checks on 1 service(s)... -[web] discovered 200 http://127.0.0.1:8088/admin -[LOW] Missing Content-Security-Policy: http://127.0.0.1:8088 -[HIGH] Permissive CORS policy: http://127.0.0.1:8088 -[HIGH] Exposed Git repository metadata: http://127.0.0.1:8088/.git/config -[CRITICAL] Exposed environment file: http://127.0.0.1:8088/.env -[HIGH] Potential secret in client-side JavaScript: http://127.0.0.1:8088/static/app.js - -[*] Scan summary - Open ports: 1 - Issues: 7 - Discovered paths: 1 - Severity: critical=1, high=3, low=3 -``` - -Reports are written to `reports/`. +The script starts an intentionally vulnerable service on `127.0.0.1:8088`, +scans it, writes JSON and Markdown reports, and stops the service. The expected +result is one web service, seven findings and one discovered path. -## Usage Examples - -Scan a single domain: - -```bash -python3 appsec_scan.py example.com --ports 80,443,8080 -``` - -Generate JSON and Markdown reports: +## Authorized external scan ```bash -python3 appsec_scan.py example.com \ - --ports 80,443,8080,8443 \ +appsec-recon example.com \ + --ports 80,443,7000,8080,9443 \ + --acknowledge-authorization \ --json-out reports/example.json \ - --markdown-out reports/example.md + --markdown-out reports/example.md \ + --nuclei-targets reports/example-urls.txt ``` -Scan a small local network: +## Safety model -```bash -python3 appsec_scan.py 192.168.1.0/24 \ - --ports 22,80,443,8080 \ - --max-hosts 256 -``` +The scanner validates concurrency, timeouts, fuzzing limits and request delay; +caps CIDR expansion and total target × port jobs; submits TCP work in bounded +batches; caps HTTP responses; avoids cross-scope redirect following; identifies +itself with a User-Agent; and requires explicit acknowledgement for any +non-loopback target. -Run only TCP scanning: +A sensitive-path finding requires both HTTP 200 and an expected content +signature. Generic soft-200 responses are not reported as exposed files. -```bash -python3 appsec_scan.py 10.0.0.5 --ports 1-1000 --no-web -``` - -Use a custom fuzzing wordlist: +## Tests ```bash -python3 appsec_scan.py example.com \ - --ports 80,443 \ - --fuzz-wordlist wordlists/common-web.txt \ - --fuzz-limit 100 +scripts/test.sh ``` -Use the wrapper script: +The suite starts real localhost TCP and HTTP services and verifies target +parsing, safety gates, bounded scanning, arbitrary-port HTTP probing, all demo +findings, soft-200 filtering, response limits, secret redaction and every report +format. CI installs the package and runs the suite on Python 3.9, 3.11 and 3.13. -```bash -scripts/run_scan.sh example.com --ports 80,443,8080 -``` - -## Subdomain Pipeline - -If `subfinder` is installed: +## Docker ```bash -scripts/pipeline_subfinder.sh example.com --ports 80,443,8080,8443 -``` - -The pipeline: - -1. Enumerates subdomains with `subfinder`. -2. Stores targets under `reports/`. -3. Passes the target file to the Python scanner. -4. Writes JSON and Markdown reports. -5. Generates a Nuclei target list. -6. Sends a Telegram summary when `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` are configured. - -## Report Format - -The JSON report has four main sections: - -- `open_ports`: discovered TCP services. -- `issues`: AppSec findings with severity, evidence, and remediation guidance. -- `discovered_paths`: paths discovered by the smart fuzzer. -- `errors`: non-fatal module errors. - -Example finding: - -```json -{ - "id": "cors_misconfiguration", - "title": "Permissive CORS policy", - "severity": "high", - "target": "127.0.0.1", - "url": "http://127.0.0.1:8088", - "evidence": "Origin https://evil.example received Access-Control-Allow-Origin: https://evil.example; credentials=true.", - "recommendation": "Restrict CORS to trusted origins and avoid reflecting arbitrary Origin values." -} +docker compose up -d demo +docker compose --profile tools run --rm scanner +docker compose down ``` -## What I Learned +The demo port is published on `127.0.0.1` only. -- How TCP connect scanning works and why it is suitable for a non-root educational scanner. -- How to design a CLI that works for both manual use and automation pipelines. -- Why fuzzing needs a baseline request to reduce false positives from custom 200 responses. -- How to test CORS behavior with a controlled untrusted `Origin`. -- Why reports should provide useful evidence without spreading full secrets. -- How to separate scanner logic, reporting, and shell orchestration. - -## My Contributions and Improvements - -- Reworked the scanner into a modular Python package instead of a single script. -- Added safety controls: `--max-hosts`, network timeout, `--fuzz-limit`, and web checks only on likely HTTP ports. -- Implemented JSON/Markdown reporting and Nuclei target generation. -- Added a local intentionally vulnerable demo service for safe presentations. -- Added shell scripts for a practical recon workflow: enumeration, scanning, reporting, and notification. -- Separated machine-readable output from human-readable reports. - -## Limitations - -- This is not a replacement for Nmap, Nuclei, Burp Suite, or a full DAST platform. -- The scanner does not perform banner-based service fingerprinting yet. -- The bundled fuzzing wordlist is intentionally small. -- JavaScript secret detection is regex-based and may produce false positives. -- Web checks cover common misconfigurations, not deep business logic flaws. - -## Roadmap - -- Add banner grabbing and better service fingerprinting. -- Add an HTML report with severity filters. -- Add SARIF output for GitHub Security tab integration. -- Add optional Nuclei execution against generated targets. -- Add web service screenshots with Playwright. -- Add Docker and docker-compose for a reproducible demo. - -## Testing - -```bash -python3 -B -m unittest discover -s tests -PYTHONPYCACHEPREFIX=/tmp/appsec_pycache python3 -m compileall appsec_framework -``` +## Scope -## Ethics +This is a reconnaissance aid, not a replacement for Nmap, Nuclei, Burp Suite +or a full DAST platform. Findings require manual validation within the agreed +assessment scope. -This project is intended for learning, portfolio demonstration, and authorized security testing only. Do not scan third-party infrastructure without permission. +See the [Russian README](README.md), [security policy](SECURITY.md) and +[MIT license](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c87d61f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security and authorized-use policy + +AppSec Recon Framework performs active TCP connections and HTTP requests. + +Use it only against localhost, systems you own, or targets for which you have +explicit written authorization. Non-loopback scans require the +`--acknowledge-authorization` flag. That flag records intent; it does not grant +permission or replace an agreed scope of work. + +The scanner applies bounded concurrency, target and job limits, response-size +limits, request timeouts and a configurable delay. Do not remove those controls +when testing shared infrastructure. + +If you discover a vulnerability in the scanner itself, report it privately to +the repository owner through GitHub. Do not include real credentials, internal +reports, customer targets or unredacted scanner evidence in a public issue. diff --git a/appsec_framework/__init__.py b/appsec_framework/__init__.py index 0c1b739..919d6fa 100644 --- a/appsec_framework/__init__.py +++ b/appsec_framework/__init__.py @@ -4,4 +4,4 @@ ссылались на одинаковый идентификатор сборки. """ -__version__ = "0.1.0" +__version__ = "1.0.0" diff --git a/appsec_framework/cli.py b/appsec_framework/cli.py index 8e21976..5ad8f82 100644 --- a/appsec_framework/cli.py +++ b/appsec_framework/cli.py @@ -7,8 +7,9 @@ from __future__ import annotations import argparse -from datetime import datetime -from pathlib import Path +import ipaddress +from datetime import datetime, timezone +from importlib.resources import files from . import __version__ from .models import ScanResult, utc_now @@ -20,10 +21,11 @@ write_nuclei_targets, ) from .targets import load_targets, parse_ports -from .web import WebScanner, is_web_port +from .models import WebService +from .web import WebScanner -DEFAULT_WORDLIST = str(Path(__file__).resolve().parent.parent / "wordlists" / "common-web.txt") +DEFAULT_WORDLIST = str(files("appsec_framework").joinpath("data/common-web.txt")) def build_parser() -> argparse.ArgumentParser: @@ -42,14 +44,31 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--threads", type=int, default=100, help="TCP scanner worker threads") parser.add_argument("--timeout", type=float, default=0.8, help="Network timeout in seconds") parser.add_argument("--max-hosts", type=int, default=256, help="Safety limit for CIDR expansion") + parser.add_argument( + "--max-scan-jobs", + type=int, + default=100_000, + help="Maximum target × port combinations for one run", + ) parser.add_argument("--web", action=argparse.BooleanOptionalAction, default=True, help="Run web AppSec modules") parser.add_argument("--skip-fuzz", action="store_true", help="Disable Smart Fuzzer module") parser.add_argument("--fuzz-wordlist", default=DEFAULT_WORDLIST, help="Path to web content wordlist") parser.add_argument("--fuzz-limit", type=int, default=200, help="Maximum wordlist entries per web service") + parser.add_argument( + "--request-delay", + type=float, + default=0.05, + help="Delay between active web requests in seconds", + ) parser.add_argument("--verify-tls", action="store_true", help="Verify TLS certificates during web checks") parser.add_argument("--json-out", help="Path to JSON report") parser.add_argument("--markdown-out", help="Optional Markdown report path") parser.add_argument("--nuclei-targets", help="Write discovered web URLs as a Nuclei target list") + parser.add_argument( + "--acknowledge-authorization", + action="store_true", + help="Confirm explicit permission for every non-loopback target", + ) parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") return parser @@ -64,10 +83,11 @@ def main(argv=None) -> int: try: targets = load_targets(args.target, args.targets_file, max_hosts=args.max_hosts) ports = parse_ports(args.ports) + validate_runtime_options(args, targets, ports) except ValueError as exc: parser.error(str(exc)) - timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S") + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") json_out = args.json_out or f"reports/scan_{timestamp}.json" result = ScanResult( @@ -79,10 +99,12 @@ def main(argv=None) -> int: "ports": ports, "threads": args.threads, "timeout": args.timeout, + "max_scan_jobs": args.max_scan_jobs, "web": args.web, "skip_fuzz": args.skip_fuzz, "fuzz_wordlist": args.fuzz_wordlist, "fuzz_limit": args.fuzz_limit, + "request_delay": args.request_delay, "verify_tls": args.verify_tls, }, ) @@ -94,20 +116,28 @@ def main(argv=None) -> int: result.open_ports = scan_targets(targets, ports, timeout=args.timeout, threads=args.threads) if args.web: - # Web-модули запускаются только по портам, которые действительно похожи - # на HTTP/HTTPS. Это делает результат чище и экономит время. - web_ports = [finding for finding in result.open_ports if is_web_port(finding.port)] - if web_ports: - print(f"\n[*] Starting web AppSec checks on {len(web_ports)} service(s)...") + if result.open_ports: + print(f"\n[*] Probing {len(result.open_ports)} open port(s) for HTTP(S)...") scanner = WebScanner( timeout=args.timeout, verify_tls=args.verify_tls, fuzz_wordlist=args.fuzz_wordlist, fuzz_limit=args.fuzz_limit, skip_fuzz=args.skip_fuzz, + request_delay=args.request_delay, ) - for finding in web_ports: + for finding in result.open_ports: output = scanner.scan_service(finding.target, finding.port) + if output.base_url: + result.web_services.append( + WebService( + target=finding.target, + port=finding.port, + base_url=output.base_url, + status_code=output.status_code or 0, + server=output.server, + ) + ) result.issues.extend(output.issues) result.discovered_paths.extend(output.discovered_paths) result.errors.extend(output.errors) @@ -136,5 +166,41 @@ def main(argv=None) -> int: return 0 +def validate_runtime_options(args, targets, ports) -> None: + """Проверить ресурсные лимиты и явное подтверждение авторизации.""" + if not 1 <= args.threads <= 512: + raise ValueError("--threads must be between 1 and 512") + if not 0.05 <= args.timeout <= 60: + raise ValueError("--timeout must be between 0.05 and 60 seconds") + if not 1 <= args.max_hosts <= 65_536: + raise ValueError("--max-hosts must be between 1 and 65536") + if not 1 <= args.max_scan_jobs <= 1_000_000: + raise ValueError("--max-scan-jobs must be between 1 and 1000000") + if not 0 <= args.fuzz_limit <= 5_000: + raise ValueError("--fuzz-limit must be between 0 and 5000") + if not 0 <= args.request_delay <= 10: + raise ValueError("--request-delay must be between 0 and 10 seconds") + + scan_jobs = len(targets) * len(ports) + if scan_jobs > args.max_scan_jobs: + raise ValueError( + f"Scan would create {scan_jobs} jobs; --max-scan-jobs is {args.max_scan_jobs}" + ) + if any(not _is_loopback_target(target) for target in targets): + if not args.acknowledge_authorization: + raise ValueError( + "Non-loopback targets require --acknowledge-authorization" + ) + + +def _is_loopback_target(target: str) -> bool: + if target.lower() == "localhost": + return True + try: + return ipaddress.ip_address(target).is_loopback + except ValueError: + return False + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/appsec_framework/data/common-web.txt b/appsec_framework/data/common-web.txt new file mode 100644 index 0000000..8e8563f --- /dev/null +++ b/appsec_framework/data/common-web.txt @@ -0,0 +1,38 @@ +# Small starter list for controlled web content discovery. +admin +administrator +api +api/v1 +api/v2 +app +assets +backup +backups +beta +config +console +dashboard +debug +dev +docs +files +graphql +internal +login +logs +manage +manager +old +private +public +secret +server-status +staging +status +swagger +swagger-ui +swagger-ui/ +test +uploads +v1 +v2 diff --git a/appsec_framework/models.py b/appsec_framework/models.py index 4388280..f6fda55 100644 --- a/appsec_framework/models.py +++ b/appsec_framework/models.py @@ -27,6 +27,17 @@ class PortFinding: protocol: str = "tcp" +@dataclass +class WebService: + """Подтвержденный HTTP(S)-сервис на открытом TCP-порту.""" + + target: str + port: int + base_url: str + status_code: int + server: str = "" + + @dataclass class DiscoveredPath: """Веб-путь, который выглядит существующим по результатам fuzzing.""" @@ -50,6 +61,7 @@ class Issue: url: str evidence: str recommendation: str + confidence: str = "confirmed" metadata: Dict[str, Any] = field(default_factory=dict) @@ -62,6 +74,7 @@ class ScanResult: finished_at: Optional[str] options: Dict[str, Any] open_ports: List[PortFinding] = field(default_factory=list) + web_services: List[WebService] = field(default_factory=list) issues: List[Issue] = field(default_factory=list) discovered_paths: List[DiscoveredPath] = field(default_factory=list) errors: List[str] = field(default_factory=list) diff --git a/appsec_framework/network.py b/appsec_framework/network.py index 5a9eeb6..c2a0faa 100644 --- a/appsec_framework/network.py +++ b/appsec_framework/network.py @@ -10,6 +10,7 @@ import socket from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import closing +from itertools import islice, product from typing import Iterable, List, Optional from .models import PortFinding @@ -26,6 +27,7 @@ def scan_targets( ports: Iterable[int], timeout: float, threads: int, + batch_multiplier: int = 4, ) -> List[PortFinding]: """Просканировать набор целей и вернуть только открытые TCP-порты. @@ -34,18 +36,22 @@ def scan_targets( понятный способ показать базовую параллелизацию I/O-bound задач. """ findings: List[PortFinding] = [] - jobs = [] - with ThreadPoolExecutor(max_workers=threads) as executor: - for target in targets: - for port in ports: - jobs.append(executor.submit(scan_port, target, port, timeout)) - - for future in as_completed(jobs): - finding = future.result() - if finding: - findings.append(finding) - print(f"[+] {finding.target:30} tcp/{finding.port:<5} {finding.service}") + work = product(targets, ports) + batch_size = max(threads, threads * batch_multiplier) + while True: + batch = list(islice(work, batch_size)) + if not batch: + break + jobs = [ + executor.submit(scan_port, target, port, timeout) + for target, port in batch + ] + for future in as_completed(jobs): + finding = future.result() + if finding: + findings.append(finding) + print(f"[+] {finding.target:30} tcp/{finding.port:<5} {finding.service}") return sorted(findings, key=lambda item: (item.target, item.port)) diff --git a/appsec_framework/reporting.py b/appsec_framework/reporting.py index 3eb9e19..bd39154 100644 --- a/appsec_framework/reporting.py +++ b/appsec_framework/reporting.py @@ -6,7 +6,6 @@ from pathlib import Path from .models import ScanResult -from .web import build_base_url, is_web_port SEVERITY_ORDER = ("critical", "high", "medium", "low", "info") @@ -33,6 +32,7 @@ def write_markdown_report(result: ScanResult, output_path: str) -> None: f"- Started: `{result.started_at}`", f"- Finished: `{result.finished_at}`", f"- Open ports: `{len(result.open_ports)}`", + f"- Web services: `{len(result.web_services)}`", f"- Issues: `{len(result.issues)}`", f"- Discovered paths: `{len(result.discovered_paths)}`", "", @@ -86,10 +86,8 @@ def write_nuclei_targets(result: ScanResult, output_path: str) -> None: """ urls = [] seen = set() - for finding in result.open_ports: - if not is_web_port(finding.port): - continue - url = build_base_url(finding.target, finding.port) + for service in result.web_services: + url = service.base_url if url not in seen: urls.append(url) seen.add(url) @@ -103,6 +101,7 @@ def print_summary(result: ScanResult, json_path: str) -> None: """Вывести короткую сводку, чтобы результат был понятен без открытия JSON.""" print("\n[*] Scan summary") print(f" Open ports: {len(result.open_ports)}") + print(f" Web services: {len(result.web_services)}") print(f" Issues: {len(result.issues)}") print(f" Discovered paths: {len(result.discovered_paths)}") diff --git a/appsec_framework/web.py b/appsec_framework/web.py index abcbb6a..01f24b8 100644 --- a/appsec_framework/web.py +++ b/appsec_framework/web.py @@ -8,6 +8,7 @@ from __future__ import annotations import re +import time import uuid from dataclasses import dataclass from html.parser import HTMLParser @@ -33,9 +34,12 @@ requests = None -# Набор web-портов держится явным списком: так отчет не пытается открывать -# HTTP-сессию на каждом найденном TCP-сервисе вроде SSH или PostgreSQL. +# Список используется только для диагностического сообщения: protocol probing +# выполняется на каждом открытом порту, включая нестандартные. WEB_PORTS = {80, 81, 443, 3000, 5000, 8000, 8008, 8080, 8081, 8088, 8443, 8888, 9000} +TLS_FIRST_PORTS = {443, 4443, 8443, 9443} +MAX_RESPONSE_BYTES = 512 * 1024 +MAX_JAVASCRIPT_BYTES = 200 * 1024 # Для sensitive paths одного HTTP 200 мало: некоторые приложения отдают # кастомную страницу ошибки с кодом 200. Поэтому у каждого пути есть несколько @@ -99,6 +103,9 @@ class WebScanOutput: issues: List[Issue] discovered_paths: List[DiscoveredPath] errors: List[str] + base_url: Optional[str] = None + status_code: Optional[int] = None + server: str = "" class ScriptParser(HTMLParser): @@ -138,13 +145,20 @@ def __init__( fuzz_wordlist: Optional[str], fuzz_limit: int, skip_fuzz: bool, + request_delay: float = 0.05, ) -> None: self.timeout = timeout self.verify_tls = verify_tls self.fuzz_wordlist = fuzz_wordlist self.fuzz_limit = fuzz_limit self.skip_fuzz = skip_fuzz + self.request_delay = request_delay + self._request_count = 0 self.session = requests.Session() if requests else None + if self.session: + self.session.headers.update( + {"User-Agent": "AppSec-Recon-Framework/1.0 authorized-security-scan"} + ) @property def available(self) -> bool: @@ -164,18 +178,24 @@ def scan_service(self, target: str, port: int) -> WebScanOutput: errors=["Python package 'requests' is not installed. Run: pip install -r requirements.txt"], ) - base_url = build_base_url(target, port) issues: List[Issue] = [] discovered_paths: List[DiscoveredPath] = [] errors: List[str] = [] - try: - homepage = self._get(base_url + "/", allow_redirects=True) - except requests.RequestException as exc: - return WebScanOutput([], [], [f"{base_url}: {exc}"]) + base_url, homepage = self._probe_http_service(target, port) + if not base_url or homepage is None: + errors = [f"{target}:{port}: HTTP(S) probe failed"] if is_web_port(port) else [] + return WebScanOutput([], [], errors) if homepage.status_code >= 500: - return WebScanOutput([], [], [f"{base_url}: HTTP {homepage.status_code} on homepage"]) + return WebScanOutput( + [], + [], + [f"{base_url}: HTTP {homepage.status_code} on homepage"], + base_url=base_url, + status_code=homepage.status_code, + server=homepage.headers.get("Server", ""), + ) issues.extend(self._check_security_headers(target, base_url, homepage)) issues.extend(self._check_cors(target, base_url)) @@ -185,7 +205,26 @@ def scan_service(self, target: str, port: int) -> WebScanOutput: if not self.skip_fuzz and self.fuzz_wordlist: discovered_paths.extend(self._fuzz_paths(target, base_url)) - return WebScanOutput(issues, discovered_paths, errors) + return WebScanOutput( + issues, + discovered_paths, + errors, + base_url=base_url, + status_code=homepage.status_code, + server=homepage.headers.get("Server", ""), + ) + + def _probe_http_service(self, target: str, port: int): + """Определить HTTP(S) по протоколу, а не только по номеру порта.""" + for base_url in candidate_base_urls(target, port): + try: + # Redirects не следуем автоматически: Location может вывести + # scanner за подтвержденный scope на другой origin. + response = self._get(base_url + "/", allow_redirects=False) + except requests.RequestException: + continue + return base_url, response + return None, None def _check_security_headers(self, target: str, base_url: str, response) -> List[Issue]: """Проверить защитные HTTP-заголовки на базовом уровне. @@ -304,10 +343,10 @@ def _check_sensitive_paths(self, target: str, base_url: str) -> List[Issue]: body = response.text[:8192] signature_hit = any(signature.lower() in body.lower() for signature in signatures) - if not signature_hit and path not in {"/swagger.json"}: - # Если код 200 есть, но сигнатуры не совпали, finding остается, - # но severity снижается. Это уменьшает шум от кастомных 200-страниц. - severity = "medium" if severity in {"high", "critical"} else severity + if not signature_hit: + # HTTP 200 без ожидаемой сигнатуры считается soft-404 или + # страницей логина, а не подтвержденной утечкой. + continue issues.append( Issue( @@ -417,25 +456,58 @@ def _fuzz_paths(self, target: str, base_url: str) -> List[DiscoveredPath]: def _safe_get_text(self, url: str) -> str: """Скачать JS-файл с ограничением размера и без падения всего скана.""" try: - response = self._get(url, headers={"Range": "bytes=0-200000"}, allow_redirects=True) - if response.status_code >= 400: + response = self._get( + url, + headers={"Range": f"bytes=0-{MAX_JAVASCRIPT_BYTES - 1}"}, + allow_redirects=False, + max_bytes=MAX_JAVASCRIPT_BYTES, + ) + if response.status_code >= 300: return "" content_type = response.headers.get("Content-Type", "") if "javascript" not in content_type and not url.lower().endswith(".js"): return "" - return response.text[:200000] + return response.text[:MAX_JAVASCRIPT_BYTES] except requests.RequestException: return "" - def _get(self, url: str, **kwargs): - """Единая точка для HTTP GET, чтобы таймауты и TLS-настройки были одинаковыми.""" - return self.session.get( + def _get(self, url: str, max_bytes: int = MAX_RESPONSE_BYTES, **kwargs): + """Выполнить bounded GET с общей задержкой и лимитом ответа.""" + if self._request_count and self.request_delay: + time.sleep(self.request_delay) + self._request_count += 1 + response = self.session.get( url, timeout=self.timeout, verify=self.verify_tls, headers=kwargs.pop("headers", None), + stream=True, **kwargs, ) + declared_length = response.headers.get("Content-Length") + if declared_length: + try: + too_large = int(declared_length) > max_bytes + except ValueError: + too_large = False + if too_large: + response.close() + raise requests.RequestException( + f"Response exceeds {max_bytes} byte safety limit" + ) + chunks = [] + received = 0 + for chunk in response.iter_content(chunk_size=16 * 1024): + received += len(chunk) + if received > max_bytes: + response.close() + raise requests.RequestException( + f"Response exceeds {max_bytes} byte safety limit" + ) + chunks.append(chunk) + response._content = b"".join(chunks) + response._content_consumed = True + return response def is_web_port(port: int) -> bool: @@ -451,6 +523,17 @@ def build_base_url(target: str, port: int) -> str: return f"{scheme}://{host}" if default_port else f"{scheme}://{host}:{port}" +def candidate_base_urls(target: str, port: int) -> List[str]: + """Вернуть HTTP/HTTPS кандидаты с наиболее вероятной схемой первой.""" + host = f"[{target}]" if ":" in target and not target.startswith("[") else target + schemes = ("https", "http") if port in TLS_FIRST_PORTS else ("http", "https") + urls = [] + for scheme in schemes: + default_port = (scheme == "http" and port == 80) or (scheme == "https" and port == 443) + urls.append(f"{scheme}://{host}" if default_port else f"{scheme}://{host}:{port}") + return urls + + def load_wordlist(path: Optional[str], limit: int) -> List[str]: """Загрузить wordlist с поддержкой комментариев и лимита на размер.""" if not path: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7a2e7cb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +services: + demo: + build: . + entrypoint: ["python", "examples/vulnerable_demo_server.py"] + environment: + DEMO_HOST: 0.0.0.0 + DEMO_PORT: 8088 + ports: + - "127.0.0.1:8088:8088" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8088/', timeout=1)"] + interval: 3s + timeout: 2s + retries: 10 + + scanner: + build: . + profiles: ["tools"] + depends_on: + demo: + condition: service_healthy + volumes: + - ./reports:/app/reports + command: + - demo + - --ports + - "8088" + - --acknowledge-authorization + - --json-out + - reports/docker-demo.json + - --markdown-out + - reports/docker-demo.md diff --git a/examples/vulnerable_demo_server.py b/examples/vulnerable_demo_server.py index 8173c96..14a6ee5 100755 --- a/examples/vulnerable_demo_server.py +++ b/examples/vulnerable_demo_server.py @@ -85,7 +85,7 @@ def main(): Порт можно переопределить через DEMO_PORT, если 8088 уже занят. """ - host = "127.0.0.1" + host = os.environ.get("DEMO_HOST", "127.0.0.1") port = int(os.environ.get("DEMO_PORT", "8088")) server = HTTPServer((host, port), DemoHandler) print(f"Demo vulnerable app listening on http://{host}:{port}") diff --git a/pyproject.toml b/pyproject.toml index 799268c..f32e841 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,31 @@ build-backend = "setuptools.build_meta" [project] name = "appsec-recon-framework" -version = "0.1.0" +version = "1.0.0" description = "Lightweight AppSec reconnaissance framework with TCP scanning, web checks, fuzzing, and JSON reports." +readme = "README_EN.md" +license = {text = "MIT"} requires-python = ">=3.9" dependencies = ["requests>=2.31.0,<3", "urllib3<2"] +keywords = ["appsec", "reconnaissance", "security", "scanner"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Security", +] + +[project.urls] +Repository = "https://github.com/fant3k/AppSec-Recon-Framework" +Issues = "https://github.com/fant3k/AppSec-Recon-Framework/issues" [project.scripts] appsec-recon = "appsec_framework.cli:main" + +[tool.setuptools.packages.find] +include = ["appsec_framework*"] + +[tool.setuptools.package-data] +appsec_framework = ["data/*.txt"] diff --git a/scripts/pipeline_subfinder.sh b/scripts/pipeline_subfinder.sh index dbd9857..497ffbf 100755 --- a/scripts/pipeline_subfinder.sh +++ b/scripts/pipeline_subfinder.sh @@ -9,6 +9,7 @@ fi DOMAIN="$1" shift +SAFE_DOMAIN="$(printf '%s' "${DOMAIN}" | tr -c 'A-Za-z0-9._-' '_')" if ! command -v subfinder >/dev/null 2>&1; then echo "[-] subfinder is not installed." @@ -18,10 +19,10 @@ fi STAMP="$(date -u +%Y%m%d_%H%M%S)" REPORT_DIR="reports" -TARGETS_FILE="${REPORT_DIR}/subdomains_${DOMAIN}_${STAMP}.txt" -JSON_OUT="${REPORT_DIR}/scan_${DOMAIN}_${STAMP}.json" -MD_OUT="${REPORT_DIR}/scan_${DOMAIN}_${STAMP}.md" -NUCLEI_TARGETS="${REPORT_DIR}/nuclei_targets_${DOMAIN}_${STAMP}.txt" +TARGETS_FILE="${REPORT_DIR}/subdomains_${SAFE_DOMAIN}_${STAMP}.txt" +JSON_OUT="${REPORT_DIR}/scan_${SAFE_DOMAIN}_${STAMP}.json" +MD_OUT="${REPORT_DIR}/scan_${SAFE_DOMAIN}_${STAMP}.md" +NUCLEI_TARGETS="${REPORT_DIR}/nuclei_targets_${SAFE_DOMAIN}_${STAMP}.txt" mkdir -p "${REPORT_DIR}" @@ -37,6 +38,7 @@ python3 appsec_scan.py \ --json-out "${JSON_OUT}" \ --markdown-out "${MD_OUT}" \ --nuclei-targets "${NUCLEI_TARGETS}" \ + --acknowledge-authorization \ "$@" if [[ -n "${TELEGRAM_BOT_TOKEN:-}" && -n "${TELEGRAM_CHAT_ID:-}" ]]; then diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..210361f --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +python3 -B -m unittest discover -s tests -v +PYTHONPYCACHEPREFIX=/tmp/appsec_recon_pycache python3 -m compileall -q appsec_framework examples diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..f2baccf --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,39 @@ +import unittest + +from appsec_framework.cli import build_parser, validate_runtime_options +from appsec_framework.targets import load_targets, parse_ports + + +class CliSafetyTests(unittest.TestCase): + def args(self, *extra): + return build_parser().parse_args(["127.0.0.1", *extra]) + + def test_loopback_demo_does_not_require_authorization_flag(self): + args = self.args("--ports", "8088") + validate_runtime_options(args, ["127.0.0.1"], [8088]) + + def test_external_target_requires_explicit_authorization(self): + args = self.args("--ports", "443") + with self.assertRaisesRegex(ValueError, "acknowledge-authorization"): + validate_runtime_options(args, ["example.com"], [443]) + + def test_invalid_resource_limits_are_rejected(self): + for option, value in [ + ("--threads", "0"), + ("--timeout", "0"), + ("--fuzz-limit", "5001"), + ("--request-delay", "-1"), + ]: + with self.subTest(option=option): + args = self.args(option, value) + with self.assertRaises(ValueError): + validate_runtime_options(args, ["127.0.0.1"], [80]) + + def test_scan_job_limit_is_enforced(self): + args = self.args("--max-scan-jobs", "2") + with self.assertRaisesRegex(ValueError, "would create"): + validate_runtime_options(args, ["127.0.0.1"], [80, 81, 82]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_network.py b/tests/test_network.py new file mode 100644 index 0000000..8b8f024 --- /dev/null +++ b/tests/test_network.py @@ -0,0 +1,40 @@ +import socket +import threading +import unittest + +from appsec_framework.network import scan_targets + + +class NetworkScannerTests(unittest.TestCase): + def test_scan_targets_finds_local_listener(self): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + + def accept_once(): + connection, _ = listener.accept() + connection.close() + listener.close() + + thread = threading.Thread(target=accept_once, daemon=True) + thread.start() + findings = scan_targets(["127.0.0.1"], [port], timeout=0.5, threads=2) + thread.join(timeout=1) + + self.assertEqual(len(findings), 1) + self.assertEqual(findings[0].port, port) + + def test_scan_targets_accepts_a_large_iterable_without_global_job_queue(self): + findings = scan_targets( + ["127.0.0.1"], + range(1, 20), + timeout=0.01, + threads=2, + batch_multiplier=1, + ) + self.assertIsInstance(findings, list) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reporting.py b/tests/test_reporting.py new file mode 100644 index 0000000..40af66f --- /dev/null +++ b/tests/test_reporting.py @@ -0,0 +1,55 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from appsec_framework.models import Issue, PortFinding, ScanResult, WebService +from appsec_framework.reporting import ( + write_json_report, + write_markdown_report, + write_nuclei_targets, +) + + +class ReportingTests(unittest.TestCase): + def result(self): + return ScanResult( + scanner="test/1", + started_at="2026-01-01T00:00:00+00:00", + finished_at="2026-01-01T00:00:01+00:00", + options={"targets": ["127.0.0.1"]}, + open_ports=[PortFinding("127.0.0.1", 7000)], + web_services=[WebService("127.0.0.1", 7000, "http://127.0.0.1:7000", 200)], + issues=[ + Issue( + id="demo", + title="Demo finding", + severity="high", + target="127.0.0.1", + url="http://127.0.0.1:7000", + evidence="safe evidence", + recommendation="Fix it", + ) + ], + ) + + def test_all_report_formats_are_consistent(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + json_path = root / "scan.json" + markdown_path = root / "scan.md" + nuclei_path = root / "targets.txt" + result = self.result() + + write_json_report(result, str(json_path)) + write_markdown_report(result, str(markdown_path)) + write_nuclei_targets(result, str(nuclei_path)) + + payload = json.loads(json_path.read_text()) + self.assertEqual(payload["web_services"][0]["port"], 7000) + self.assertIn("Demo finding", markdown_path.read_text()) + self.assertEqual(nuclei_path.read_text(), "http://127.0.0.1:7000\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..ff043d1 --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,101 @@ +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib.resources import files + +import requests + +from appsec_framework.web import ( + MAX_RESPONSE_BYTES, + WebScanner, + candidate_base_urls, +) +from examples.vulnerable_demo_server import DemoHandler + + +class QuietSoft404Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"generic login page" + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + return + + +class LargeResponseHandler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"x" * (MAX_RESPONSE_BYTES + 1) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + return + + +class WebScannerTests(unittest.TestCase): + def start_server(self, handler): + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(server.shutdown) + self.addCleanup(server.server_close) + return server.server_address[1] + + def scanner(self, wordlist=None): + return WebScanner( + timeout=1, + verify_tls=False, + fuzz_wordlist=wordlist, + fuzz_limit=100, + skip_fuzz=wordlist is None, + request_delay=0, + ) + + def test_demo_detects_all_documented_findings_on_arbitrary_port(self): + port = self.start_server(DemoHandler) + wordlist = str(files("appsec_framework").joinpath("data/common-web.txt")) + output = self.scanner(wordlist).scan_service("127.0.0.1", port) + + self.assertEqual(output.base_url, f"http://127.0.0.1:{port}") + self.assertEqual(output.status_code, 200) + self.assertEqual(len(output.issues), 7) + self.assertEqual( + {issue.id for issue in output.issues}, + { + "missing_csp", + "missing_clickjacking_protection", + "missing_nosniff", + "cors_misconfiguration", + "exposed_sensitive_path", + "potential_js_secret", + }, + ) + self.assertEqual([path.url.rsplit("/", 1)[-1] for path in output.discovered_paths], ["admin"]) + secret = next(issue for issue in output.issues if issue.id == "potential_js_secret") + self.assertIn("