diff --git a/.github/workflows/python-ci.yaml b/.github/workflows/python-ci.yaml new file mode 100644 index 0000000..0fc18d2 --- /dev/null +++ b/.github/workflows/python-ci.yaml @@ -0,0 +1,106 @@ +name: Python CI + +on: + push: + branches: [ main, develop, release ] + pull_request: + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create venv + run: python3 -m venv .venv + + - name: Install build deps + run: | + .venv/bin/pip install --upgrade pip + .venv/bin/pip install .[dev] + + - name: Validate config + run: | + set -e + .venv/bin/python -m altbrow --validate-config --config="tests/data/altbrow.toml" + + - name: Build package + run: .venv/bin/python -m build + + - name: Build docs + run: .venv/bin/mkdocs build -f docs/mkdocs.yml + + validate: + runs-on: ubuntu-latest + # parallel to build — ruff and pytest don't need a built package + steps: + - uses: actions/checkout@v4 + + - name: Create venv + run: python3 -m venv .venv + + - name: Install dev deps + run: | + .venv/bin/pip install --upgrade pip + .venv/bin/pip install .[dev] + + - name: Ruff + run: .venv/bin/ruff check altbrow + + - name: Pytest + run: .venv/bin/pytest + + integration: + runs-on: ubuntu-latest + needs: [ build, validate ] + steps: + - uses: actions/checkout@v4 + + - name: Create venv + run: python3 -m venv .venv + + - name: Install deps + run: | + .venv/bin/pip install --upgrade pip + .venv/bin/pip install .[dev] + + - name: Start mock server + run: | + python3 -m http.server 8080 --directory tests/data & + for i in $(seq 1 10); do + python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/')" 2>/dev/null && break + sleep 1 + done + + - name: Build cache with mock provider + run: | + .venv/bin/python -m altbrow \ + --config="tests/data/altbrow.toml" \ + --build-cache + + - name: Run integration tests + run: .venv/bin/pytest tests/test_cache.py -v + +# integration-dns: +# runs-on: ubuntu-latest +# needs: validate +# # only runs when PIHOLE_IP is configured as repository variable +# if: vars.PIHOLE_IP != '' +# steps: +# - uses: actions/checkout@v4 +# +# - name: Create venv +# run: python3 -m venv .venv +# +# - name: Install deps +# run: | +# .venv/bin/pip install --upgrade pip +# .venv/bin/pip install .[dev] +# +# - name: Run DNS provider tests +# env: +# ALTBROW_PIHOLE_IP: ${{ vars.PIHOLE_IP }} +# run: .venv/bin/pytest tests/test_dns_provider.py -v diff --git a/.gitignore b/.gitignore index 5f138fe..5907c1f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,12 +17,15 @@ __pycache__/ # --- OS / Editor --- .DS_Store Thumbs.db +.stack.md # --- Test / build --- .pytest_cache/ .mypy_cache/ *.egg-info/ .coverage +.altbrow.cache +config/.altbrow.cache dist/ build/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 469683c..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,31 +0,0 @@ -image: python:3.12-slim - -stages: - - build - - lint - -variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - -cache: - paths: - - .cache/pip - -build: - stage: build - allow_failure: false - script: - - python -m pip install --upgrade pip - - pip install . - - altbrow --validate-config - - -lint: - stage: lint - allow_failure: false - script: - - python -m pip install --upgrade pip - - pip install .[dev] - - ruff check app - - cd docs - - mkdocs build --strict diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..94da310 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +// .vscode/launch.json +{ + "configurations": [ + { + "name": "altbrow dev", + "type": "python", + "request": "launch", + "module": "altbrow", + "args": ["--config", "tests/data/altbrow.toml", "--validate-config"] + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 0bf6b15..82f458c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,22 +2,22 @@ "files.eol": "\n", "files.exclude": { - "**/.venv312": true + "**/.venv": true }, "search.exclude": { - "**/.venv312": true + "**/.venv": true }, "python.analysis.exclude": [ - "**/.venv312" + "**/.venv" ], "python.analysis.typeCheckingMode": "basic", - "python.defaultInterpreterPath": ".venv312\\Scripts\\python.exe", + "python.defaultInterpreterPath": ".venv\\Scripts\\python.exe", "python.terminal.activateEnvironment": true, "files.watcherExclude": { - "**/.venv312/**": true + "**/.venv/**": true } } diff --git a/README.md b/README.md index 1cc3e43..8e61e65 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,192 @@ # Altbrow -An alternative, crawler-like browser that looks beneath the surface of the semantic web. This project is in a very early alpha stage. +Alternative crawler-like browser for a deep look into a website's semantic structure and external dependencies. +Includes a provider system for domain and IP classification via inline lists, local files, remote feeds, and DNS resolvers. +> Early alpha — built with AI assistance. + +## Goals + +Rapid review and evaluation of a website based on publicly available information: semantic content, formal structure, and exportable data for further analysis. ## Usage -``` +### Install -.venv312\Scripts\activate +```bash pip install -e . -altbrow --help -altbrow https://tmp.gedankenfalle.de/html5 +``` + +### CLI ``` + +usage: altbrow [-h] [-V] [--config CONFIG] [-o OUTPUT] [-f {text,yaml,json}] [-v] [--client-profile {passive,browser,consented}] [--no-cert-check] [--validate-config] [--build-cache] [--debug] [--log-file PATH] + [url] + +positional arguments: + url URL to analyze + +options: + -h, --help show this help message and exit + -V, --version show program's version number and exit + --config CONFIG Path to configuration file (TOML) + -o OUTPUT, --output OUTPUT + Write result to file + -f {text,yaml,json}, --format {text,yaml,json} + Output format (default: text) + -v, --verbose Increase text detail level (-vv, -vvv) + --client-profile {passive,browser,consented} + HTTP client behavior profile (default: from config) + --no-cert-check Disable TLS certificate verification (self-signed certs, local hosts) + --validate-config Validate altbrow.toml & provider.toml and exit + --build-cache (Re)build provider cache DB, unpack geoIP mmdd files and exit + --debug Enable debug logging (steps, DNS queries, cache hits) + --log-file PATH Write log to file (default: altbrow.log next to altbrow.toml) + + ``` + + +### Config discovery + + +1) --config /etc/altbrow.toml → /etc/provider.toml (explicit) +2) ~/.altbrow/altbrow.toml → ~/.altbrow/provider.toml (user) +3) ./altbrow.toml → ./provider.toml (portable) + +If no config exists, defaults are generated in `~/.altbrow/` on first run. + +> Cache size: up to ~0.5 GB with full remote provider lists enabled. + + + +## Exit Codes + +| Code | Meaning | +|------|-----------------------| +| 0 | Success | +| 1 | Unhandled exception | +| 2 | CLI usage error | +| 3 | Config error | +| 4 | Network / TLS error | +| 5 | HTTP error (4xx/5xx) | + + +## Provider System + +Classifies domains and IPs using configurable providers: inline lists, local files, remote blocklists, DNS resolvers, and GeoIP databases. + +### Categories + +Each enabled provider requires at least one enabled category. +`name = "string"` is optional in all sections — TOML key is used as fallback. + +#### 8 regular categories + +| Category | Description | +|-------------|----------------------------------------------------------| +| `ads` | Advertising networks and ad delivery | +| `analytics` | User behaviour measurement and reporting | +| `cdn` | Content delivery networks and static asset hosting | +| `malware` | Malware, phishing, known hostile domains | +| `social` | Social networks, dating, gambling, adult content | +| `suspicious`| Unverified or potentially hostile | +| `telemetry` | Error reporting, performance monitoring, device telemetry| +| `tracking` | Cross-site user tracking and profiling | + +#### 3 special categories + +| Category | Description | +|-----------------|------------------------------------------------------| +| `local` | RFC1918, localhost, loopback, your domains | +| `infrastructure`| Web standards, semantic namespaces, DNS resolvers | +| `unknown` | default if no provider hit | + +#### Automatic classifications (no provider needed) + +| Value | Description | +|---------------|--------------------------------------------------------------------| +| `FIRST_PARTY` | Same registrable domain as the analysed page | +| `PEER` | Lateral sibling (same registrable domain, different subdomain) | +| `SUBDOMAIN` | Strict subdomain of the TARGET host | +| `EXTERNAL` | Different registrable domain | + +### Provider schema + +```toml + +[provider.name] +name = "Human readable label" # optional +location = "local|inline|remote|dns" +type = "ip|domain" +enabled = true|false +subdomain_match = true|false # optional, default: true + +[[provider.name.category]] +name = "Human readable label" # optional +enabled = true|false # optional, default: true +tier = # optional, inline/local default: 1, dns/remote default: 2 +mapping = [""] # one or more regular or special categories + +# location = "inline" — domain or IP/CIDR list directly in config +source = ["example.com", "cdn.example.net"] +source = ["192.168.1.0/24", "10.0.0.1"] + +# location = "local" — file path relative to provider.toml or absolute +source = ["./blocklist.txt"] +source = ["C:\\Windows\\System32\\drivers\\etc\\hosts"] + +# location = "remote" — HTTP/HTTPS URL +source = ["https://example.com/list.txt"] + +# location = "dns" — resolver IPs per category, sinkhole required +source = ["208.67.222.222", "208.67.220.220"] +sinkhole = ["146.112.61.104", "::ffff:146.112.61.104"] + +``` + +#### GeoIP provider (MaxMind GeoLite2) + +GeoIP uses `location = "local"` or `location = "remote"` with `mapping = ["geoip"]`. +Processed only during `--build-cache`. Requires free registration at MaxMind. + +```toml + +[provider.maxmind] +location = "local" +type = "ip" +enabled = true + +[[provider.maxmind.category]] +name = "ASN" +mapping = ["geoip"] +source = ["./GeoLite2-ASN_*.tar.gz"] # glob, newest match is used + +``` + +## Example + +```bash + +altbrow -v tmp.gedankenfalle.de/html5 + +=== Summary === +External domains : 25 (infrastructure: 9, local: 1, malware: 1, telemetry: 2, unknown: 12) (US: 20, DE: 3, AT: 1, IE: 1) +External IPs : 0 +Cookies : 0 +JSON-LD blocks : 0 +Microdata blocks : 2 + +=== External Domains === +FIRST_PARTY TARGET tmp.gedankenfalle.de local [AT AS197540 netcup GmbH] +EXTERNAL LINK_ONLY creativecommons.org unknown [US AS13335 Cloudflare, Inc.] +EXTERNAL LINK_ONLY developer.mozilla.org infrastructure [DE/Munich AS54113 Fastly, Inc.] +EXTERNAL LINK_ONLY rdflib.readthedocs.io malware [US AS13335 Cloudflare, Inc.] +EXTERNAL LINK_ONLY schema.org infrastructure [US AS15169 Google LLC] +EXTERNAL LINK_ONLY www.w3.org infrastructure [US AS13335 Cloudflare, Inc.] +... + +external domains classified by provider category and GeoIP — with one false positive: +`rdflib.readthedocs.io` as `malware` is NOT true. + +``` \ No newline at end of file diff --git a/altbrow/__init__.py b/altbrow/__init__.py new file mode 100644 index 0000000..7de9ad5 --- /dev/null +++ b/altbrow/__init__.py @@ -0,0 +1,9 @@ +# altbrow/__init__.py + +from importlib.metadata import version, PackageNotFoundError + +try: + __version__ = version("altbrow") +except PackageNotFoundError: + # fallback when running from source tree to minor + __version__ = "0.1.0" diff --git a/altbrow/__main__.py b/altbrow/__main__.py new file mode 100644 index 0000000..76e4666 --- /dev/null +++ b/altbrow/__main__.py @@ -0,0 +1,7 @@ +# altbrow/__main__.py + +import sys +from .main import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/altbrow/cache.py b/altbrow/cache.py new file mode 100644 index 0000000..812f274 --- /dev/null +++ b/altbrow/cache.py @@ -0,0 +1,511 @@ +# altbrow/cache.py +# +# build_cache() +# get_or_build_cache() +# lookup_domain() +# lookup_ip() + +import ipaddress +import logging +import sqlite3 + +from datetime import datetime, timezone +from pathlib import Path + +from altbrow import __version__ +from .config import LOCATION_DEFAULT_TIER + +logger = logging.getLogger(__name__) + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS domains ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL, + registrable_domain TEXT, + category TEXT NOT NULL, + provider TEXT NOT NULL, + provider_location TEXT NOT NULL, + category_name TEXT, + updated_at TEXT NOT NULL, + subdomain_match INTEGER NOT NULL DEFAULT 1, + tier INTEGER NOT NULL DEFAULT 2, + UNIQUE(value, category, provider) +); + +CREATE INDEX IF NOT EXISTS idx_domains_value + ON domains(value); + +CREATE INDEX IF NOT EXISTS idx_domains_registrable + ON domains(registrable_domain); + +CREATE TABLE IF NOT EXISTS ips ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL, + is_cidr INTEGER NOT NULL DEFAULT 0, + category TEXT NOT NULL, + provider TEXT NOT NULL, + provider_location TEXT NOT NULL, + category_name TEXT, + tier INTEGER NOT NULL DEFAULT 2, + updated_at TEXT NOT NULL, + UNIQUE(value, category, provider) +); + +CREATE INDEX IF NOT EXISTS idx_ips_value + ON ips(value); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +""" + + +def _now() -> str: + """Return current UTC time as ISO8601 string.""" + return datetime.now(timezone.utc).isoformat() + + +def _get_registrable_domain(domain: str) -> str | None: + """Extract registrable domain (e.g. 'example.com' from 'cdn.example.com'). + + Args: + domain: Fully qualified domain name. + + Returns: + Registrable domain string, or None if not determinable. + """ + try: + import tldextract + ext = tldextract.extract(domain) + if ext.domain and ext.suffix: + return f"{ext.domain}.{ext.suffix}".lower() + except Exception: + pass + return None + + +def _is_cidr(value: str) -> bool: + """Return True if value is an IP network in CIDR notation. + + Args: + value: String to check. + + Returns: + True if value is a valid CIDR block, False otherwise. + """ + try: + ipaddress.ip_network(value, strict=False) + return "/" in value + except ValueError: + return False + + +def _load_local_source(path_str: str, config_path: Path) -> list[str]: + """Read entries from a local file in altbrow list or hosts format. + + Supports both plain domain/IP lists and hosts file format + (``0.0.0.0 domain`` / ``127.0.0.1 domain`` lines) via parse_entries(). + Absolute paths (e.g. /etc/hosts) are used as-is. + + Args: + path_str: File path as defined in provider source (absolute or relative + to altbrow.toml). + config_path: Path to altbrow.toml, used to resolve relative paths. + + Returns: + List of domain or IP strings. + """ + from .fetch_remote import parse_entries + + source_path = Path(path_str) + if not source_path.is_absolute(): + source_path = config_path.parent / source_path + + if not source_path.exists(): + logger.warning("Local source not found: %s", source_path) + return [] + + return parse_entries(source_path.read_text(encoding="utf-8", errors="replace")) + + +def build_cache( + cache_path: Path, + provider_config: dict, + config_path: Path, +) -> None: + """Build the SQLite cache from all enabled static providers. + + Reads inline, local, and remote providers from provider_config. + DNS providers are skipped — they are queried live at runtime. + + Args: + cache_path: Path where the SQLite DB will be created or replaced. + provider_config: Parsed provider.toml dictionary. + config_path: Path to altbrow.toml (used to resolve relative paths). + """ + logger.info("Building cache: %s", cache_path) + cache_path.parent.mkdir(parents=True, exist_ok=True) + + # always start fresh — partial updates make no sense when all sources are re-read + if cache_path.exists(): + cache_path.unlink() + logger.info("Removed existing cache: %s", cache_path) + + con = sqlite3.connect(cache_path) + + # performance pragmas — must be set before schema creation + con.execute("PRAGMA page_size = 4096") + con.execute("PRAGMA journal_mode = OFF") + con.execute("PRAGMA synchronous = OFF") + con.execute("PRAGMA temp_store = MEMORY") + con.execute("PRAGMA cache_size = -64000") # 64MB build cache + + con.executescript(SCHEMA) + + now = _now() + domain_rows = [] + ip_rows = [] + active_providers = [] + + providers = provider_config.get("provider", {}) + + for pname, p in providers.items(): + if not p.get("enabled", False): + logger.debug("Provider '%s' disabled, skipping", pname) + continue + + location = p.get("location") + ptype = p.get("type") + + # DNS providers are live-only, not cached statically + if location == "dns": + logger.debug("Provider '%s' is dns, skipping for static cache", pname) + continue + + # geoip providers are handled by extract_geodbs, not inserted into DB + if ptype == "geoip": + logger.debug("Provider '%s' is geoip, handled by extract_geodbs", pname) + continue + + active_providers.append(pname) + logger.info("Loading provider '%s' (%s/%s)", pname, location, ptype) + subdomain_match = 1 if p.get("subdomain_match", True) else 0 + + # remote providers: fetch_remote_provider handles all categories internally + if location == "remote": + from .fetch_remote import fetch_remote_provider + for entry, ctx in fetch_remote_provider(pname, p): + remote_tier = ctx.get("tier", LOCATION_DEFAULT_TIER["remote"]) + if ctx["ptype"] == "domain": + reg = _get_registrable_domain(entry) + domain_rows.append(( + entry, reg, ctx["category"], pname, + "remote", ctx["category_name"], remote_tier, now, subdomain_match, + )) + elif ctx["ptype"] == "ip": + cidr_flag = 1 if _is_cidr(entry) else 0 + ip_rows.append(( + entry, cidr_flag, ctx["category"], pname, + "remote", ctx["category_name"], remote_tier, now, + )) + continue + + for cat in p.get("category", []): + if not cat.get("enabled", True): + continue + + cat_name = cat.get("name") + mappings = cat.get("mapping", []) + sources = cat.get("source", []) + tier = cat.get("tier", LOCATION_DEFAULT_TIER.get(location, 2)) + + entries: list[str] = [] + + if location == "local": + for src in sources: + entries.extend(_load_local_source(src, config_path)) + + elif location == "inline": + entries = list(sources) + + # Insert each entry × each mapping + for entry in entries: + entry = entry.strip().lower() + if not entry: + continue + + for category in mappings: + + if ptype == "domain": + reg = _get_registrable_domain(entry) + domain_rows.append(( + entry, reg, category, pname, + location, cat_name, tier, now, subdomain_match, + )) + + elif ptype == "ip": + cidr_flag = 1 if _is_cidr(entry) else 0 + ip_rows.append(( + entry, cidr_flag, category, pname, + location, cat_name, tier, now, + )) + + con.executemany( + """INSERT OR IGNORE INTO domains + (value, registrable_domain, category, provider, provider_location, category_name, tier, updated_at, subdomain_match) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + domain_rows, + ) + + con.executemany( + """INSERT OR IGNORE INTO ips + (value, is_cidr, category, provider, provider_location, category_name, tier, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ip_rows, + ) + + con.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", + ("built_at", now), + ) + con.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", + ("altbrow_version", __version__), + ) + + con.commit() + con.close() + + # VACUUM must run outside any transaction in a fresh connection + con2 = sqlite3.connect(cache_path) + con2.execute("VACUUM") + con2.close() + + logger.info( + "Cache built: %d domain entries, %d ip entries from %d providers (%s)", + len(domain_rows), + len(ip_rows), + len(active_providers), + ", ".join(active_providers), + ) + + +def _ensure_schema(cache_path: Path) -> None: + """Create DB tables if they do not exist yet. + + Safe to call on every startup — uses CREATE TABLE IF NOT EXISTS. + + Args: + cache_path: Path to the SQLite cache file. + """ + cache_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(cache_path) + + # performance pragmas — must be set before schema creation + con.execute("PRAGMA page_size = 4096") + con.execute("PRAGMA journal_mode = OFF") + con.execute("PRAGMA synchronous = OFF") + con.execute("PRAGMA temp_store = MEMORY") + con.execute("PRAGMA cache_size = -64000") # 64MB build cache + + con.executescript(SCHEMA) + con.commit() + con.close() + + +def get_or_build_cache( + cache_path: Path, + provider_config: dict | None, + config_path: Path, +) -> Path: + """Return cache path, initialising schema and building lazily if needed. + + Always ensures the DB schema exists. If provider_config is given and + the DB has no domain entries yet, triggers a full build. + + Args: + cache_path: Expected path of the SQLite cache file. + provider_config: Parsed provider.toml dict, or None if disabled. + config_path: Path to altbrow.toml. + + Returns: + Path to the ready cache DB. + """ + _ensure_schema(cache_path) + + if provider_config is None: + return cache_path + + # check if DB is empty — build if so + con = sqlite3.connect(cache_path) + count = con.execute("SELECT COUNT(*) FROM domains").fetchone()[0] + con.close() + + if count == 0: + logger.info("Cache empty, building...") + build_cache(cache_path, provider_config, config_path) + + return cache_path + + +def lookup_domain( + domain: str, + cache_path: Path, + config: dict | None = None, +) -> list[dict]: + """Look up a domain, merging static DB and live DNS provider results. + + Tries exact value match first, then registrable domain fallback. + If config is provided and contains dns providers, queries them live in parallel. + + Args: + domain: Fully qualified domain name to look up. + cache_path: Path to the SQLite cache file. + config: Merged altbrow config dict for DNS provider lookup. Optional. + + Returns: + List of category dicts with keys: + category, provider, provider_location, category_name, tier + Empty list if no match. + """ + domain = domain.lower() + reg = _get_registrable_domain(domain) + + con = sqlite3.connect(cache_path) + con.row_factory = sqlite3.Row + + # exact match — always + rows_exact = con.execute( + "SELECT * FROM domains WHERE value = ?", + (domain,), + ).fetchall() + + # registrable domain match — only for providers with subdomain_match = 1 + # always run, merged with exact results + rows_reg = [] + if reg: + rows_reg = con.execute( + "SELECT * FROM domains WHERE registrable_domain = ? AND subdomain_match = 1", + (reg,), + ).fetchall() + + # merge: exact match + registrable match, skip rows already covered by exact + exact_ids = {r["id"] for r in rows_exact} + rows = list(rows_exact) + [r for r in rows_reg if r["id"] not in exact_ids] + + con.close() + + seen = set() + results = [] + + for r in rows: + key = (r["category"], r["provider"], r["category_name"]) + if key not in seen: + seen.add(key) + results.append({ + "category": r["category"], + "provider": r["provider"], + "provider_location": r["provider_location"], + "category_name": r["category_name"], + "tier": r["tier"], + }) + + # DNS live lookup — all enabled DNS providers are always queried. + # DNS providers (Pi-hole, OpenDNS) are independent classification sources. + # dns-resolve-filter controls which provider categories are queried + # inside dns_provider_lookup, not whether DNS runs at all. + if config: + from .dns_lookup import dns_provider_lookup + logger.debug("DNS provider lookup for: %s", domain) + dns_results = dns_provider_lookup(domain, config) + if dns_results: + existing = {(r["category"], r["provider"]) for r in results} + for r in dns_results: + if (r["category"], r["provider"]) not in existing: + results.append(r) + + # resolve-domains: resolve domain to IP and check against IP provider lists + if config: + resolve = config.get("resolve", {}) + from .config import RESOLVE_DEFAULTS + if resolve.get("resolve-domains", RESOLVE_DEFAULTS["resolve-domains"]): + import socket as _socket + try: + addr_infos = _socket.getaddrinfo(domain, None, _socket.AF_INET) + if addr_infos: + ip_str = addr_infos[0][4][0] + ip_results = lookup_ip(ip_str, cache_path) + if ip_results: + existing = {(r["category"], r["provider"]) for r in results} + for r in ip_results: + if (r["category"], r["provider"]) not in existing: + r["resolved_from"] = domain + results.append(r) + except Exception as exc: + logger.debug("resolve-domains failed for %s: %s", domain, exc) + + return results + + +def lookup_ip(ip_str: str, cache_path: Path) -> list[dict]: + """Look up an IP address, matching exact IPs and CIDRs. + + Exact matches via SQL, CIDR matches resolved in Python via ipaddress stdlib. + DNS results are not merged for IPs (DNS providers work on domain level). + + Args: + ip_str: IP address string (IPv4 or IPv6). + cache_path: Path to the SQLite cache file. + + Returns: + List of category dicts with keys: + category, provider, provider_location, category_name + Empty list if no match or invalid IP. + """ + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + logger.warning("Invalid IP for lookup: %s", ip_str) + return [] + + con = sqlite3.connect(cache_path) + con.row_factory = sqlite3.Row + + rows = con.execute( + "SELECT * FROM ips WHERE value = ? AND is_cidr = 0", + (ip_str,), + ).fetchall() + + results = [ + { + "category": r["category"], + "provider": r["provider"], + "provider_location": r["provider_location"], + "category_name": r["category_name"], + "tier": r["tier"], + } + for r in rows + ] + + cidr_rows = con.execute( + "SELECT * FROM ips WHERE is_cidr = 1" + ).fetchall() + + con.close() + + for row in cidr_rows: + try: + network = ipaddress.ip_network(row["value"], strict=False) + if ip in network: + results.append({ + "category": row["category"], + "provider": row["provider"], + "provider_location": row["provider_location"], + "category_name": row["category_name"], + "tier": row["tier"], + }) + except ValueError: + logger.warning("Invalid CIDR in cache: %s", row["value"]) + + return results diff --git a/altbrow/classify_cookies.py b/altbrow/classify_cookies.py new file mode 100644 index 0000000..434476f --- /dev/null +++ b/altbrow/classify_cookies.py @@ -0,0 +1,45 @@ +# altbrow/classify_cookies.py +# +# classify_cookies() + + +def classify_cookies(raw_cookie: str, page_domain: str) -> dict: + """Parse and classify a raw Set-Cookie header value. + + Determines whether the cookie is third-party and whether it is + configured for cross-site delivery (SameSite=None). + + Args: + raw_cookie: Raw Set-Cookie header string (single cookie). + page_domain: Hostname of the analysed page. + + Returns: + Dict with keys: + name - cookie name + third_party - True if cookie domain differs from page domain + cross_site - True if SameSite=None is set + attributes - list of raw attribute strings (e.g. 'HttpOnly', 'Path=/') + """ + parts = [p.strip() for p in raw_cookie.split(";")] + name, *_ = parts[0].split("=", 1) + + result = { + "name": name, + "third_party": False, + "cross_site": False, + "attributes": [], + } + + for part in parts[1:]: + lower = part.lower() + result["attributes"].append(part) + + if lower.startswith("domain="): + domain = lower.split("=", 1)[1].lstrip(".") + if domain and domain != page_domain: + result["third_party"] = True + + if lower == "samesite=none": + result["cross_site"] = True + + return result diff --git a/altbrow/classify_domain.py b/altbrow/classify_domain.py new file mode 100644 index 0000000..b9d346d --- /dev/null +++ b/altbrow/classify_domain.py @@ -0,0 +1,132 @@ +# altbrow/classify_domain.py +# +# classify_domain() +# classify_ip() + +import logging + +from pathlib import Path +from .domain_utils import get_registrable_domain +from .cache import lookup_domain, lookup_ip + +logger = logging.getLogger(__name__) + + +def _relation(domain: str, page_domain: str) -> str: + """Determine structural relation of domain to the analysed page. + + Rules (TARGET example: www.example.com, registrable: example.com): + + FIRST_PARTY — domain is exactly the TARGET host (www.example.com) + or exactly the registrable root (example.com) + SUBDOMAIN — domain is a strict child of the TARGET host + (m.example.com is child of www.example.com) + PEER — same registrable domain, but neither TARGET nor its child + (ai.example.com, images.example.com — lateral siblings) + EXTERNAL — different registrable domain + + Args: + domain: The domain to classify. + page_domain: The hostname of the analysed page (TARGET host). + + Returns: + 'FIRST_PARTY' | 'SUBDOMAIN' | 'PEER' | 'EXTERNAL' + """ + domain = domain.lower() + page_domain = page_domain.lower() + + reg_domain = get_registrable_domain(domain) + reg_page = get_registrable_domain(page_domain) + + if reg_domain != reg_page: + return "EXTERNAL" + + # same registrable domain — determine exact relation + if domain == page_domain: + return "FIRST_PARTY" + + if domain == reg_page: + return "FIRST_PARTY" + + # strict child: domain ends with "." + page_domain + if domain.endswith("." + page_domain): + return "SUBDOMAIN" + + return "PEER" + + +def classify_domain( + domain: str, + page_domain: str, + cache_path: Path, + config: dict | None = None, +) -> dict: + """Classify an external domain against the provider cache. + + Determines the structural relation to the analysed page and looks up + all matching provider categories from the SQLite cache, including any + live DNS provider results. + + Args: + domain: Fully qualified domain name to classify. + page_domain: Hostname of the analysed page (used for relation). + cache_path: Path to the SQLite cache file. + + Returns: + Dict with keys: + value - normalised domain string + registrable_domain - e.g. 'example.com' + relation - 'FIRST_PARTY' | 'SUBDOMAIN' | 'PEER' | 'EXTERNAL' + categories - list of dicts (category, provider, + provider_location, category_name) + empty list if no provider match + """ + domain = domain.lower() + page_domain = page_domain.lower() + + reg_domain = get_registrable_domain(domain) + relation = _relation(domain, page_domain) + + categories = sorted( + lookup_domain(domain, cache_path, config), + key=lambda c: c.get("tier", 2), + ) + + return { + "value": domain, + "registrable_domain": reg_domain, + "relation": relation, + "categories": categories, + } + + +def classify_ip( + ip_str: str, + cache_path: Path, +) -> dict: + """Classify an IP address against the provider cache. + + Matches exact IPs and CIDR blocks from the SQLite cache. + + Args: + ip_str: IP address string (IPv4 or IPv6). + cache_path: Path to the SQLite cache file. + + Returns: + Dict with keys: + value - original IP string + relation - always 'EXTERNAL' (IPs are never first-party) + categories - list of dicts (category, provider, + provider_location, category_name) + empty list if no match + """ + categories = sorted( + lookup_ip(ip_str, cache_path), + key=lambda c: c.get("tier", 2), + ) + + return { + "value": ip_str, + "relation": "EXTERNAL", + "categories": categories, + } diff --git a/altbrow/config.py b/altbrow/config.py new file mode 100644 index 0000000..00263e6 --- /dev/null +++ b/altbrow/config.py @@ -0,0 +1,1297 @@ +from pathlib import Path +from altbrow import __version__ +from datetime import date +import tomllib + +# config.py +# discover_config_path() +# load_toml() +# get_client_profile() +# validate_altbrow_config() +# summarize_provider +# validate_provider_config() + +VALID_PROFILES = {"passive", "browser", "consented"} +ALLOWED_LOCATIONS = {"local", "inline", "remote", "dns"} +ALLOWED_TYPES = {"ip", "domain"} +ALLOWED_MAPPINGS = { + "ads", + "analytics", + "cdn", + "malware", + "social", + "suspicious", + "telemetry", + "tracking", + "local", + "infrastructure", + "unknown", + "geoip", +} + +# Default tier per provider location — lower tier wins (first match in DB on tie) +# Configuration for provider.name.category overwrites. +# If no tier is configured at category level, this browser location mapping is used +LOCATION_DEFAULT_TIER = { + "inline": 1, + "local": 1, + "dns": 2, + "remote": 2, +} + +# Default [resolve] section values for provider.toml +RESOLVE_DEFAULTS: dict = { + "resolve-domains": False, + "resolver": ["os"], + "resolver-timeout": 2, +} + +class ConfigError(Exception): + pass + +def discover_config_path(cli_path: str | None = None) -> Path: + """Discover the Altbrow configuration file. + + If no configuration file is found, generates defaults in ~/.altbrow/. + + Args: + cli_path: CLI given PATH or None. + + Returns: + Path to the active altbrow.toml. + + Raises: + ConfigError: If the CLI path/file is not found. + + Search order: + 1. --config PATH (CLI, explicit) + 2. ./altbrow.toml (portable / development) + 3. ~/.altbrow/altbrow.toml (user, existing or generated) + """ + # 1. explicit CLI path + if cli_path: + path = Path(cli_path).expanduser().resolve() + if not path.exists(): + raise ConfigError(f"Config not found: {path}") + return path + + # 2. portable mode — config in current working directory + local = Path("altbrow.toml") + if local.exists(): + return local + + # 3. user mode — ~/.altbrow/ + user_dir = Path.home() / ".altbrow" + user_cfg = user_dir / "altbrow.toml" + + if not user_cfg.exists(): + # first run — generate defaults in ~/.altbrow/ + user_dir.mkdir(parents=True, exist_ok=True) + user_cfg.write_text(default_config_altbrow(), encoding="utf-8") + print(f"Created default config: {user_cfg}") + + user_provider = user_dir / "provider.toml" + if not user_provider.exists(): + user_provider.write_text(default_config_provider(), encoding="utf-8") + print(f"Created default config: {user_provider}") + + return user_cfg + +def load_toml(path: Path) -> dict: + """Load and parse a TOML file. + + Args: + path: Absolute or relative path to the TOML file. + + Returns: + Parsed TOML content as a nested dictionary. + + Raises: + ConfigError: If the file cannot be read or is not valid TOML. + """ + try: + with path.open("rb") as f: + return tomllib.load(f) + except Exception as exc: + raise ConfigError(f"Failed to load config {path}: {exc}") from exc + +def get_client_profile(config: dict, override: str | None) -> dict: + """Resolve the active HTTP client profile from config. + + Merges `[client.defaults]` with the selected profile's overrides. + The profile is taken from *override* first, then from `client.profile` + in the config, falling back to `"passive"`. + + Args: + config: Parsed altbrow config dictionary. + override: Optional profile name supplied via CLI (`--client-profile`). + + Returns: + Merged dictionary of client settings for the active profile. + + Raises: + ConfigError: If no profiles are defined, the profile name is invalid, + or the named profile does not exist in `[client.profiles]`. + """ + client_cfg = config.get("client", {}) + defaults = client_cfg.get("defaults", {}) + profiles = client_cfg.get("profiles", {}) + + + if not profiles: + raise ConfigError("No client profiles defined in [client.profiles]") + + profile_name = override or client_cfg.get("profile", "passive") + + if profile_name not in VALID_PROFILES: + raise ConfigError( + f"Invalid client profile '{profile_name}'. " + f"Must be one of: {', '.join(sorted(VALID_PROFILES))}" + ) + if profile_name not in profiles: + raise ConfigError( + f"Client profile '{profile_name}' is not defined in [client.profiles]. " + f"Available: {', '.join(profiles) or 'none'}" + ) + + merged = {**defaults, **profiles[profile_name]} + + # inject [client.headers] if use_header is set in the merged profile + if merged.get("use_header", False): + global_headers = client_cfg.get("headers", {}) + if global_headers: + merged["headers"] = global_headers + + return merged + +def default_config_altbrow() -> str: + return f""" +# default ./altbrow.toml config file + +# you may use privat permanent config by moving to: +# ~/.altbrow/altbrow.toml and ~/.altbrow/provider.toml +# if path exsits a Provider cache file will be created there, too + +[meta] +version = 1 +created = "{date.today()}" +use-provider = false + +[validation.schema_org] +allow_unknown_properties = false + +[validation] +microdata_vs_jsonld.tolerance = "loose" + +[output] +explicit_format = "json" + +[client] +profile = "passive" + +# ********************** +# * profile definition * +# ********************** + +[client.defaults] +follow_redirects = true # follow HTTP 301/302 +timeout = 10 # seconds +fetch_subresources = 0 # raw URL, no external javascript, css +use_session = false +use_header = false +accept_cookies = false +check_cert = true + +[client.profiles.passive] +# inherits all defaults — no overrides + +[client.profiles.browser] +use_session = true +use_header = true + +[client.profiles.consented] +use_session = true +use_header = true +accept_cookies = true +fetch_subresources = 1 + + +# Header definition needs periodic update to appear as a normal browser. +# Applied to all profiles where use_header = true. + +[client.headers] +"User-Agent" = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36" +"Accept-Language" = "de-DE,de;q=0.9,en;q=0.8" +# "referer" = "Referer: https://www.google.com/" + +""" + +def default_config_provider() -> str: + return f""" +# provider.toml is only used when in "altbrow.toml" is set: `meta.use-provider = true` + +# 1st, given, cli: --config /etc/altbrow.toml -> /etc/provider.toml +# 2nd, if exits, user: ~/.altbrow/altbrow.toml -> ~/.altbrow/provider.toml +# 3rd, default, portable: ./altbrow.toml -> ./provider.toml + +# --- 8< --- +# [provider.name] +# name = "human readable label" # optional +# location = "[local|inline|remote|dns]" +# type = "[ip|domain|geoip]" +# enabled = [true|false] +# subdomain_match = [true|false] +# +# # Every enabled provider needs at least one enabled category: +# +# [[provider.name.category]] +# name = "human readable label" # optional +# enabled = [true|false] +# tier = # optional, default: inline/local=1, dns/remote=2 (0 reserved) +# mapping = [""] # one or more from list below +# sinkhole = ["", ...] # dns only: block page IPs for this category +# +# source = ["./file.txt"] # local: file path(s) relative to provider.toml +# source = ["example.com"] # inline domain: domain list +# source = ["192.168.1.0/24"] # inline ip: ip or cidr list +# source = ["example.com", "iana.org"] # inline domain: domain list +# source = ["1.1.1.0/24", "8.8.8.8"] # inline ip: ip or cidr list +# source = ["https://example.com/list.txt"] # remote: URL(s) +# source = ["", ...] # dns: resolver IP(s) per category — allows multiple resolver categories +# --- 8< --- + +# altbrow internal 8 categories: +# +# ads - advertising networks and ad delivery +# analytics - user behaviour measurement and reporting +# cdn - content delivery networks and static asset hosting +# malware - malware, phishing, known hostile domains +# social - social networks, dating, gambling, adult content +# suspicious - unverified or potentially hostile +# telemetry - error reporting, performance monitoring, device telemetry +# tracking - cross-site user tracking and profiling + +# altbrow special 4 categories: +# +# local - RFC1918, localhost, loopback, your domains +# infrastructure - technical and semantic web standards, DNS resolvers +# unknown - no category match +# geoip - used for location service, not a regular category + +# automatic categories (derived from structure, no provider needed): +# +# FIRST_PARTY - same registrable domain (example.com) as the analysed page (e.g. www.example.com) +# PEER - siblings like images.example.com +# SUBDOMAIN - subdomain of the analysed page domain, e.g. us.www.example.com +# SELF_REF - domain appears only in JSON-LD @id / Microdata, not in HTML traffic +# EXTERNAL - external domain + +# --------------------------------------------------------------------------- +# Resolve Configuration +# Controls domain-to-IP resolution and DNS resolver settings. +# If this section is absent, RESOLVE_DEFAULTS apply. +# --------------------------------------------------------------------------- + +[resolve] +resolve-domains = false # resolve domains to IP and check against IP provider lists +resolver = ["os"] # DNS resolver: "os" = system, or IP e.g. ["1.1.1.1","8.8.8.8"] +resolver-timeout = 2 # seconds per DNS query + +# --------------------------------------------------------------------------- +# DNS Resolve Filter +# Controls which provider categories trigger a live DNS query. +# With empty section: all enabled dns provider categories are queried. +# filter-mode = "or" -> category match OR tier <= max-tier +# filter-mode = "and" -> category match AND tier <= max-tier +# Disable all DNS queries: set enabled-categories = [] with filter-mode = "and" +# or simply disable all DNS providers in provider.toml +# --------------------------------------------------------------------------- + +[dns-resolve-filter] +enabled-categories = ["malware", "suspicious"] +max-tier = 1 +filter-mode = "and" + +[meta] +version = 1 +created = "{date.today()}" + +# --------------------------------------------------------------------------- +# Local Provider +# --------------------------------------------------------------------------- + +[provider.fail2ban] +location = "local" +type = "ip" +enabled = false + +[[provider.fail2ban.category]] +name = "fail2ban SSH bans" +mapping = ["suspicious"] +source = ["./fail2ban.txt"] + +# --------------------------------------------------------------------------- +# Inline Domain Providers +# --------------------------------------------------------------------------- + +[provider.infrastructure] +location = "inline" +type = "domain" +enabled = true + +[[provider.infrastructure.category]] +name = "Semantic Web Standards" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "schema.org", + "schema.googleapis.com", + "w3.org", + "w3c.org", + "purl.org", + "xmlns.com", + "rdf.data-vocabulary.org", + "ogp.me", + "dublincore.org", + "json-ld.org", +] + +[[provider.infrastructure.category]] +name = "Web Standards Bodies" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "iana.org", + "mozilla.org", + "whatwg.org", +] + +[provider.cdn] +location = "inline" +type = "domain" +enabled = true + +[[provider.cdn.category]] +name = "Example Major CDN" +enabled = true +mapping = ["cdn"] +source = [ + "cdnjs.cloudflare.com", + "cdn.cloudflare.com", + "akamai.net", + "akamaiedge.net", + "akamaized.net", + "edgesuite.net", + "fastly.net", + "fastlylb.net", + "cloudfront.net", + "amazonaws.com", + "gstatic.com", + "azureedge.net", + "msecnd.net", + "jsdelivr.net", + "unpkg.com", + "bootstrapcdn.com", + "stackpathcdn.com", + "b-cdn.net", + "kxcdn.com", +] + +[provider.analytics] +location = "inline" +type = "domain" +enabled = true + +[[provider.analytics.category]] +name = "Example Web Analytics" +enabled = true +mapping = ["analytics"] +source = [ + "google-analytics.com", + "googletagmanager.com", + "googleadservices.com", + "cloudflareinsights.com", + "matomo.org", + "plausible.io", + "fathom.com", + "segment.com", + "mixpanel.com", + "amplitude.com", + "hotjar.com", + "clarity.ms", + "fullstory.com", + "logrocket.com", +] + +[[provider.analytics.category]] +name = "Example Error and Performance Monitoring" +enabled = true +mapping = ["telemetry"] +source = [ + "sentry.io", + "bugsnag.com", + "rollbar.com", + "newrelic.com", + "datadoghq.com", + "elastic.co", + "dynatrace.com", + "appdynamics.com", +] + +[provider.tracking] +location = "inline" +type = "domain" +enabled = true + +[[provider.tracking.category]] +name = "Example Social Tracking Pixels" +enabled = true +mapping = ["tracking"] +source = [ + "facebook.net", + "connect.facebook.net", + "analytics.twitter.com", + "t.co", + "snapchat.com", + "sc-static.net", + "ads.pinterest.com", + "licdn.com", +] + +[[provider.tracking.category]] +name = "Example Ad Network Tracking" +enabled = true +mapping = ["tracking"] +source = [ + "bat.bing.com", + "taboola.com", + "outbrain.com", + "criteo.com", + "adroll.com", + "quantserve.com", + "scorecardresearch.com", + "bluekai.com", + "zemanta.com", + "doubleclick.net", +] + + +[provider.ads] +location = "inline" +type = "domain" +enabled = true + +[[provider.ads.category]] +name = "Example Ad Delivery" +enabled = true +mapping = ["ads"] +source = [ + "googlesyndication.com", + "googleadservices.com", + "doubleclick.net", + "amazon-adsystem.com", + "media.net", + "moatads.com", + "adsrvr.org", + "advertising.com", + "adnxs.com", + "rubiconproject.com", + "pubmatic.com", + "openx.net", + "smartadserver.com", +] + +# --------------------------------------------------------------------------- +# Inline IP Provider +# --------------------------------------------------------------------------- + +[provider.inlineip] +location = "inline" +type = "ip" +enabled = true + +[[provider.inlineip.category]] +name = "RFC1918 Private" +enabled = true +tier = 0 +mapping = ["local"] +source = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", +] + +[[provider.inlineip.category]] +name = "Loopback" +enabled = true +tier = 0 +mapping = ["local"] +source = [ + "127.0.0.0/8", + "::1/128", +] + +[[provider.inlineip.category]] +name = "Link-Local and Multicast" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "169.254.0.0/16", + "224.0.0.0/4", + "255.255.255.255/32", + "fe80::/10", + "ff00::/8", +] + +[[provider.inlineip.category]] +name = "Carrier-Grade NAT" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "100.64.0.0/10", +] + +[[provider.inlineip.category]] +name = "Example suspicious IPs" +enabled = true +mapping = ["suspicious"] +source = [ + "2.57.122.210", + "46.101.74.113", + "81.192.46.45", + "92.118.39.56", + "92.118.39.72", + "92.118.39.76", + "102.88.137.80", + "118.193.36.205", + "162.223.91.130", + "193.32.162.151", + "197.5.145.102", +] + +# --------------------------------------------------------------------------- +# Remote Provider +# --------------------------------------------------------------------------- + +[provider.ipfire] +name = "IPFire" +location = "remote" +type = "domain" +enabled = false + +[[provider.ipfire.category]] +name = "Advertising" +enabled = false +tier = 3 +mapping = ["ads"] +source = ["https://dbl.ipfire.org/lists/ads/domains.txt"] + +[[provider.ipfire.category]] +name = "Dating" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/dating/domains.txt"] + +[[provider.ipfire.category]] +name = "DNS-over-HTTPS" +enabled = false +mapping = ["telemetry"] +source = ["https://dbl.ipfire.org/lists/doh/domains.txt"] + +[[provider.ipfire.category]] +name = "Gambling" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/gambling/domains.txt"] + +[[provider.ipfire.category]] +name = "Malware" +tier = 1 +enabled = false +mapping = ["malware"] +source = ["https://dbl.ipfire.org/lists/malware/domains.txt"] + +[[provider.ipfire.category]] +name = "Phishing" +tier = 1 +enabled = false +mapping = ["malware"] +source = ["https://dbl.ipfire.org/lists/phishing/domains.txt"] + +[[provider.ipfire.category]] +name = "Piracy" +enabled = false +mapping = ["suspicious"] +source = ["https://dbl.ipfire.org/lists/piracy/domains.txt"] + +[[provider.ipfire.category]] +name = "Pornography" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/porn/domains.txt"] + +[[provider.ipfire.category]] +name = "Smart TV Telemetry" +enabled = false +mapping = ["telemetry"] +source = ["https://dbl.ipfire.org/lists/smart-tv/domains.txt"] + +[[provider.ipfire.category]] +name = "Social Networks" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/social/domains.txt"] + +[[provider.ipfire.category]] +name = "Violence" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/violence/domains.txt"] + +# -------------------- + +# same list is used by pi hole + +[provider.stevenblack] +location = "remote" +type = "domain" +enabled = true + +[[provider.stevenblack.category]] +name = "PIHole" +mapping = ["ads"] +source = ["https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"] + + +# --------------------------------------------------------------------------- +# GeoIP Provider (MaxMind GeoLite2) +# Download: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data +# --------------------------------------------------------------------------- + +# Local example — files next to altbrow.toml, glob resolves newest version: +[provider.maxmind] +location = "local" +type = "ip" +enabled = false + +[[provider.maxmind.category]] +name = "Country" +mapping = ["geoip"] +source = ["./GeoLite2-Country_*.tar.gz"] + +[[provider.maxmind.category]] +name = "ASN" +mapping = ["geoip"] +source = ["./GeoLite2-ASN_*.tar.gz"] + +[[provider.maxmind.category]] +name = "City" +mapping = ["geoip"] +source = ["./GeoLite2-City_*.tar.gz"] + +# Remote example — shared server in local network: +# [provider.maxmind-net] +# location = "remote" +# type = "ip" +# enabled = false +# [[provider.maxmind-net.category]] +# name = "City" +# mapping = ["geoip"] +# source = ["http://192.168.1.1:8080/GeoLite2-City.tar.gz"] + +# --------------------------------------------------------------------------- +# DNS Provider +# sinkhole per category, source = resolver IPs per category +# --------------------------------------------------------------------------- + +[provider.opendns] +name = "OpenDNS" +location = "dns" +type = "domain" +enabled = true + +[[provider.opendns.category]] +name = "Malware/Phishing" +mapping = ["malware"] +source = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"] +sinkhole = [ + "146.112.61.104", "146.112.61.105", "146.112.61.107", "146.112.61.108", + "::ffff:146.112.61.104", "::ffff:146.112.61.105", + "::ffff:146.112.61.107", "::ffff:146.112.61.108", +] + +[[provider.opendns.category]] +name = "Content/Adult (FamilyShield)" +enabled = false +mapping = ["social"] +source = ["208.67.222.123", "208.67.220.123"] +sinkhole = ["146.112.61.106", "::ffff:146.112.61.106"] + +[[provider.opendns.category]] +name = "Suspicious/DNS Tunneling" +enabled = false +mapping = ["suspicious"] +source = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"] +sinkhole = ["146.112.61.110", "::ffff:146.112.61.110"] + +# -------------------- + +# uses the list of stevenblack + +[provider.pihole] +location = "dns" +type = "domain" +enabled = false + +[[provider.pihole.category]] +name = "PiHole local" +mapping = ["ads"] +source = ["192.168.1.1"] +sinkhole = ["0.0.0.0", "::", "::ffff:0.0.0.0"] + +# --------------------------------------------------------------------------- +# Tier 0 Provider +# defaults, normally you do not need to change, only activate your hosts OS +# --------------------------------------------------------------------------- + +[provider.system-hosts] +name = "hosts" +location = "local" +type = "domain" +enabled = false + +[[provider.system-hosts.category]] +name = "linux" +tier = 0 +enabled = false +mapping = ["local"] +source = ["/etc/hosts"] + +[[provider.system-hosts.category]] +name = "windows" +tier = 0 +enabled = true +mapping = ["local"] +source = ["C:\Windows\System32\drivers\etc\hosts"] + +# -------------------- + +[provider.definedip] +location = "inline" +type = "ip" +enabled = true + +[[provider.definedip.category]] +name = "RFC1918 Private" +enabled = true +tier = 0 +mapping = ["local"] +source = [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", +] + +[[provider.definedip.category]] +name = "Loopback" +enabled = true +tier = 0 +mapping = ["local"] +source = [ + "127.0.0.0/8", + "::1/128", +] + +[[provider.definedip.category]] +name = "Link-Local" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "169.254.0.0/16", + "fe80::/10", +] + +[[provider.definedip.category]] +name = "Multicast" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "224.0.0.0/4", + "ff00::/8", +] + +[[provider.definedip.category]] +name = "Broadcast" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "255.255.255.255/32", +] + +[[provider.definedip.category]] +name = "Carrier-Grade NAT" +enabled = true +tier = 0 +mapping = ["infrastructure"] +source = [ + "100.64.0.0/10", +] + + +""" + +def _strip_provider_sources(provider_cfg: dict) -> dict: + """Return a copy of provider config with non-essential source lists removed. + + inline, local, and remote sources are only needed during cache build. + DNS sources (resolver IPs) and sinkholes are kept — needed for live DNS lookup. + The stripped version is merged into the main config dict as config["provider"]. + + Args: + provider_cfg: Full parsed provider.toml dictionary. + + Returns: + Provider dict with non-dns sources removed from each category. + """ + import copy + stripped = copy.deepcopy(provider_cfg) + for p in stripped.get("provider", {}).values(): + location = p.get("location") + for cat in p.get("category", []): + if location != "dns": + cat.pop("source", None) # inline/local/remote: in DB, not needed at runtime + # dns: keep source (resolver IPs) and sinkhole for live lookup + + # keep dns-resolve-filter and resolve at top level for runtime use + for key in ("dns-resolve-filter", "resolve"): + if key in provider_cfg: + stripped[key] = copy.deepcopy(provider_cfg[key]) + + return stripped + + +def load_provider_config(main_config_path: Path, config: dict) -> dict | None: + """Load provider.toml, merge stripped version into config, return full config for cache. + + Reads `meta.use-provider` from config. If false, sets config["provider"] = False + and returns None. If true, loads provider.toml, validates it, merges a source-stripped + version into config["provider"], and returns the full provider config for build_cache(). + + Args: + main_config_path: Path to the active altbrow.toml file. + config: Parsed altbrow config dictionary — modified in place with ["provider"] key. + + Returns: + Full parsed provider config dict (with sources) for build_cache(), or None if disabled. + + Raises: + ConfigError: If use-provider is true but provider.toml does not exist or is invalid. + """ + meta = config.get("meta", {}) + use_provider = meta.get("use-provider", False) + + if not use_provider: + config["provider"] = False + return None + + provider_path = main_config_path.parent / "provider.toml" + + if not provider_path.exists(): + raise ConfigError(f"Provider config not found: {provider_path}") + + provider_cfg = load_toml(provider_path) + validate_provider_config(provider_cfg) + + # merge provider and dns-resolve-filter into main config + # do NOT merge meta — provider.toml meta would overwrite altbrow.toml meta + stripped = _strip_provider_sources(provider_cfg) + MERGE_KEYS = {"provider", "dns-resolve-filter", "resolve"} + for key, value in stripped.items(): + if key in MERGE_KEYS: + config[key] = value + + return provider_cfg + +def validate_altbrow_config(config: dict) -> str: + """Validate an altbrow config dictionary and return a human-readable summary. + + Checks all required sections (`[meta]`, `[client]`, `[client.profiles]`) and + optional sections (`[validation]`, `[output]`). Provider summary is read from + config["provider"] which is set by load_provider_config(). + + Args: + config: Merged altbrow config dict — must have been processed by + load_provider_config() so config["provider"] is set. + + Returns: + Multi-line string summarizing the active configuration. + + Raises: + ConfigError: On any missing or invalid field. + """ + # --- meta --- + if "meta" not in config: + raise ConfigError("Missing [meta] section") + meta = config["meta"] + + # --- version --- + if "version" not in meta: + raise ConfigError("Missing meta.version") + + config_version = meta["version"] + config_date = meta.get("created", "unknown") + + # --- Provider --- + use_provider = meta.get("use-provider", False) + if not isinstance(use_provider, bool): + raise ConfigError("meta.use-provider must be boolean") + + provider_data = config.get("provider", False) + if use_provider and provider_data: + dns_rf = config.get("dns-resolve-filter", {}) + resolve_cfg = config.get("resolve", {}) + provider_text = summarize_provider({"provider": provider_data, "dns-resolve-filter": dns_rf, "resolve": resolve_cfg}) + elif use_provider and not provider_data: + provider_text = "Provider configuration enabled but not loaded." + else: + provider_text = "Provider configuration disabled." + + # --- client --- + if "client" not in config: + raise ConfigError("Missing [client] section") + + client = config["client"] + + if "profile" not in client: + raise ConfigError("Missing client.profile") + + if "profiles" not in client: + raise ConfigError("Missing [client.profiles] section") + + default_profile = client["profile"] + profiles = client["profiles"] + + if default_profile not in profiles: + raise ConfigError(f"Default profile '{default_profile}' not found in client.profiles") + + # --- validation (optional) --- + validation = config.get("validation", {}) + + if "microdata_vs_jsonld" in validation: + tolerance = validation["microdata_vs_jsonld"].get("tolerance") + if tolerance not in (None, "strict", "loose"): + raise ConfigError("validation.microdata_vs_jsonld.tolerance must be 'strict' or 'loose'") + + # --- output --- + output = config.get("output", {}) + explicit_format = output.get("explicit_format", "json") + + if explicit_format not in ("json", "yaml", "text"): + raise ConfigError("output.explicit_format must be 'json', 'yaml' or 'text'") + + if explicit_format == "yaml": + output_text = "YAML" + elif explicit_format == "json": + output_text = "JSON" + else: + output_text = "text" + + # --- profile description --- + profile = profiles[default_profile] + use_session = profile.get("use_session", False) + headers = profile.get("headers", {}) + + activity = "active" if use_session else "passive" + consented = "with consented headers" if headers else "without consent headers" + + + + # --- description sentence --- + lines = [ + f"Altbrow Version v{__version__} reads with config Version {config_version} from {config_date}.", + f"It operates {activity} {consented} and counts domains, cookies, html, jsonld and microdata.", + f"Default structured output format is {output_text}.", + f"It {'does' if 'microdata_vs_jsonld' in validation else 'does not'} analyse microdata vs jsonld comparison for the summary.", + provider_text, + "Output may be written to STDOUT or to a file depending on CLI options." + ] + + return "\n".join(lines) + +def summarize_provider(provider_cfg: dict) -> str: + """Summarize active providers and categories grouped by location/tier. + + Output format: + Provider config: 8 provider (5 inline, 1 local, 1 dns, 1 remote) + of total 11 provider active with 15 of total 18 categories enabled + + Location order follows default tier: inline/local (tier 1) before dns/remote (tier 2). + + Args: + provider_cfg: Parsed provider.toml dictionary. + + Returns: + Multi-line summary string for --validate-config output. + """ + providers = provider_cfg.get("provider", {}) + + pcount_sum = len(providers) + ccount_sum = 0 + ccount = 0 + + # count enabled providers per location (tier order: inline, local, dns, remote) + location_order = ["inline", "local", "dns", "remote"] + loc_counts: dict[str, int] = {loc: 0 for loc in location_order} + + for p in providers.values(): + categories = p.get("category", []) + ccount_sum += len(categories) + ccount += sum(1 for c in categories if c.get("enabled", True)) + + if p.get("enabled", False): + loc = p.get("location", "remote") + loc_counts[loc] = loc_counts.get(loc, 0) + 1 + + pcount = sum(loc_counts.values()) + + # build location breakdown string — only non-zero locations + loc_parts = [ + f"{v} {k}" for k, v in loc_counts.items() if v > 0 + ] + loc_str = f" ({', '.join(loc_parts)})" if loc_parts else "" + + # count non-dns categories validated by dns-resolve-filter + # without filter: all enabled non-dns categories are validated + # with filter: only those matching tier/category criteria + dns_filter = provider_cfg.get("dns-resolve-filter", {}) + from .dns_lookup import _should_query_category + validated = 0 + for p in providers.values(): + if p.get("location") == "dns": + continue + if not p.get("enabled", False): + continue + for cat in p.get("category", []): + if not cat.get("enabled", True): + continue + if _should_query_category(cat, dns_filter): + validated += 1 + + if loc_counts.get("dns", 0) > 0: + if dns_filter: + dns_str = f"\n {validated} of {ccount} categories are validated additional by dns-resolve-filter" + else: + dns_str = f"\n {ccount} categories are validated by dns-resolve-filter" + else: + dns_str = "" + + # geoip provider info + geo_names = [ + cat.get("name") for p in providers.values() + if isinstance(p, dict) and p.get("enabled") and p.get("location") not in ("dns",) + for cat in p.get("category", []) + if cat.get("enabled", True) and "geoip" in cat.get("mapping", []) and cat.get("name") + ] + if geo_names: + geo_str = f"\n geoIP classification activated for {", ".join(geo_names)}" + else: + geo_str = "\n no geoIP provider activated" + + # domain resolve status + resolve = provider_cfg.get("resolve", {}) + resolve_enabled = resolve.get("resolve-domains", RESOLVE_DEFAULTS["resolve-domains"]) + resolve_str = ", domain lookup enabled" if resolve_enabled else ", domain lookup disabled" + + return ( + f"Provider config: {pcount}{loc_str} of total {pcount_sum} provider active " + f"with {ccount} of total {ccount_sum} categories enabled." + f"{dns_str}" + f"{geo_str}{resolve_str}" + ) + +def _as_list(value): + if isinstance(value, list): + return value + return [value] + +def validate_provider_config(cfg: dict) -> None: + + # --- meta --- + if "meta" not in cfg: + raise ConfigError("Missing [meta] section") + + meta = cfg["meta"] + + if "version" not in meta: + raise ConfigError("Missing meta.version") + + if "created" not in meta: + raise ConfigError("Missing meta.created") + + # --- resolve (optional, defaults from RESOLVE_DEFAULTS) --- + resolve = cfg.get("resolve", {}) + if resolve: + rd = resolve.get("resolve-domains") + if rd is not None and not isinstance(rd, bool): + raise ConfigError("resolve.resolve-domains must be boolean") + resolver = resolve.get("resolver") + if resolver is not None: + if not isinstance(resolver, list) or not resolver: + raise ConfigError("resolve.resolver must be a non-empty list") + for r in resolver: + if not isinstance(r, str): + raise ConfigError("resolve.resolver entries must be strings") + if r != "os" and not any(c.isdigit() for c in r): + raise ConfigError(f"resolve.resolver invalid entry '{r}' (use \"os\" or IP address)") + timeout = resolve.get("resolver-timeout") + if timeout is not None: + if not isinstance(timeout, int) or timeout <= 0: + raise ConfigError("resolve.resolver-timeout must be a positive integer (seconds)") + + # --- dns-resolve-filter (optional) --- + dns_filter = cfg.get("dns-resolve-filter", {}) + if dns_filter: + cats = dns_filter.get("enabled-categories") + if cats is not None: + if not isinstance(cats, list): + raise ConfigError("dns-resolve-filter.enabled-categories must be a list") + for c in cats: + if c not in ALLOWED_MAPPINGS: + raise ConfigError(f"dns-resolve-filter.enabled-categories invalid: '{c}'") + max_tier = dns_filter.get("max-tier") + if max_tier is not None: + if not isinstance(max_tier, int) or max_tier < 0: + raise ConfigError("dns-resolve-filter.max-tier must be a non-negative integer") + filter_mode = dns_filter.get("filter-mode") + if filter_mode is not None and filter_mode not in ("or", "and"): + raise ConfigError("dns-resolve-filter.filter-mode must be \"or\" or \"and\"") + + # --- provider section --- + providers = cfg.get("provider", {}) + + if not isinstance(providers, dict): + raise ConfigError("[provider] must be a table of named providers") + + for pname, p in providers.items(): + + # --- required provider fields --- + for field in ("location", "type", "enabled"): + if field not in p: + raise ConfigError(f"Provider '{pname}' missing '{field}'") + + location = p["location"] + ptype = p["type"] + + if location not in ALLOWED_LOCATIONS: + raise ConfigError(f"Provider '{pname}' invalid location '{location}'") + + if ptype not in ALLOWED_TYPES: + raise ConfigError(f"Provider '{pname}' invalid type '{ptype}'") + + # --- optional subdomain_match --- + if "subdomain_match" in p: + if not isinstance(p["subdomain_match"], bool): + raise ConfigError(f"Provider '{pname}' subdomain_match must be boolean") + + # --- dns: sinkhole per category, resolver IPs in category source --- + + # --- categories (all provider types) --- + categories = p.get("category") + + if not isinstance(categories, list) or len(categories) == 0: + raise ConfigError(f"Provider '{pname}' must define at least one category") + + enabled_category_count = 0 + + for i, c in enumerate(categories): + + cname = f"{pname}.category[{i}]" + + # --- enabled default --- + enabled = c.get("enabled", True) + + if enabled: + enabled_category_count += 1 + + # --- tier (optional, defaults to LOCATION_DEFAULT_TIER[location]) --- + if "tier" in c: + tier = c["tier"] + if not isinstance(tier, int) or tier < 0: + raise ConfigError(f"{cname} tier must be a non-negative integer >= 1 (0 is reserved for altbrow internals)") + + # --- mapping --- + if "mapping" not in c: + raise ConfigError(f"{cname} missing 'mapping'") + + mapping = _as_list(c["mapping"]) + + for m in mapping: + if m not in ALLOWED_MAPPINGS: + raise ConfigError(f"{cname} invalid mapping '{m}'") + + # --- sinkhole: required per dns category --- + if location == "dns": + if "sinkhole" not in c: + raise ConfigError(f"{cname} missing 'sinkhole' (required for dns categories)") + sinkholes = c["sinkhole"] + if not isinstance(sinkholes, list) or len(sinkholes) == 0: + raise ConfigError(f"{cname} sinkhole must be a non-empty list") + for s in sinkholes: + if not isinstance(s, str): + raise ConfigError(f"{cname} sinkhole entries must be strings") + + # --- source: resolver IPs for dns, file/url/domain for others --- + if "source" not in c: + raise ConfigError(f"{cname} missing 'source'") + + sources = c["source"] + + if not isinstance(sources, list) or len(sources) == 0: + raise ConfigError(f"{cname}.source must be a non-empty list") + + if location == "dns": + # source = resolver IP addresses + for src in sources: + if not isinstance(src, str): + raise ConfigError(f"{cname} dns source entries must be strings (resolver IPs)") + continue + + is_geoip = "geoip" in mapping + + for src in sources: + + if not isinstance(src, str): + raise ConfigError(f"{cname}.source entries must be strings") + + if location == "remote": + if not (src.startswith("http://") or src.startswith("https://")): + raise ConfigError(f"{cname} remote source must be URL") + if is_geoip and not src.endswith(".tar.gz"): + raise ConfigError(f"{cname} geoip remote source must be a .tar.gz URL") + + elif location == "local": + if src.startswith("http://") or src.startswith("https://"): + raise ConfigError(f"{cname} local source must be file path or glob") + if is_geoip: + # glob patterns allowed for geoip local sources + if not (src.endswith(".tar.gz") or src.endswith(".mmdb") or "*" in src or "?" in src): + raise ConfigError(f"{cname} geoip local source must be .tar.gz, .mmdb or glob pattern") + + elif location == "inline": + if ptype == "domain": + if "." not in src: + raise ConfigError(f"{cname} invalid domain '{src}'") + + elif ptype == "ip": + if not any(ch.isdigit() for ch in src): + raise ConfigError(f"{cname} invalid ip/cidr '{src}'") + + if p["enabled"] and enabled_category_count == 0: + raise ConfigError(f"Provider '{pname}' enabled but no category is enabled") diff --git a/altbrow/dns_lookup.py b/altbrow/dns_lookup.py new file mode 100644 index 0000000..6b8051f --- /dev/null +++ b/altbrow/dns_lookup.py @@ -0,0 +1,278 @@ +# altbrow/dns_lookup.py +# +# dns_provider_lookup() +# _should_query_category() + +import ipaddress +import logging +import socket + +from concurrent.futures import ThreadPoolExecutor, as_completed + +from .config import RESOLVE_DEFAULTS + +logger = logging.getLogger(__name__) + + +def _should_query_category( + cat: dict, + dns_filter: dict, +) -> bool: + """Determine if a cache result category should trigger a live DNS re-query. + + Used in cache.py lookup_domain() to decide whether a static cache hit + warrants an additional live DNS verification pass. + NOT used to gate which DNS providers are queried — all enabled DNS + providers are always queried unconditionally. + + Args: + cat: Dict with "mapping" (list[str]) and "tier" (int) keys. + dns_filter: Parsed [dns-resolve-filter] section, or empty dict if not configured. + + Returns: + True if this category should trigger a live DNS re-query. + """ + if not dns_filter: + return True # no filter → query all enabled dns categories + + enabled_cats = dns_filter.get("enabled-categories") + max_tier = dns_filter.get("max-tier") + filter_mode = dns_filter.get("filter-mode", "or") + + cat_mappings = cat.get("mapping", []) + cat_tier = cat.get("tier", 2) + + cat_match = bool(enabled_cats) and any(m in enabled_cats for m in cat_mappings) + tier_match = max_tier is not None and cat_tier <= max_tier + + # if neither filter is configured → query all + if not enabled_cats and max_tier is None: + return True + + if filter_mode == "and": + if enabled_cats and max_tier is not None: + return cat_match and tier_match + if enabled_cats: + return cat_match + return tier_match + + # "or" mode + checks = [] + if enabled_cats: + checks.append(cat_match) + if max_tier is not None: + checks.append(tier_match) + return any(checks) + + +def _query_resolver( + domain: str, + resolver_ip: str, + timeout_s: float, +) -> list[str]: + """Query a single DNS resolver for A/AAAA records of domain. + + Uses getaddrinfo with explicit nameserver via socket — works without + dnspython by temporarily patching is not possible, so we use a raw + UDP DNS query via socket for non-os resolvers. + + For "os" resolver: uses socket.getaddrinfo directly. + For IP resolvers: uses dnspython if available, otherwise skips with warning. + + Args: + domain: Domain to resolve. + resolver_ip: IP address string or "os". + timeout_s: Timeout in seconds. + + Returns: + List of resolved IP address strings. Empty on failure or timeout. + """ + if resolver_ip == "os": + try: + results = socket.getaddrinfo(domain, None, proto=socket.IPPROTO_TCP) + return list({r[4][0] for r in results}) + except Exception as exc: + logger.debug("OS resolver failed for %s: %s", domain, exc) + return [] + + # explicit IP resolver — requires dnspython + try: + import dns.resolver + import dns.exception + + res = dns.resolver.Resolver(configure=False) + res.nameservers = [resolver_ip] + res.timeout = timeout_s + res.lifetime = timeout_s + + ips = [] + for qtype in ("A", "AAAA"): + try: + answers = res.resolve(domain, qtype) + ips.extend(str(r) for r in answers) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, + dns.resolver.NoNameservers, dns.exception.Timeout): + pass + return ips + + except ImportError: + logger.warning( + "dnspython not installed — cannot query resolver %s. " + "Install with: pip install dnspython", + resolver_ip, + ) + return [] + except Exception as exc: + logger.debug("Resolver %s failed for %s: %s", resolver_ip, domain, exc) + return [] + + +def _check_sinkhole(resolved_ips: list[str], sinkhole: list[str]) -> bool: + """Return True if any resolved IP matches a sinkhole entry (exact or IPv4-mapped). + + Args: + resolved_ips: List of IP strings returned by DNS resolver. + sinkhole: List of sinkhole IP strings from provider category config. + + Returns: + True if at least one resolved IP is in the sinkhole list. + """ + sinkhole_set = set() + for s in sinkhole: + sinkhole_set.add(s.lower()) + # normalise ::ffff:x.x.x.x ↔ x.x.x.x + try: + addr = ipaddress.ip_address(s) + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped: + sinkhole_set.add(str(addr.ipv4_mapped)) + elif isinstance(addr, ipaddress.IPv4Address): + sinkhole_set.add(f"::ffff:{addr}") + except ValueError: + pass + + for ip in resolved_ips: + try: + addr = ipaddress.ip_address(ip) + if str(addr).lower() in sinkhole_set: + return True + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped: + if str(addr.ipv4_mapped) in sinkhole_set: + return True + except ValueError: + pass + return False + + +def _query_category( + domain: str, + pname: str, + cat: dict, + timeout_s: float, +) -> dict | None: + """Query one DNS provider category and return a result dict if blocked. + + Args: + domain: Domain to check. + pname: Provider key (e.g. "pihole"). + cat: Category dict with source (resolvers), sinkhole, mapping, name, tier. + timeout_s: Per-resolver timeout in seconds. + + Returns: + Category result dict if domain is blocked, None otherwise. + """ + resolvers = cat.get("source", []) + sinkhole = cat.get("sinkhole", []) + mappings = cat.get("mapping", []) + cat_name = cat.get("name") + tier = cat.get("tier", 2) + + if not resolvers or not sinkhole or not mappings: + return None + + # query resolvers sequentially — first resolver that answers is authoritative, + # remaining resolvers are skipped (fallback only on timeout/error) + for resolver_ip in resolvers: + logger.debug("DNS querying %s via %s", domain, resolver_ip) + resolved = _query_resolver(domain, resolver_ip, timeout_s) + if not resolved: + # timeout or error — try next resolver + continue + # first resolver that answered is authoritative + if _check_sinkhole(resolved, sinkhole): + logger.debug( + "DNS block: %s matched %s/%s via %s", + domain, pname, cat_name, resolver_ip, + ) + return { + "category": mappings[0], + "provider": pname, + "provider_location": "dns", + "category_name": cat_name, + "tier": tier, + } + # not in sinkhole — domain is not blocked by this provider + return None + + return None + + +def dns_provider_lookup( + domain: str, + config: dict, +) -> list[dict]: + """Query all enabled DNS provider categories for a domain in parallel. + + All enabled DNS providers are always queried unconditionally. + dns-resolve-filter does NOT gate which providers are queried here — + it is used in cache.py to decide whether DNS lookup is triggered at all. + + Args: + domain: Fully qualified domain name to check. + config: Merged altbrow config dict (with config["provider"] set). + + Returns: + List of category result dicts for blocked domains, sorted by tier. + Empty list if not blocked or no DNS providers active. + """ + providers = config.get("provider") or {} + resolve = config.get("resolve", {}) + + timeout_s = float(resolve.get("resolver-timeout", RESOLVE_DEFAULTS["resolver-timeout"])) + + # collect (pname, cat) pairs — all enabled DNS provider categories + tasks: list[tuple[str, dict]] = [] + + for pname, p in providers.items(): + if not isinstance(p, dict): + continue + if p.get("location") != "dns": + continue + if not p.get("enabled", False): + continue + + for cat in p.get("category", []): + if not cat.get("enabled", True): + continue + tasks.append((pname, cat)) + + if not tasks: + return [] + + results: list[dict] = [] + + # parallel queries — one thread per (provider, category) pair + with ThreadPoolExecutor(max_workers=min(len(tasks), 8)) as executor: + futures = { + executor.submit(_query_category, domain, pname, cat, timeout_s): (pname, cat) + for pname, cat in tasks + } + for future in as_completed(futures): + try: + result = future.result() + if result: + results.append(result) + except Exception as exc: + pname, cat = futures[future] + logger.warning("DNS query failed for %s/%s: %s", pname, cat.get("name"), exc) + + return sorted(results, key=lambda r: r.get("tier", 2)) \ No newline at end of file diff --git a/app/domain_utils.py b/altbrow/domain_utils.py similarity index 100% rename from app/domain_utils.py rename to altbrow/domain_utils.py diff --git a/altbrow/extract.py b/altbrow/extract.py new file mode 100644 index 0000000..bb370d6 --- /dev/null +++ b/altbrow/extract.py @@ -0,0 +1,208 @@ +# altbrow/extract.py +# +# extract_data() +# extract_cookies() + +import logging +import ipaddress + +from bs4 import BeautifulSoup +from pathlib import Path +from urllib.parse import urlparse + +from extruct import extract +from w3lib.html import get_base_url + +from .classify_domain import classify_domain, classify_ip +from .classify_cookies import classify_cookies + +logger = logging.getLogger(__name__) + +# HTML tags that reference external resources +_ASSET_TAGS = { + "img": "src", + "script": "src", + "link": "href", + "iframe": "src", +} +_LINK_TAGS = { + "a": "href", +} + + +def _is_ip_address(host: str) -> bool: + """Return True if host is a bare IP address (v4 or v6), not a hostname. + + Args: + host: netloc value from urlparse, may include port (e.g. '10.0.0.1:8080'). + + Returns: + True if host resolves to an IP address. + """ + # strip optional port + host = host.rsplit(":", 1)[0].strip("[]") + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +def extract_data(fetch_result: dict, cache_path: Path, config: dict | None = None, geo_readers=None) -> dict: + """Extract and classify structured data, external domains and cookies. + + Collects all external domain references from HTML tags, classifies each + against the provider cache, and extracts structured data (JSON-LD, + Microdata). + + When the target URL itself is a bare IP address, that IP is added to + external_ips using classify_ip() so RFC1918 and other provider matches + are visible in the output even when no HTML links were found. + + Args: + fetch_result: Dict returned by fetch_url(), containing html, final_url, + headers, cookies. + cache_path: Path to the SQLite provider cache file. + + Returns: + Dict with keys: + structured_data - JSON-LD and Microdata blocks + signals - external_domains, external_ips, cookies + """ + html = fetch_result["html"] + orig_url = fetch_result["url"] # original URL before redirects + final_url = fetch_result["final_url"] + headers = fetch_result["headers"] + + # use final_url for HTML analysis (links, assets) + # use orig_url for TARGET DNS lookup (before redirect to block page) + page_host = urlparse(final_url).netloc + page_domain = page_host.rsplit(":", 1)[0].strip("[]") # strip port / brackets + orig_host = urlparse(orig_url).netloc.rsplit(":", 1)[0].strip("[]") + base_url = get_base_url(html, final_url) + soup = BeautifulSoup(html, "lxml") + + # ---------- Structured Data ---------- + structured = extract( + html, + base_url=base_url, + syntaxes=["json-ld", "microdata"], + ) + + # ---------- Classify target host ---------- + classified_ips: list[dict] = [] + + if _is_ip_address(page_domain): + ip_result = classify_ip(page_domain, cache_path) + ip_result["occurrence"] = "TARGET" + if geo_readers: + from .geoip import lookup_ip as _geo_ip, format_geo + ip_result["geo"] = format_geo(_geo_ip(page_domain, geo_readers)) + classified_ips.append(ip_result) + + # TARGET domain entry — only for real hostnames, not bare IPs + # (IPs are already handled in classified_ips above) + target_result = None + if not _is_ip_address(page_domain): + # classify orig_host for DNS lookup (pre-redirect), page_domain for relation + dns_domain = orig_host if orig_host and orig_host != page_domain else page_domain + target_result = classify_domain(dns_domain, dns_domain, cache_path, config) + target_result["value"] = page_domain # display final domain + target_result["occurrence"] = "TARGET" + + # ---------- External Domains ---------- + # track occurrence per domain: asset | link | mixed + # TODO: SELF_REF — domains found only in JSON-LD @id / Microdata itemid, + # not present in any HTML tag. Requires post-extraction diffing of + # structured_data domains against domain_occurrence. See OCCURRENCE.md. + # TODO: COOKIE — domains found only in Set-Cookie Domain= attribute, + # not present in any HTML tag. Requires classify_cookies() integration + # into the domain signal pipeline. See OCCURRENCE.md. + domain_occurrence: dict[str, set] = {} + + for tag in soup.find_all(list(_ASSET_TAGS) + list(_LINK_TAGS)): + attr = _ASSET_TAGS.get(tag.name) or _LINK_TAGS.get(tag.name) + url = tag.get(attr) + if not url: + continue + + parsed = urlparse(url) + if not parsed.netloc or parsed.netloc == page_host: + continue + + kind = "asset" if tag.name in _ASSET_TAGS else "link" + domain_occurrence.setdefault(parsed.netloc, set()).add(kind) + + # add geo data to a result dict in-place + def _add_geo(result: dict, name: str) -> None: + if geo_readers and result: + from .geoip import lookup_domain as _geo_d, format_geo + result["geo"] = format_geo(_geo_d(name, geo_readers)) + + if target_result: + _add_geo(target_result, target_result["value"]) + classified_domains = [target_result] if target_result else [] + + for domain, kinds in sorted(domain_occurrence.items()): + if "asset" in kinds and "link" in kinds: + occurrence = "MIXED" + elif "asset" in kinds: + occurrence = "ASSET" + else: + occurrence = "LINK_ONLY" + + result = classify_domain(domain, page_domain, cache_path, config) + result["occurrence"] = occurrence + _add_geo(result, domain) + classified_domains.append(result) + + # ---------- Cookies ---------- + raw_cookies = headers.get("Set-Cookie") + classified_cookies = [] + + if raw_cookies: + try: + for raw in raw_cookies.split(","): + classified_cookies.append( + classify_cookies(raw.strip(), page_domain) + ) + except Exception as exc: + logger.warning("Cookie parsing failed: %s", exc) + + return { + "structured_data": structured, + "signals": { + "external_domains": classified_domains, + "external_ips": classified_ips, + "cookies": classified_cookies, + }, + } + + +def extract_cookies(cookiejar, page_domain: str, cache_path: Path, config: dict | None = None) -> list[dict]: + """Classify cookies from a requests CookieJar. + + Args: + cookiejar: requests.cookies.RequestsCookieJar from fetch_url(). + page_domain: Hostname of the analysed page. + cache_path: Path to the SQLite provider cache file. + + Returns: + List of dicts with cookie attributes and domain classification. + """ + cookies = [] + + for c in cookiejar: + domain = c.domain.lstrip(".") + cookies.append({ + "name": c.name, + "domain": domain, + "path": c.path, + "secure": c.secure, + "httponly": c.has_nonstandard_attr("HttpOnly"), + "samesite": c.get_nonstandard_attr("SameSite"), + "expires": c.expires, + "class": classify_domain(domain, page_domain, cache_path, config), + }) + + return cookies diff --git a/altbrow/fetch.py b/altbrow/fetch.py new file mode 100644 index 0000000..a22e35a --- /dev/null +++ b/altbrow/fetch.py @@ -0,0 +1,153 @@ +import logging +import requests +import unicodedata +import urllib3 + +from urllib.parse import urlparse, quote, unquote + +logger = logging.getLogger(__name__) + + +def _normalize_url(url: str) -> str: + """Normalize a URL for safe HTTP transmission. + + Applies Unicode NFC normalization to the path component, then + re-encodes it with percent-encoding. Already-encoded sequences + are decoded first to avoid double-encoding. + + This handles IRI (Internationalized Resource Identifiers) as used + in Malayalam, Arabic, CJK and other non-ASCII URLs pasted from + browsers or copy-paste with shell encoding artifacts. + + Args: + url: Raw URL string, may contain Unicode path or broken percent-encoding. + + Returns: + URL with NFC-normalized, correctly percent-encoded path. + """ + parsed = urlparse(url) + + # decode any existing percent-encoding, then NFC-normalize, then re-encode + path_decoded = unquote(parsed.path, encoding="utf-8", errors="replace") + path_nfc = unicodedata.normalize("NFC", path_decoded) + path_encoded = quote(path_nfc, safe="/:@!$&'()*+,;=") + + return parsed._replace(path=path_encoded).geturl() + + +def fetch_url(url: str, client_profile: dict) -> dict: + """Fetch a URL using the given client profile settings. + + The URL is NFC-normalized before the request to handle non-ASCII paths + (IRI) correctly across shells and operating systems. + + Args: + url: Target URL (http or https). + client_profile: Merged client profile dict from config. + Relevant keys: + headers (dict) - HTTP request headers + use_session (bool) - use requests.Session for cookies + check_cert (bool) - verify TLS certificate (default True) + timeout (int) - request timeout in seconds (default 10) + + Returns: + Dict with keys: + url - original URL as given + final_url - URL after redirects + status_code - HTTP response status + headers - response headers dict + encoding - response encoding + html - response body as string + cookies - CookieJar + + Raises: + requests.exceptions.MissingSchema: Invalid URL. + requests.exceptions.Timeout: Request timed out. + requests.exceptions.ConnectionError: Network unreachable. + requests.exceptions.HTTPError: 4xx/5xx response. + requests.exceptions.RequestException: Any other request failure. + """ + headers = client_profile.get("headers", {}) + use_session = client_profile.get("use_session", False) + check_cert = client_profile.get("check_cert", True) + timeout = client_profile.get("timeout", 10) + + if not check_cert: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + normalized_url = _normalize_url(url) + if normalized_url != url: + logger.debug("URL normalized: %s -> %s", url, normalized_url) + + try: + if use_session: + session = requests.Session() + if headers: + session.headers.update(headers) + response = session.get(normalized_url, timeout=timeout, verify=check_cert) + cookies = session.cookies + else: + response = requests.get( + normalized_url, + headers=headers or None, + timeout=timeout, + verify=check_cert, + ) + cookies = response.cookies + + response.raise_for_status() + + logger.debug( + "Fetched %s (status %s)", + response.url, + response.status_code + ) + + return { + "url": url, # original as given by user + "final_url": response.url, + "status_code": response.status_code, + "headers": dict(response.headers), + "encoding": response.encoding, + "html": response.text, + "cookies": cookies, + } + + except requests.exceptions.MissingSchema: + logger.error("Invalid URL (missing scheme): %s", url) + raise + + except requests.exceptions.Timeout: + logger.error("Timeout while fetching %s", url) + raise + + except requests.exceptions.ConnectionError as exc: + exc_str = str(exc) + if "no.access" in exc_str or "NameResolutionError" in exc_str and "no.access" in exc_str: + logger.error( + "Server redirected %s to a block page (no.access) — " + "site may require browser headers (try --client-profile browser)", url + ) + elif "NewConnectionError" in exc_str or "Connection refused" in exc_str or "WinError" in exc_str: + logger.error("Connection refused for %s (host unreachable or port closed)", url) + elif "SSLError" in exc_str or "SSL" in exc_str: + logger.error("TLS error for %s (try --no-cert-check for self-signed certs)", url) + else: + logger.error("Connection error for %s: %s", url, exc) + raise + + except requests.exceptions.HTTPError as exc: + status = exc.response.status_code if exc.response else "?" + has_non_ascii = any(ord(c) > 127 for c in url) + is_404 = status == 404 or "404" in str(exc) + hint = ( + " (non-ASCII URL path: shell may have dropped combining characters)" + if is_404 and has_non_ascii + else "" + ) + logger.error("HTTP error %s for %s%s", status, url, hint) + raise + + except requests.exceptions.RequestException as exc: + logger.error("Request failed for %s: %s", url, exc) + raise diff --git a/altbrow/fetch_remote.py b/altbrow/fetch_remote.py new file mode 100644 index 0000000..8328c71 --- /dev/null +++ b/altbrow/fetch_remote.py @@ -0,0 +1,235 @@ +# altbrow/fetch_remote.py +# +# parse_entries() +# parse_list() +# fetch_remote_source() +# fetch_remote_provider() + +import logging +import re + +import requests + +from .config import LOCATION_DEFAULT_TIER + +logger = logging.getLogger(__name__) + +# Metadata key pattern: "# Key : Value" or "# key: value" +_META_RE = re.compile( + r'^[ \t]{0,9}#[ \t]{0,2}([A-Za-z0-9][A-Za-z0-9._ -]{0,31})[ \t]{0,9}:[ \t]{0,5}(.{0,256})$' +) + +_ALLOWED_META_KEYS = { + "name", "list", "license", "source", "created", + "updated", "version", "description", "total entries", +} + +# hosts-format: line starts with an IP address token followed by whitespace +_HOSTS_LINE_RE = re.compile(r'^\s*([0-9a-fA-F.:]+)\s+(.+)') + +# well-known non-domain tokens to skip from hosts files +_HOSTS_SKIP = { + "localhost", "localhost.localdomain", "local", + "broadcasthost", "ip6-localhost", "ip6-loopback", "ip6-localnet", + "ip6-mcastprefix", "ip6-allnodes", "ip6-allrouters", "ip6-allhosts", +} + + +def _is_ip(token: str) -> bool: + """Return True if token is a valid IPv4 or IPv6 address. + + Args: + token: String to check. + + Returns: + True if token parses as an IP address. + """ + import ipaddress + try: + ipaddress.ip_address(token) + return True + except ValueError: + return False + + +def parse_entries(text: str) -> list[str]: + """Extract domain or IP entries from a list in altbrow or hosts format. + + Supports two formats transparently: + + - **altbrow list format**: one domain/IP per line, ``#`` comments ignored + - **hosts file format**: `` hostname [hostname ...]`` lines — + any IP address is accepted (not just 0.0.0.0/127.0.0.1), multiple + hostnames per line are all extracted, inline ``#`` comments are stripped + + Format is detected per-line so mixed files work correctly. + Leading comment blocks (metadata) are skipped like all other comments. + + Args: + text: Raw text content of a list file or HTTP response. + + Returns: + List of domain or IP strings, lowercased and stripped. + """ + entries: list[str] = [] + + for line in text.splitlines(): + stripped = line.strip() + + if not stripped or stripped.startswith("#"): + continue + + m = _HOSTS_LINE_RE.match(stripped) + if m and _is_ip(m.group(1)): + # hosts format — extract all hostnames after the IP, strip inline comment + rest = m.group(2) + if "#" in rest: + rest = rest[:rest.index("#")] + for token in rest.split(): + token = token.lower() + if token and token not in _HOSTS_SKIP: + entries.append(token) + else: + # plain list format — one entry per line, strip inline comment + entry = stripped + if "#" in entry: + entry = entry[:entry.index("#")].strip() + if entry: + entries.append(entry.lower()) + + return entries + + +def parse_list(text: str) -> tuple[dict, list[str]]: + """Parse a domain or IP list, extracting metadata and entries. + + Reads optional metadata from leading comment lines (``# key: value``), + then delegates entry extraction to ``parse_entries()`` which handles + both altbrow list format and hosts file format. + + Metadata is only read from leading comment lines before the first + non-comment line — same convention as altbrow list format. + + Args: + text: Raw text content of the list file or HTTP response. + + Returns: + Tuple of: + meta - dict of parsed metadata keys (lowercased, spaces→underscore) + entries - list of domain or IP strings from parse_entries() + """ + meta: dict = {} + header_done = False + + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + if stripped.startswith("#"): + if not header_done: + m = _META_RE.match(stripped) + if m: + key = m.group(1).strip().lower().replace(" ", "_") + val = m.group(2).strip() + if key in _ALLOWED_META_KEYS or any( + key == k.replace(" ", "_") for k in _ALLOWED_META_KEYS + ): + meta[key] = val + continue + header_done = True + break # stop at first non-comment line — entries handled by parse_entries + + return meta, parse_entries(text) + + +def fetch_remote_source(url: str, timeout: int = 15) -> tuple[dict, list[str]]: + """Fetch a remote list URL and parse it. + + Args: + url: HTTP or HTTPS URL of the list. + timeout: Request timeout in seconds. + + Returns: + Tuple of (meta, entries) from parse_list(). + + Raises: + requests.exceptions.RequestException: On network or HTTP errors. + """ + logger.debug("Fetching remote source: %s", url) + + response = requests.get(url, timeout=timeout) + response.raise_for_status() + + meta, entries = parse_list(response.text) + + logger.debug( + "Fetched %s: %d entries, meta=%s", + url, len(entries), meta + ) + + return meta, entries + + +def fetch_remote_provider( + pname: str, + p: dict, +) -> list[tuple[str, dict]]: + """Fetch all enabled categories of a remote provider. + + Fetches each unique source URL only once, then distributes entries + to all categories that reference the same URL. + + Args: + pname: Provider name (for logging). + p: Provider config dict from provider.toml. + + Returns: + List of tuples: (entry, context) where context contains: + category, provider, provider_location, category_name, meta + """ + results = [] + ptype = p.get("type") + + # fetch each URL only once + url_cache: dict[str, tuple[dict, list[str]]] = {} + + for cat in p.get("category", []): + if not cat.get("enabled", True): + continue + + cat_name = cat.get("name") + mappings = cat.get("mapping", []) + sources = cat.get("source", []) + tier = cat.get("tier", LOCATION_DEFAULT_TIER["remote"]) + + for url in sources: + if url not in url_cache: + try: + url_cache[url] = fetch_remote_source(url) + logger.info( + "Provider '%s' fetched %d entries from %s", + pname, len(url_cache[url][1]), url + ) + except Exception as exc: + logger.warning("Failed to fetch '%s' from %s: %s", pname, url, exc) + url_cache[url] = ({}, []) + + meta, entries = url_cache[url] + + for entry in entries: + entry = entry.strip().lower() + if not entry: + continue + + for category in mappings: + results.append((entry, { + "category": category, + "provider": pname, + "provider_location": "remote", + "category_name": cat_name, + "tier": tier, + "meta": meta, + "ptype": ptype, + })) + + return results \ No newline at end of file diff --git a/altbrow/geoip.py b/altbrow/geoip.py new file mode 100644 index 0000000..eeaf0d5 --- /dev/null +++ b/altbrow/geoip.py @@ -0,0 +1,290 @@ +# altbrow/geoip.py +# +# open_geodbs() +# lookup_ip() +# lookup_domain() +# GeoReaders (namedtuple) + +import logging +import socket +import tarfile + +from pathlib import Path +from typing import NamedTuple + +logger = logging.getLogger(__name__) + + +class GeoReaders(NamedTuple): + """Holds open MaxMind MMDB reader handles for the session. + + All fields may be None if the respective DB file is not found. + City is used if present, Country as fallback. + """ + country: object | None # maxminddb.Reader or None + asn: object | None # maxminddb.Reader or None + city: object | None # maxminddb.Reader or None (superset of country) + + +def _extract_mmdb(tar_path: Path, dest_dir: Path) -> Path | None: + """Extract .mmdb file from a MaxMind .tar.gz archive. + + Args: + tar_path: Path to the .tar.gz file. + dest_dir: Directory to extract the .mmdb file into. + + Returns: + Path to the extracted .mmdb file, or None on failure. + """ + try: + with tarfile.open(tar_path, "r:gz") as tar: + for member in tar.getmembers(): + if member.name.endswith(".mmdb"): + member.name = Path(member.name).name # strip directory prefix + tar.extract(member, path=dest_dir) + extracted = dest_dir / member.name + logger.info("Extracted %s from %s", extracted.name, tar_path.name) + return extracted + except Exception as exc: + logger.warning("Failed to extract %s: %s", tar_path.name, exc) + return None + + +def extract_geodbs(config_path: Path, provider_cfg: dict | None = None) -> None: + """Extract GeoLite2 .tar.gz files defined in geoip provider categories. + + Called during --build-cache. Existing .mmdb files are overwritten when + a newer .tar.gz is found (determined by mtime). Garbage collection of + old .tar.gz files is left to the user/admin. + + Args: + config_path: Path to the active altbrow.toml file. + provider_cfg: Full provider config dict (with sources). If None, + falls back to scanning config dir for GeoLite2-*.tar.gz. + """ + + base = config_path.parent + + # collect (source_path, dest_dir) pairs from provider config + tar_paths: list[Path] = [] + + if provider_cfg: + for pname, p in provider_cfg.get("provider", {}).items(): + if not p.get("enabled", False): + continue + location = p.get("location") + for cat in p.get("category", []): + if not cat.get("enabled", True): + continue + if "geoip" not in cat.get("mapping", []): + continue + for src in cat.get("source", []): + if location == "local": + src_path = Path(src) + if not src_path.is_absolute(): + src_path = base / src + # resolve glob pattern + parent = src_path.parent + pattern = src_path.name + matches = sorted(parent.glob(pattern)) + if matches: + tar_paths.append(matches[-1]) # newest by name + else: + logger.warning("GeoIP: no file matching %s", src) + elif location == "remote": + # download .tar.gz to config dir + try: + import requests as _req + fname = src.rstrip("/").split("/")[-1] + dest = base / fname + logger.info("GeoIP: downloading %s", src) + r = _req.get(src, timeout=30) + r.raise_for_status() + dest.write_bytes(r.content) + tar_paths.append(dest) + except Exception as exc: + logger.warning("GeoIP: failed to download %s: %s", src, exc) + else: + # fallback: scan config dir + tar_paths = sorted(base.glob("GeoLite2-*.tar.gz")) + + for tar_path in tar_paths: + stem = tar_path.name.split("_")[0] # GeoLite2-Country + if not stem.startswith("GeoLite2-"): + # try to get stem from mmdb inside archive + stem = None + mmdb_path = base / f"{stem}.mmdb" if stem else None + # overwrite if tar is newer than existing mmdb + if mmdb_path and mmdb_path.exists(): + if tar_path.stat().st_mtime <= mmdb_path.stat().st_mtime: + logger.info("GeoIP: %s up to date, skipping", mmdb_path.name) + continue + _extract_mmdb(tar_path, base) + + +def open_geodbs(config_path: Path, allowed_names: set[str] | None = None) -> GeoReaders | None: + """Open GeoLite2 MMDB readers from the directory of altbrow.toml. + + Looks for GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb next to altbrow.toml. + Missing files are logged as warnings, not errors — altbrow continues without GeoIP. + + Args: + config_path: Path to the active altbrow.toml file. + allowed_names: Set of DB names to open e.g. {"Country", "ASN", "City"}. + If None, all found DBs are opened. + + Returns: + GeoReaders namedtuple with open readers, or None if maxminddb is not installed. + """ + try: + import maxminddb + except ImportError: + logger.warning( + "maxminddb not installed — GeoIP disabled. " + "Install with: pip install maxminddb" + ) + return None + + base = config_path.parent + + def _open(name: str, db_type: str) -> object | None: + if allowed_names is not None and db_type not in allowed_names: + logger.debug("GeoIP: %s disabled by provider config", db_type) + return None + path = base / name + if not path.exists(): + logger.info("GeoIP: %s not found — lookup disabled", name) + return None + try: + reader = maxminddb.open_database(str(path)) + logger.debug("GeoIP DB opened: %s", name) + return reader + except Exception as exc: + logger.warning("Failed to open %s: %s", name, exc) + return None + + city_reader = _open("GeoLite2-City.mmdb", "City") + country_reader = _open("GeoLite2-Country.mmdb", "Country") + asn_reader = _open("GeoLite2-ASN.mmdb", "ASN") + + if city_reader is None and country_reader is None and asn_reader is None: + return None + + active = [n for n, r in [("City", city_reader), ("Country", country_reader), ("ASN", asn_reader)] if r] + logger.debug("GeoIP ready: %s", ", ".join(active)) + + return GeoReaders(country=country_reader, asn=asn_reader, city=city_reader) + + +def lookup_ip(ip_str: str, readers: GeoReaders) -> dict: + """Look up GeoIP data for an IP address. + + Args: + ip_str: IPv4 or IPv6 address string. + readers: Open GeoReaders from open_geodbs(). + + Returns: + Dict with keys (all optional, None if not available): + country_code - ISO 3166-1 alpha-2 (e.g. 'DE') + country_name - English name (e.g. 'Germany') + asn - AS number as string (e.g. 'AS13184') + asn_org - Organisation name (e.g. 'Deutsche Telekom') + Empty dict if no data available. + """ + result: dict = {} + + # City DB is superset of Country — use it first if available + country_reader = readers.city or readers.country + if country_reader: + try: + rec = country_reader.get(ip_str) + if rec: + country = rec.get("country") or rec.get("registered_country", {}) + result["country_code"] = country.get("iso_code") + names = country.get("names", {}) + result["country_name"] = names.get("en") + if readers.city and rec.get("city"): + city_names = rec["city"].get("names", {}) + result["city"] = city_names.get("en") + except Exception as exc: + logger.debug("GeoIP country/city lookup failed for %s: %s", ip_str, exc) + + if readers.asn: + try: + rec = readers.asn.get(ip_str) + if rec: + asn_num = rec.get("autonomous_system_number") + result["asn"] = f"AS{asn_num}" if asn_num else None + result["asn_org"] = rec.get("autonomous_system_organization") + except Exception as exc: + logger.debug("GeoIP ASN lookup failed for %s: %s", ip_str, exc) + + return result + + +def lookup_domain(domain: str, readers: GeoReaders) -> dict: + """Resolve domain to IP and look up GeoIP data. + + Uses the system resolver — no DNS provider involved. + Takes the first resolved IPv4 address for lookup. + + Args: + domain: Fully qualified domain name. + readers: Open GeoReaders from open_geodbs(). + + Returns: + GeoIP dict from lookup_ip(), or empty dict on failure. + """ + try: + # prefer IPv4 for GeoIP lookup — more reliable coverage + results = socket.getaddrinfo(domain, None, socket.AF_INET) + if not results: + results = socket.getaddrinfo(domain, None) + ip = results[0][4][0] + return lookup_ip(ip, readers) + except Exception as exc: + logger.debug("GeoIP domain resolve failed for %s: %s", domain, exc) + return {} + + +def format_geo(geo: dict) -> str: + """Format GeoIP result as compact display string. + + Args: + geo: Dict from lookup_ip() or lookup_domain(). + + Returns: + Compact string e.g. 'DE/AS13184 Deutsche Telekom', or empty string. + """ + if not geo: + return "" + + parts = [] + loc = geo.get("country_code", "") + if geo.get("city"): + loc += f"/{geo['city']}" + if loc: + parts.append(loc) + if geo.get("asn"): + asn_str = geo["asn"] + if geo.get("asn_org"): + asn_str += f" {geo['asn_org']}" + parts.append(asn_str) + + return " ".join(parts) if parts else "" + + +def close_geodbs(readers: GeoReaders | None) -> None: + """Close open MMDB reader handles. + + Args: + readers: GeoReaders from open_geodbs(), or None. + """ + if readers is None: + return + for reader in [readers.city, readers.country, readers.asn]: + if reader: + try: + reader.close() + except Exception: + pass diff --git a/app/logging_config.py b/altbrow/logging_config.py similarity index 100% rename from app/logging_config.py rename to altbrow/logging_config.py diff --git a/altbrow/main.py b/altbrow/main.py new file mode 100644 index 0000000..b854f1e --- /dev/null +++ b/altbrow/main.py @@ -0,0 +1,244 @@ +import argparse +import logging +import sys + +from pathlib import Path +from urllib.parse import urlparse + +from altbrow import __version__ +from .fetch import fetch_url +from .extract import extract_data +from .logging_config import setup_logging +from .config import ( + load_toml, + get_client_profile, + validate_altbrow_config, + discover_config_path, + load_provider_config, + ConfigError, +) +from .cache import build_cache, get_or_build_cache +from .output import render_output, write_log +from .geoip import open_geodbs, close_geodbs, extract_geodbs + + +def main() -> int: + """Entry point for the altbrow CLI. + + Parses arguments, loads config, builds/validates cache, + fetches the target URL and renders the analysis output. + + Returns: + Exit code: + 0 - success + 2 - CLI usage error + 3 - config/cache error + 4 - network/analysis error + """ + parser = argparse.ArgumentParser() + + parser.add_argument("url", nargs="?", help="URL to analyze") + + parser.add_argument( + "-V", "--version", + action="version", + version=f"Altbrow v{__version__}" + ) + + parser.add_argument( + "--config", + help="Path to configuration file (TOML)" + ) + + parser.add_argument( + "-o", "--output", + help="Write result to file" + ) + + parser.add_argument( + "-f", "--format", + choices=["text", "yaml", "json"], + default="text", + help="Output format (default: text)" + ) + + parser.add_argument( + "-v", "--verbose", + action="count", + default=0, + help="Increase text detail level (-vv, -vvv)" + ) + + parser.add_argument( + "--client-profile", + choices=["passive", "browser", "consented"], + help="HTTP client behavior profile (default: from config)", + ) + + parser.add_argument( + "--no-cert-check", + dest="check_cert", + action="store_false", + help="Disable TLS certificate verification (self-signed certs, local hosts)" + ) + + parser.add_argument( + "--validate-config", + action="store_true", + help="Validate altbrow.toml & provider.toml and exit" + ) + + parser.add_argument( + "--build-cache", + action="store_true", + help="(Re)build provider cache DB, unpack geoIP mmdd files and exit" + ) + + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging (steps, DNS queries, cache hits)" + ) + + parser.add_argument( + "--log-file", + metavar="PATH", + help="Write log to file (default: altbrow.log next to altbrow.toml)" + ) + + args = parser.parse_args() + + # pre-parse --debug before full setup so logging is active during config load + debug_mode = "--debug" in sys.argv + setup_logging(debug=debug_mode) + logger = logging.getLogger("altbrow") + + try: + config_path = discover_config_path(args.config) + config = load_toml(config_path) + client_profile = get_client_profile(config, args.client_profile) + provider_config = load_provider_config(config_path, config) + # config["provider"] is now set by load_provider_config() + + # log file — next to altbrow.toml or explicit path + if args.log_file or args.debug: + log_path = ( + Path(args.log_file) if args.log_file + else config_path.parent / "altbrow.log" + ) + fh = logging.FileHandler(log_path, encoding="utf-8") + fh.setLevel(logging.DEBUG if args.debug else logging.INFO) + fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")) + logging.getLogger().addHandler(fh) + logger.debug("Log file: %s", log_path) + except ConfigError as exc: + logger.error("Config error: %s", exc) + return 3 + + # --no-cert-check overrides check_cert from profile + if not args.check_cert: + client_profile["check_cert"] = False + + if args.validate_config: + try: + text = validate_altbrow_config(config) + print(text) + if args.verbose >= 1: + import json + # -v: merged config without sources (config["provider"] already stripped) + # -vv: merged config + full provider sources from raw provider_config + if args.verbose >= 2 and provider_config: + display = dict(config) + display["provider"] = provider_config.get("provider", {}) + else: + display = config + print(json.dumps(display, indent=2, ensure_ascii=False, default=str)) + return 0 + except ConfigError as exc: + logger.error("Config validation failed: %s", exc) + return 3 + + # --- cache --- + cache_path = config_path.parent / ".altbrow.cache" + + if args.build_cache: + if provider_config is None: + logger.error("--build-cache requires provider config (meta.use-provider = true)") + return 3 + build_cache(cache_path, provider_config, config_path) + print(f"Cache built: {cache_path}") + # GeoIP — extract tar.gz archives if geoip provider enabled + has_geoip_provider = any( + "geoip" in cat.get("mapping", []) + for p in (config.get("provider") or {}).values() + if isinstance(p, dict) and p.get("enabled") + for cat in p.get("category", []) if cat.get("enabled", True) + ) + if has_geoip_provider: + extract_geodbs(config_path, provider_config) + geo_readers = open_geodbs(config_path) + if geo_readers: + close_geodbs(geo_readers) + else: + print("GeoIP disabled: no GeoLite2-*.mmdb found next to altbrow.toml") + return 0 + + try: + get_or_build_cache(cache_path, provider_config, config_path) + except Exception as exc: + logger.error("Cache build failed: %s", exc) + return 3 + + # GeoIP readers — open if any geoip provider is enabled + geo_readers = None + providers = config.get("provider") or {} + has_geoip = any( + "geoip" in cat.get("mapping", []) + for p in providers.values() if isinstance(p, dict) and p.get("enabled") + for cat in p.get("category", []) if cat.get("enabled", True) + ) + if has_geoip: + # collect enabled geoip category names (Country/ASN/City) + allowed_geo = { + cat.get("name") + for p in providers.values() if isinstance(p, dict) and p.get("enabled") + for cat in p.get("category", []) + if cat.get("enabled", True) and "geoip" in cat.get("mapping", []) and cat.get("name") + } + geo_readers = open_geodbs(config_path, allowed_geo or None) + if geo_readers: + logger.debug("GeoIP readers opened") + + if not args.url: + parser.print_usage() + logger.error("URL is required (or use --validate-config / --build-cache)") + return 2 + + url = args.url + if not url.startswith(("http://", "https://")): + url = "https://" + url + + parsed = urlparse(url) + + if parsed.scheme not in ("http", "https") or not parsed.netloc: + logger.error("Invalid URL") + return 2 + + try: + fetched = fetch_url(url, client_profile) + extracted = extract_data(fetched, cache_path, config, geo_readers) + except Exception as exc: + logger.error("Analysis failed: %s", exc) + return 4 + + render_output(extracted, args.format, config, verbosity=args.verbose) + + if args.output: + write_log(extracted, args.output) + + close_geodbs(geo_readers) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/altbrow/output.py b/altbrow/output.py new file mode 100644 index 0000000..5ee33ff --- /dev/null +++ b/altbrow/output.py @@ -0,0 +1,222 @@ +# altbrow/output.py +# +# render_output() +# write_log() + +import json +import logging +import sys + +logger = logging.getLogger(__name__) + +# ensure UTF-8 output on Windows (cp1252 default breaks unicode chars) +if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + +try: + import yaml +except ImportError: + yaml = None + + +def _format_categories(cats: list[dict], providers: dict | None = None) -> tuple[str, str]: + """Split categories into winning (lowest tier) and full sorted list. + + Categories must be pre-sorted by tier ascending (classify_domain does this). + providers is config["provider"] — used to resolve human-readable provider names. + category_name is already stored in DB and used directly. + + Returns: + Tuple (winning, all_str): + winning - lowest-tier category name, or 'unknown' if no match + all_str - all as 'category(provider/category_name)', or '-' if no match + """ + if not cats: + return "unknown", "-" + winning = cats[0]["category"] + providers = providers or {} + def _fmt(c: dict) -> str: + p_label = providers.get(c["provider"], {}).get("name") or c["provider"] + cat_label = c.get("category_name") + label = f"{p_label}/{cat_label}" if cat_label else p_label + return c["category"] + "(" + label + ")" + all_str = ", ".join(_fmt(c) for c in cats) + return winning, all_str + + +def _render_text(extracted: dict, verbosity: int, providers: dict | None = None) -> None: + """Render human-readable text output to STDOUT. + + Args: + extracted: Dict returned by extract_data(). + verbosity: Detail level (0=summary, 1=domains+ips, 2=full). + providers: config["provider"] for human-readable label lookup. + geo_readers: Open GeoReaders for live GeoIP lookup, or None. + """ + signals = extracted.get("signals", {}) + structured = extracted.get("structured_data", {}) + + domains = signals.get("external_domains", []) + ips = signals.get("external_ips", []) + cookies = signals.get("cookies", []) + jsonld = structured.get("json-ld", []) + microdata = structured.get("microdata", []) + + # count by winning category (lowest tier) per domain + cat_counts: dict[str, int] = {} + for d in domains: + cats = d.get("categories", []) + winning = cats[0]["category"] if cats else "unknown" + cat_counts[winning] = cat_counts.get(winning, 0) + 1 + + cat_summary = ", ".join( + f"{k}: {v}" for k, v in sorted(cat_counts.items()) + ) + + # count by country code from geo field + geo_counts: dict[str, int] = {} + for d in domains: + geo = d.get("geo", "") + cc = geo.split("/")[0].split(" ")[0] if geo else None + if cc and len(cc) == 2 and cc.isalpha(): + geo_counts[cc] = geo_counts.get(cc, 0) + 1 + geo_summary = ", ".join( + f"{k}: {v}" for k, v in sorted(geo_counts.items(), key=lambda x: -x[1]) + ) + + # count by winning category (lowest tier) per IP + ip_cat_counts: dict[str, int] = {} + for ip in ips: + cats = ip.get("categories", []) + winning = cats[0]["category"] if cats else "unknown" + ip_cat_counts[winning] = ip_cat_counts.get(winning, 0) + 1 + + ip_cat_summary = ", ".join( + f"{k}: {v}" for k, v in sorted(ip_cat_counts.items()) + ) + + print("\n=== Summary ===") + geo_part = f" ({geo_summary})" if geo_summary else "" + print(f"External domains : {len(domains)}" + (f" ({cat_summary})" if cat_summary else "") + geo_part) + print(f"External IPs : {len(ips)}" + (f" ({ip_cat_summary})" if ip_cat_summary else "")) + print(f"Cookies : {len(cookies)}") + print(f"JSON-LD blocks : {len(jsonld)}") + print(f"Microdata blocks : {len(microdata)}") + + if verbosity < 1: + return + + print("\n=== External Domains ===") + for d in domains: + winning, all_str = _format_categories(d.get("categories", []), providers) + geo = d.get("geo", "") + geo_col = f"[{geo}]" if geo else "-" + if verbosity >= 2: + print( + f" {d['relation']:<12} {d.get('occurrence',''):<10} " + f"{d['value']:<40} {winning:<15} {geo_col:<20} {all_str}" + ) + else: + print( + f" {d['relation']:<12} {d.get('occurrence',''):<10} " + f"{d['value']:<40} {winning:<15} {geo_col}" + ) + + print("\n=== External IPs ===") + if not ips: + print(" (none)") + else: + for ip in ips: + winning, all_str = _format_categories(ip.get("categories", []), providers) + occurrence = ip.get("occurrence", "") + geo = ip.get("geo", "") + geo_col = f"[{geo}]" if geo else "-" + if verbosity >= 2: + print( + f" {ip['relation']:<12} {occurrence:<10} " + f"{ip['value']:<40} {winning:<15} {geo_col:<20} {all_str}" + ) + else: + print( + f" {ip['relation']:<12} {occurrence:<10} " + f"{ip['value']:<40} {winning:<15} {geo_col}" + ) + + if verbosity < 2: + return + + print("\n=== Cookies ===") + for c in cookies: + flags = [] + if c.get("third_party"): + flags.append("3rd-party") + if c.get("cross_site"): + flags.append("cross-site") + print(f" {c['name']:<30} {', '.join(flags)}") + + print("\n=== JSON-LD ===") + if not jsonld: + print(" (none)") + else: + for i, block in enumerate(jsonld, 1): + print(f" Block {i}: {block.get('@type', '?')}") + + print("\n=== Microdata ===") + if not microdata: + print(" (none)") + else: + for i, block in enumerate(microdata, 1): + print(f" Block {i}: {block.get('type', '?')}") + + +def render_output( + extracted: dict, + output_mode: str, + config: dict, + verbosity: int = 0, +) -> None: + """Render analysis results to STDOUT in the requested format. + + Args: + extracted: Dict returned by extract_data(). + output_mode: 'text' | 'json' | 'yaml' + config: Merged altbrow config — config["provider"] used for label lookup. + verbosity: Detail level for text mode (0=summary, 1=domains, 2=full). + """ + if output_mode == "text": + providers = config.get("provider") or {} + _render_text(extracted, verbosity, providers) + return + + if output_mode == "json": + print(json.dumps(extracted, indent=2, ensure_ascii=False)) + return + + if output_mode == "yaml": + if yaml is None: + raise RuntimeError("YAML output requested but PyYAML is not installed") + print(yaml.safe_dump(extracted, sort_keys=False, allow_unicode=True)) + return + + # fallback: explicit_format from config + fmt = config.get("output", {}).get("explicit_format", "json") + + if fmt == "json": + print(json.dumps(extracted, indent=2, ensure_ascii=False)) + elif fmt == "yaml": + if yaml is None: + raise RuntimeError("YAML output requested but PyYAML is not installed") + print(yaml.safe_dump(extracted, sort_keys=False, allow_unicode=True)) + else: + raise ValueError(f"Unknown output format: {fmt}") + + +def write_log(extracted: dict, path: str) -> None: + """Write full analysis result as JSON to a file. + + Args: + extracted: Dict returned by extract_data(). + path: Output file path. + """ + with open(path, "w", encoding="utf-8") as f: + json.dump(extracted, f, indent=2, ensure_ascii=False) \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index ce33138..0000000 --- a/app/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# app/__main__.py - -__version__ = "0.1.0" - -# from .main import main - -# if __name__ == "__main__": -# main() diff --git a/app/__main__.py b/app/__main__.py deleted file mode 100644 index 25b2c0f..0000000 --- a/app/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -# app/__main__.py - -from .main import main - -main() \ No newline at end of file diff --git a/app/classify.py b/app/classify.py deleted file mode 100644 index 7d6e737..0000000 --- a/app/classify.py +++ /dev/null @@ -1,34 +0,0 @@ -# from urllib.parse import urlparse - - -KNOWN_CDN_HINTS = ( - "cdn", - "cloudflare", - "akamai", - "fastly", - "jsdelivr", - "unpkg", -) - - -KNOWN_ANALYTICS_HINTS = ( - "analytics", - "gtag", - "google-analytics", - "matomo", -) - - -def classify_domain(domain: str, page_domain: str, config) -> str: - if domain == page_domain or domain.endswith("." + page_domain): - return "FIRST_PARTY" - - lowered = domain.lower() - - if any(hint in lowered for hint in KNOWN_CDN_HINTS): - return "CDN" - - if any(hint in lowered for hint in KNOWN_ANALYTICS_HINTS): - return "ANALYTICS" - - return "OTHER" diff --git a/app/classify_cookies.py b/app/classify_cookies.py deleted file mode 100644 index 5c10344..0000000 --- a/app/classify_cookies.py +++ /dev/null @@ -1,26 +0,0 @@ -def classify_cookies(raw_cookie: str, page_domain: str, config) -> dict: - parts = [p.strip() for p in raw_cookie.split(";")] - - name, *_ = parts[0].split("=", 1) - - result = { - "name": name, - "third_party": False, - "cross_site": False, - "attributes": [], - } - - for part in parts[1:]: - lower = part.lower() - result["attributes"].append(part) - - if lower.startswith("domain="): - domain = lower.split("=", 1)[1].lstrip(".") - if domain != page_domain: - result["third_party"] = True - - if lower == "samesite=none": - result["cross_site"] = True - - return result - diff --git a/app/classify_domain.py b/app/classify_domain.py deleted file mode 100644 index 9fc75c2..0000000 --- a/app/classify_domain.py +++ /dev/null @@ -1,50 +0,0 @@ -from .domain_utils import get_registrable_domain - -def classify_domain(domain: str, page_domain: str, config: dict) -> dict: - domain = domain.lower() - page_domain = page_domain.lower() - - reg_domain = get_registrable_domain(domain) - reg_page = get_registrable_domain(page_domain) - - trust = "UNKNOWN" - trust_cfg = config.get("trust", {}) - - if reg_domain in trust_cfg.get("trusted_domains", []): - trust = "TRUSTED" - elif reg_domain in trust_cfg.get("insecure_domains", []): - trust = "INSECURE" - - # First Party: gleiche registrable Domain - if reg_domain == reg_page: - return { - "domain": domain, - "class": "FIRST_PARTY", - "registrable_domain": reg_domain, - "relation": "SUBDOMAIN", - "trust": trust - } - - # Klassifikation über Config-Gruppen - domain_groups = config.get("domains", {}) - - for group, entries in domain_groups.items(): - for entry in entries: - entry = entry.lower() - entry_reg = get_registrable_domain(entry) - if reg_domain == entry_reg: - return { - "domain": domain, - "class": group.upper(), - "registrable_domain": reg_domain, - "relation": "EXTERNAL", - "trust": trust - } - - return { - "domain": domain, - "class": "OTHER", - "registrable_domain": reg_domain, - "relation": "EXTERNAL", - "trust": trust - } diff --git a/app/compare.py b/app/compare.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/config.py b/app/config.py deleted file mode 100644 index 19fc774..0000000 --- a/app/config.py +++ /dev/null @@ -1,121 +0,0 @@ -from pathlib import Path -import tomllib - - -class ConfigError(Exception): - """ - Raised when the Altbrow configuration is invalid. - """ - pass - - -def load_toml(path: str = "config/altbrow.toml") -> dict: - """ - Load a TOML configuration file. - - Args: - path: Path to the TOML file. - - Returns: - Parsed configuration as a dictionary. - - Raises: - ConfigError: If the file cannot be loaded. - """ - - config_path = Path(path) - - if not config_path.exists(): - raise ConfigError(f"Config not found: {path}") - - with config_path.open("rb") as f: - config = tomllib.load(f) - - return config - - -def get_client_profile(config: dict, override: str | None) -> dict: - client_cfg = config.get("client", {}) - default = client_cfg.get("default_profile", "passive") - - profile_name = override or default - profiles = client_cfg.get("profiles", {}) - - if profile_name not in profiles: - raise ConfigError(f"Unknown client profile: {profile_name}") - - return profiles[profile_name] - - -def validate_altbrow_config(config: dict) -> str: - # --- meta --- - if "meta" not in config: - raise ConfigError("Missing [meta] section") - - if "version" not in config["meta"]: - raise ConfigError("Missing meta.version") - - version = config["meta"]["version"] - - # --- client --- - if "client" not in config: - raise ConfigError("Missing [client] section") - - client = config["client"] - - if "default_profile" not in client: - raise ConfigError("Missing client.default_profile") - - if "profiles" not in client: - raise ConfigError("Missing [client.profiles] section") - - default_profile = client["default_profile"] - profiles = client["profiles"] - - if default_profile not in profiles: - raise ConfigError( - f"Default profile '{default_profile}' not found in client.profiles" - ) - - # --- validation (optional but structured) --- - validation = config.get("validation", {}) - - if "microdata_vs_jsonld" in validation: - tolerance = validation["microdata_vs_jsonld"].get("tolerance") - if tolerance not in (None, "strict", "loose"): - raise ConfigError( - "validation.microdata_vs_jsonld.tolerance must be 'strict' or 'loose'" - ) - - output = config.get("output", {}) - explicit_format = output.get("explicit_format", "json") - - if explicit_format not in ("json", "yaml"): - raise ConfigError( - "output.explicit_format must be 'json' or 'yaml'" - ) - - if explicit_format == "yaml": - output_text = "explicit YAML" - else: - output_text = "explicit JSON" - - # --- description sentence --- - profile = profiles[default_profile] - use_session = profile.get("use_session", False) - headers = profile.get("headers", {}) - - activity = "active" if use_session else "passive" - consented = "with consented" if headers else "without consent" - - lines = [ - f"Altbrow reads with config {version} a HTTP URL " - f"{activity} {consented} and counts domains, cookies, html, jsonld, microdata.", - f"It writes {output_text} to STDOUT and not to file.", - f"It {'does' if 'microdata_vs_jsonld' in validation else 'does not'} " - "analyse for a comparison added to summary.", - "It does not log to ." - ] - sentence = "\n".join(lines) - - return sentence diff --git a/app/extract.py b/app/extract.py deleted file mode 100644 index 726c663..0000000 --- a/app/extract.py +++ /dev/null @@ -1,91 +0,0 @@ -from bs4 import BeautifulSoup -from urllib.parse import urlparse - -from extruct import extract -from w3lib.html import get_base_url - -from .classify_domain import classify_domain -from .classify_cookies import classify_cookies - -import logging -logger = logging.getLogger(__name__) - -def extract_data(fetch_result: dict, config: dict) -> dict: - html = fetch_result["html"] - final_url = fetch_result["final_url"] - headers = fetch_result["headers"] - - page_domain = urlparse(final_url).netloc - base_url = get_base_url(html, final_url) - soup = BeautifulSoup(html, "lxml") - - # ---------- Structured Data ---------- - structured = extract( - html, - base_url=base_url, - syntaxes=["json-ld", "microdata"], - ) - - # ---------- External Domains ---------- - external_domains = set() - - for tag in soup.find_all(["a", "img", "script", "link", "iframe"]): - url = tag.get("href") or tag.get("src") - if not url: - continue - parsed = urlparse(url) - if parsed.netloc and parsed.netloc != page_domain: - external_domains.add(parsed.netloc) - - classified_domains = [ - { - "domain": d, - # "class": classify_domain(d, page_domain, config), - "class": classify_domain(d, page_domain, config), - } - for d in sorted(external_domains) - ] - - # ---------- Cookies ---------- - raw_cookies = headers.get("Set-Cookie") - classified_cookies = [] - - - if raw_cookies: - try: - for raw in raw_cookies.split(","): - classified_cookies.append( - classify_cookies(raw.strip(), page_domain, config) - ) - except Exception as exc: - logger.warning("Cookie parsing failed: %s", exc) - - - return { - "structured_data": structured, - "signals": { - "external_domains": classified_domains, - "cookies": classified_cookies, - }, - } - -def extract_cookies(cookiejar, page_domain: str, config: dict) -> list[dict]: - cookies = [] - - for c in cookiejar: - cookies.append({ - "name": c.name, - "domain": c.domain.lstrip("."), - "path": c.path, - "secure": c.secure, - "httponly": c.has_nonstandard_attr("HttpOnly"), - "samesite": c.get_nonstandard_attr("SameSite"), - "expires": c.expires, - "class": classify_domain( - c.domain.lstrip("."), - page_domain, - config - ), - }) - - return cookies diff --git a/app/fetch.py b/app/fetch.py deleted file mode 100644 index 2dacc72..0000000 --- a/app/fetch.py +++ /dev/null @@ -1,62 +0,0 @@ -import logging -import requests - -logger = logging.getLogger(__name__) - - -def fetch_url(url: str, client_profile: dict) -> dict: - headers = client_profile.get("headers", {}) - use_session = client_profile.get("use_session", False) - - try: - if use_session: - session = requests.Session() - if headers: - session.headers.update(headers) - response = session.get(url, timeout=10) - cookies = session.cookies - else: - response = requests.get(url, headers=headers or None, timeout=10) - cookies = response.cookies - - response.raise_for_status() - - logger.debug( - "Fetched %s (status %s)", - response.url, - response.status_code - ) - - return { - "url": url, - "final_url": response.url, - "status_code": response.status_code, - "headers": dict(response.headers), - "encoding": response.encoding, - "html": response.text, - "cookies": cookies, - } - - except requests.exceptions.MissingSchema: - logger.error("Invalid URL (missing scheme): %s", url) - raise - - except requests.exceptions.Timeout: - logger.error("Timeout while fetching %s", url) - raise - - except requests.exceptions.ConnectionError as exc: - logger.error("Connection error for %s: %s", url, exc) - raise - - except requests.exceptions.HTTPError as exc: - logger.error( - "HTTP error %s for %s", - exc.response.status_code if exc.response else "?", - url - ) - raise - - except requests.exceptions.RequestException as exc: - logger.error("Request failed for %s: %s", url, exc) - raise diff --git a/app/main.py b/app/main.py deleted file mode 100644 index c6247a1..0000000 --- a/app/main.py +++ /dev/null @@ -1,76 +0,0 @@ -import argparse -import logging - -from .fetch import fetch_url -from .extract import extract_data -from .logging_config import setup_logging -from .config import load_toml, get_client_profile, ConfigError, validate_altbrow_config -from .output import render_output, write_log - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("url", nargs="?", help="URL to analyze") - parser.add_argument("--debug", action="store_true", help="Enable debug logging") - parser.add_argument( - "--output-mode", - choices=["silent", "summary", "explicit"], - default="summary", - help="Control result visibility", - ) - parser.add_argument("--log-file", help="Write full analysis result to file (JSON)") - parser.add_argument( - "--client-profile", - choices=["passive", "browser", "consented"], - help="HTTP client behavior profile (default: from config)", - ) - parser.add_argument( - "--validate-config", - action="store_true", - help="Validate altbrow.toml and exit" - ) - - args = parser.parse_args() - - setup_logging(debug=args.debug) - logger = logging.getLogger(__name__) - - try: - config = load_toml("config/altbrow.toml") - client_profile = get_client_profile(config, args.client_profile) - except ConfigError as exc: - logger.error("Config error: %s", exc) - return - - if args.validate_config: - try: - text = validate_altbrow_config(config) - print(text) - return - except ConfigError as exc: - logger.error("Config validation failed: %s", exc) - return - - if not args.url: - logger.error("URL is required") - return - - url = args.url - if not url.startswith(("http://", "https://")): - url = "https://" + url - - - try: - fetched = fetch_url(url, client_profile) - extracted = extract_data(fetched, config) - except Exception as exc: - logger.error("Analysis failed: %s", exc) - return - - render_output(extracted, args.output_mode, config) - - if args.log_file: - write_log(extracted, args.log_file) - -if __name__ == "__main__": - main() diff --git a/app/output.py b/app/output.py deleted file mode 100644 index f83bb33..0000000 --- a/app/output.py +++ /dev/null @@ -1,48 +0,0 @@ -# from pprint import pprint -import json - -try: - import yaml -except ImportError: - yaml = None - - -def render_output(extracted: dict, output_mode: str, config: dict) -> None: - if output_mode == "silent": - return - - if output_mode == "summary": - structured = extracted.get("structured_data", {}) - print("\n=== Summary ===") - print("External domains:", len(extracted["signals"]["external_domains"])) - print("Cookies:", len(extracted["signals"]["cookies"])) - print("JSON-LD blocks:", len(structured.get("json-ld", []))) - print("Microdata blocks:", len(structured.get("microdata", []))) - return - - if output_mode == "explicit": - fmt = config.get("output", {}).get("explicit_format", "json" ) - - if fmt == "json": - print(json.dumps(extracted, indent=2, ensure_ascii=False)) - return - - if fmt == "yaml": - if yaml is None: - raise RuntimeError("YAML output requested but PyYAML is not installed") - print( - yaml.safe_dump( - extracted, - sort_keys=False, - allow_unicode=True - ) - ) - return - - raise ValueError(f"Unknown explicit output format: {fmt}") - - - -def write_log(extracted: dict, path: str) -> None: - with open(path, "w", encoding="utf-8") as f: - json.dump(extracted, f, indent=2, ensure_ascii=False) diff --git a/config/altbrow.toml b/config/altbrow.toml deleted file mode 100644 index d7d5184..0000000 --- a/config/altbrow.toml +++ /dev/null @@ -1,39 +0,0 @@ -[meta] -source = "local" -version = "2025-02-08" - -[validation.linked_data] -max_depth = 2 -follow_same_domain_only = true -timeout = 3 - -[validation.schema_org] -allow_unknown_properties = false - -[validation.microdata_vs_jsonld] -tolerance = "strict" - -[client] -default_profile = "passive" - -[client.profiles.passive] -use_session = false -headers = {} - -[client.profiles.browser] -use_session = true - -[client.profiles.browser.headers] -"User-Agent" = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" -"Accept-Language" = "de-DE,de;q=0.9,en;q=0.8" - -[client.profiles.consented] -use_session = true - -[client.profiles.consented.headers] -"User-Agent" = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" -"Accept-Language" = "de-DE,de;q=0.9,en;q=0.8" - -[output] -explicit_format = "yaml" # or: "json" -# explicit_format = "json" # or: "yaml" diff --git a/config/domains.toml b/config/domains.toml deleted file mode 100644 index b1ad23f..0000000 --- a/config/domains.toml +++ /dev/null @@ -1,51 +0,0 @@ -[meta] -source = "local" -version = "2025-01" -sha256 = "abc123..." - - - -[domains] - -cdn = [ - "cloudflare.com", - "jsdelivr.net", - "unpkg.com", - "akamai.net" -] - -analytics = [ - "google-analytics.com", - "gtagmanager.com", - "matomo.org" -] - -social = [ - "facebook.com", - "twitter.com", - "linkedin.com", - "instagram.com" -] - - - -[claims] -require_sameAs_for = ["ANALYTICS", "SOCIAL"] -allowed_external_classes = ["CDN", "FONT"] - -[comparison] -ignore_domains = ["localhost", "127.0.0.1"] - -[trust] -trusted_domains = [ - "br.de", - "ard.de", - "gedankenfalle.de", - "tmp.gedankenfalle.de" -] - - -insecure_domains = [ - "doubleclick.net", - "scorecardresearch.com" -] \ No newline at end of file diff --git a/docs/docs/api.md b/docs/docs/api.md new file mode 100644 index 0000000..515be53 --- /dev/null +++ b/docs/docs/api.md @@ -0,0 +1,25 @@ +# API Help + + +``` + +usage: altbrow [-h] [-V] [--config CONFIG] [-o OUTPUT] [-f {text,yaml,json}] [-v] [--client-profile {passive,browser,consented}] [--validate-config] [--build-cache] [url] + +positional arguments: + url URL to analyze + +options: + -h, --help show this help message and exit + -V, --version show program's version number and exit + --config CONFIG Path to configuration file (TOML) + -o OUTPUT, --output OUTPUT + Write result to file + -f {text,yaml,json}, --format {text,yaml,json} + Output format (default: text) + -v, --verbose Increase text detail level (-vv, -vvv) + --client-profile {passive,browser,consented} + HTTP client behavior profile (default: from config) + --validate-config Validate altbrow.toml and exit + --build-cache (Re)build provider cache DB and exit + +``` \ No newline at end of file diff --git a/docs/docs/config.md b/docs/docs/config.md new file mode 100644 index 0000000..fad6d0c --- /dev/null +++ b/docs/docs/config.md @@ -0,0 +1,16 @@ +# API Config of Altbrow + + +::: altbrow.config.ConfigError + +::: altbrow.config.load_toml + +::: altbrow.config.get_client_profile + +::: altbrow.config.validate_altbrow_config + + +## altbrow.toml + + +## domains.toml diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..3e147e4 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,23 @@ +# Altbrow + +Developer documentation. + + + +## Autodoc with mkdocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +### Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs -h` - Print help message and exit. + +### Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 0000000..036b256 --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,13 @@ +site_name: Altbrow + +nav: + - Home: index.md + - Configuration API: api.md + - Configuration TOML: config.md + +plugins: + - mkdocstrings: + handlers: + python: + options: + docstring_style: google diff --git a/pyproject.toml b/pyproject.toml index b78b94a..82a427a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,16 @@ [project] name = "altbrow" -version = "0.1.1" description = "An alternative, crawler-like browser that looks beneath the surface of the semantic web." readme = "README.md" requires-python = ">=3.12" +dynamic = ["version"] authors = [ { name = "Electrical Smile", email = "esmile@posteo.net" } ] -license = { text = "GPL-2.0-or-later" } +license = "GPL-3.0-or-later" +# license = { text = "GPL-2.0-or-later" } # license = { file = "LICENSE" } dependencies = [ @@ -19,15 +20,19 @@ dependencies = [ "extruct", "rdflib", "tldextract", - "PyYAML" + "PyYAML", + "dnspython", + "maxminddb" ] [project.optional-dependencies] dev = [ + "build", "ruff", "pytest", "mkdocs", - "mkdocstrings[python]" + "mkdocstrings[python]", + "pip-tools" ] [project.urls] @@ -39,11 +44,15 @@ Source = "https://github.com/gesmile/altbrow" # -------------------- [build-system] -requires = ["setuptools>=61.0"] +requires = ["setuptools>=64", "setuptools-scm"] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] +version_scheme = "post-release" +local_scheme = "no-local-version" + [tool.setuptools.packages.find] -include = ["app"] +include = ["altbrow"] [tool.ruff] line-length = 88 @@ -58,4 +67,4 @@ line_length = 88 typeCheckingMode = "basic" [project.scripts] -altbrow = "app.main:main" +altbrow = "altbrow.main:main" \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7b6cb0f --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,167 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --extra=dev --output-file=requirements-dev.txt pyproject.toml +# +beautifulsoup4==4.14.3 + # via + # altbrow (pyproject.toml) + # mf2py +build==1.4.3 + # via altbrow (pyproject.toml) +certifi==2026.2.25 + # via requests +charset-normalizer==3.4.7 + # via requests +click==8.3.2 + # via mkdocs +colorama==0.4.6 + # via + # build + # click + # mkdocs + # pytest +dnspython==2.8.0 + # via altbrow (pyproject.toml) +extruct==0.18.0 + # via altbrow (pyproject.toml) +filelock==3.25.2 + # via tldextract +ghp-import==2.1.0 + # via mkdocs +griffelib==2.0.2 + # via mkdocstrings-python +html-text==0.7.1 + # via extruct +html5lib==1.1 + # via + # mf2py + # pyrdfa3 +idna==3.11 + # via + # requests + # tldextract +iniconfig==2.3.0 + # via pytest +jinja2==3.1.6 + # via + # mkdocs + # mkdocstrings +jstyleson==0.0.2 + # via extruct +lxml==6.0.4 + # via + # altbrow (pyproject.toml) + # extruct + # html-text + # lxml-html-clean +lxml-html-clean==0.4.4 + # via + # extruct + # html-text +markdown==3.10.2 + # via + # mkdocs + # mkdocs-autorefs + # mkdocstrings + # pymdown-extensions +markupsafe==3.0.3 + # via + # jinja2 + # mkdocs + # mkdocs-autorefs + # mkdocstrings +maxminddb==3.1.1 + # via altbrow (pyproject.toml) +mergedeep==1.3.4 + # via + # mkdocs + # mkdocs-get-deps +mf2py==2.0.1 + # via extruct +mkdocs==1.6.1 + # via + # altbrow (pyproject.toml) + # mkdocs-autorefs + # mkdocstrings +mkdocs-autorefs==1.4.4 + # via + # mkdocstrings + # mkdocstrings-python +mkdocs-get-deps==0.2.2 + # via mkdocs +mkdocstrings[python]==1.0.3 + # via + # altbrow (pyproject.toml) + # mkdocstrings-python +mkdocstrings-python==2.0.3 + # via mkdocstrings +packaging==26.0 + # via + # build + # mkdocs + # pytest +pathspec==1.0.4 + # via mkdocs +platformdirs==4.9.6 + # via mkdocs-get-deps +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pymdown-extensions==10.21.2 + # via mkdocstrings +pyparsing==3.3.2 + # via rdflib +pyproject-hooks==1.2.0 + # via build +pyrdfa3==3.6.5 + # via extruct +pytest==9.0.3 + # via altbrow (pyproject.toml) +python-dateutil==2.9.0.post0 + # via ghp-import +pyyaml==6.0.3 + # via + # altbrow (pyproject.toml) + # mkdocs + # mkdocs-get-deps + # pymdown-extensions + # pyyaml-env-tag +pyyaml-env-tag==1.1 + # via mkdocs +rdflib==7.6.0 + # via + # altbrow (pyproject.toml) + # extruct + # pyrdfa3 +requests==2.33.1 + # via + # altbrow (pyproject.toml) + # mf2py + # pyrdfa3 + # requests-file + # tldextract +requests-file==3.0.1 + # via tldextract +ruff==0.15.10 + # via altbrow (pyproject.toml) +six==1.17.0 + # via + # html5lib + # python-dateutil +soupsieve==2.8.3 + # via beautifulsoup4 +tldextract==5.3.1 + # via altbrow (pyproject.toml) +typing-extensions==4.15.0 + # via beautifulsoup4 +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via extruct +watchdog==6.0.0 + # via mkdocs +webencodings==0.5.1 + # via html5lib diff --git a/requirements.txt b/requirements.txt index 2218d2f..9b8a838 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile pyproject.toml +# pip-compile --output-file=requirements.txt pyproject.toml # beautifulsoup4==4.14.3 # via @@ -12,6 +12,8 @@ certifi==2026.1.4 # via requests charset-normalizer==3.4.4 # via requests +dnspython==2.8.0 + # via altbrow (pyproject.toml) extruct==0.18.0 # via altbrow (pyproject.toml) filelock==3.20.3 @@ -38,6 +40,8 @@ lxml-html-clean==0.4.3 # via # extruct # html-text +maxminddb==3.1.1 + # via altbrow (pyproject.toml) mf2py==2.0.1 # via extruct pyparsing==3.3.2 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0a69065 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,39 @@ +# tests/conftest.py +# +# Shared pytest fixtures for altbrow integration tests. +# Provides a local HTTP server serving test HTML pages from tests/data/. + +import threading +import pytest + +from http.server import HTTPServer, SimpleHTTPRequestHandler +from pathlib import Path + +TEST_DATA_DIR = Path(__file__).parent / "data" +MOCK_HOST = "127.0.0.1" +MOCK_PORT = 8080 +MOCK_BASE_URL = f"http://{MOCK_HOST}:{MOCK_PORT}" + + +class _SilentHandler(SimpleHTTPRequestHandler): + """SimpleHTTPRequestHandler with logging suppressed.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(TEST_DATA_DIR), **kwargs) + + def log_message(self, fmt, *args): + pass # suppress request logs in test output + + +@pytest.fixture(scope="session") +def mock_server(): + """Start a local HTTP server serving tests/data/ for the test session. + + Returns: + Base URL string e.g. 'http://127.0.0.1:8080' + """ + server = HTTPServer((MOCK_HOST, MOCK_PORT), _SilentHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield MOCK_BASE_URL + server.shutdown() diff --git a/tests/data/altbrow.toml b/tests/data/altbrow.toml new file mode 100644 index 0000000..f8af11d --- /dev/null +++ b/tests/data/altbrow.toml @@ -0,0 +1,58 @@ +[meta] +source = "local" +version = "2025-02-08" +use-provider = true + + +[output] +explicit_format = "yaml" # or: "json" + +[validation.linked_data] +max_depth = 2 +follow_same_domain_only = true +timeout = 3 + +[validation.schema_org] +allow_unknown_properties = false + +[validation.microdata_vs_jsonld] +tolerance = "strict" + + +[client] +profile = "passive" + +# ********************** +# * profile definition * +# ********************** + +[client.defaults] +follow_redirects = true # follow HTTP 301/302 +timeout = 10 # seconds +fetch_subresources = 0 # raw URL, no external javascript, css +use_session = false +use_header = false +accept_cookies = false +check_cert = true + +[client.profiles.passive] +# inherits all defaults — no overrides + +[client.profiles.browser] +use_session = true +use_header = true + +[client.profiles.consented] +use_session = true +use_header = true +accept_cookies = true +fetch_subresources = 1 + + +# Header definition needs periodic update to appear as a normal browser. +# Applied to all profiles where use_header = true. + +[client.headers] +"User-Agent" = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36" +"Accept-Language" = "de-DE,de;q=0.9,en;q=0.8" +# "referer" = "Referer: https://www.google.com/" diff --git a/tests/data/fail2ban.txt b/tests/data/fail2ban.txt new file mode 100644 index 0000000..289425b --- /dev/null +++ b/tests/data/fail2ban.txt @@ -0,0 +1,109 @@ +# provider: local +# source_updated: 2026-03-05T22:00:02 +# total_entries: 100 +# name: fail2ban export +# source: sshd +# created: 2026-03-05T19:10:00Z +# category: suspicious-ip +# description: ssh brute force bans + +104.248.148.195 +106.13.85.199 +107.170.19.211 +107.170.56.120 +115.190.238.64 +118.193.33.74 +119.192.55.89 +122.224.109.51 +129.212.178.102 +134.122.44.83 +134.199.155.98 +134.199.158.180 +134.199.165.187 +134.199.173.206 +134.209.158.246 +134.209.190.73 +134.209.23.172 +134.209.243.253 +134.209.93.72 +137.184.133.118 +137.184.163.254 +137.184.82.33 +138.197.134.165 +138.197.158.92 +138.197.163.192 +138.197.47.187 +138.197.77.107 +138.68.135.6 +138.68.144.81 +138.68.164.103 +138.68.185.126 +139.59.13.44 +139.59.29.58 +139.59.47.236 +139.59.5.226 +139.59.62.177 +139.59.69.67 +139.59.9.137 +14.22.82.116 +142.93.134.255 +142.93.159.3 +142.93.165.197 +142.93.193.238 +142.93.219.82 +142.93.226.30 +142.93.34.44 +143.110.169.194 +143.110.191.86 +143.198.111.15 +143.198.14.52 +143.198.212.85 +143.198.223.229 +143.198.74.76 +143.244.179.58 +143.244.186.2 +144.126.207.37 +144.126.226.139 +146.190.22.175 +146.190.236.131 +146.190.35.99 +146.190.37.79 +147.182.199.159 +157.230.105.14 +157.230.124.135 +157.245.106.61 +157.245.119.185 +157.245.124.13 +157.245.37.166 +157.245.66.71 +159.203.190.186 +159.203.71.155 +159.203.76.70 +159.223.0.189 +159.223.1.188 +159.223.13.92 +159.223.14.55 +159.223.217.86 +159.223.58.124 +159.223.66.92 +159.224.132.77 +159.65.1.155 +159.65.158.102 +159.65.199.140 +159.65.56.166 +159.65.60.221 +159.65.82.44 +159.65.84.73 +159.89.162.109 +159.89.166.110 +159.89.171.4 +159.89.206.142 +159.89.53.142 +159.89.97.153 +161.35.147.57 +161.35.173.104 +161.35.214.155 +161.35.34.29 +161.35.38.153 +161.35.42.99 +162.19.170.81 diff --git a/tests/data/ipfire-ads.txt b/tests/data/ipfire-ads.txt new file mode 100644 index 0000000..88f6669 --- /dev/null +++ b/tests/data/ipfire-ads.txt @@ -0,0 +1,123 @@ +############################################################################### +# T E S T Domain Blocklist for IPFire +############################################################################### +# +# This file contains domains that are part of the official IPFire Domain +# Blocklist Project. It is intended for use with DNS servers and firewalls +# that support blocklisting of known malicious or unwanted domains. +# +# List : Advertising +# +# Blocks domains used for ads, tracking, and ad delivery +# +# License : CC BY-SA 4.0 +# Updated : 2026-03-05T22:00:02.606195+00:00 +# Total Entries : 100 +# +# Copyright (C) 2026 - IPFire Team +# +# For more information or to contribute: +# https://dbl.ipfire.org/ +# +############################################################################### + +186576.tlnk.io +186.6.87.194.dynamic.dol.ru +186781.measurementapi.com +186789.measurementapi.com +18.6.87.194.dynamic.dol.ru +186977.measurementapi.com +187587.measurementapi.com +187.6.87.194.dynamic.dol.ru +188617.measurementapi.com +188.6.87.194.dynamic.dol.ru +188941.measurementapi.com +189137.measurementapi.com +189229.measurementapi.com +189333.measurementapi.com +1s7vmel6xi.execute-api.us-east-1.amazonaws.com +1sb.illusionsplasticsurgery.com +1.skimresources.com +1sputnik.ru +1st-iklan.com +1stpool.com +1streamline.go2cloud.org +1to1.bbva.com +1traff.ru +1traf.ru +1txt.ru +1ul8dvwz0a.s.ad6media.fr +1va3vpbjxq.kameleoon.eu +1vhrgv0dy5bwe5a7.kdpwizard.app +1vp67nxi0d.kameleoon.eu +1vyt1eguj27.ommasign.com +1web.me +1wljgi8eyy.kameleoon.eu +1x1.a-mo.net +1x1rank.com +1xbet.com +edjsl.hierbasorganicas.com.mx +ed.koeln.de +ed.large.be +ed.large.nl +edmanalytics.pandahall.com +edmodo-d.openx.net +edm.pressflex.net +ednplus.com +edococounter.de +edomz.net +e-dot.hut1.ru +edrone.me +edrta.mol.im +eds.ca.matchbox.maruhub.com +edskes.com.site-id.nl +edt02.net +edtp.de +ed.tripledart.com +e.dtscout.com +edua29146y.com +eduardo.eduardofurtadog.com.br +educate.digital-launchpad.com +educate.monetise.com +education.tubemogul.com +eduesse-d.openx.net +edu.retireone.com +ftps.com +eftrk.legalclaimassistant.com +efzuf.revscale.com +egamiplatform.tv +egclbw.footcarelab.co.kr +egdeb.slaterockautomation.com +egenberger-analytics.stryzmedia.de +egencia-d.openx.net +e-generator.com +egeszsegespont.hu +e.getjibjab.com +egger01.webtrekk.net +eggplant.cloud +eghuntsrizvbt.com +eghyahl.cn +egifter-offers.evyy.net +e.glbimg.comvmdwuq.chadotel.com +vmes.vertamedia.com +vmi.datalliance.net +vm-matomo.occitanie-en-scene.fr +vmmmye.defender-usa.com +vmmpxl.com +vmm-satellite1.com +vmonitor.ws.netease.com +vmp.boldchat.com +vmsbe.kassazaak.be +vms.kassazaak.nl +vms.kassensystemevergleich.com +vms.laadpalenwijzer.be +vms.pos-software.co.uk +vmss-clarity-ingest-eus2.eastus2.cloudapp.azure.com +vmss-clarity-tag-eus2.eastus2.cloudapp.azure.com +vmsst.j4ksports.co.uk +vmssts.skyenergy.com.au +vmszxzvk.mysecondchancechurch.com +vmtg.iesve.com +vmt.londondentalinstitute.com +vmtp.boxrstore.com +vmtrk.com diff --git a/tests/data/provider.toml b/tests/data/provider.toml new file mode 100644 index 0000000..25ca073 --- /dev/null +++ b/tests/data/provider.toml @@ -0,0 +1,367 @@ +# provider.toml is only used when in ./altbrow.toml is set: `meta.use-provider = true` + +[meta] +version = 2 +created = "2026-04-13" + +# Local (file) Provider + +[provider.fail2ban] + +location = "local" +type = "ip" +enabled = false + +[[provider.fail2ban.category]] +mapping = ["suspicious"] +source = ["./fail2ban.txt"] + + +# --------------------------------------------------------------------------- +# Inline Domain Providers +# --------------------------------------------------------------------------- + +[provider.infrastructure] +location = "inline" +type = "domain" +enabled = true + +[[provider.infrastructure.category]] +name = "Semantic Web Standards" +enabled = true +mapping = ["infrastructure"] +source = [ + "schema.org", + "schema.googleapis.com", + "w3.org", + "w3c.org", + "purl.org", + "xmlns.com", + "rdf.data-vocabulary.org", + "ogp.me", + "dublincore.org", + "json-ld.org", +] + +[[provider.infrastructure.category]] +name = "Web Standards Bodies" +enabled = true +mapping = ["infrastructure"] +source = [ + "iana.org", + "mozilla.org", + "whatwg.org", +] + +[provider.cdn] +location = "inline" +type = "domain" +enabled = true + +[[provider.cdn.category]] +name = "Major CDN" +enabled = true +mapping = ["cdn"] +source = [ + "cdnjs.cloudflare.com", + "cdn.cloudflare.com", + "akamai.net", + "akamaiedge.net", + "akamaized.net", + "edgesuite.net", + "fastly.net", + "fastlylb.net", + "cloudfront.net", + "amazonaws.com", + "gstatic.com", + "azureedge.net", + "msecnd.net", + "jsdelivr.net", + "unpkg.com", + "bootstrapcdn.com", + "stackpathcdn.com", + "b-cdn.net", + "kxcdn.com", +] + +[provider.analytics] +location = "inline" +type = "domain" +enabled = true + +[[provider.analytics.category]] +name = "Web Analytics" +enabled = true +mapping = ["analytics"] +source = [ + "google-analytics.com", + "googletagmanager.com", + "googleadservices.com", + "cloudflareinsights.com", + "matomo.org", + "plausible.io", + "fathom.com", + "segment.com", + "mixpanel.com", + "amplitude.com", + "hotjar.com", + "clarity.ms", + "fullstory.com", + "logrocket.com", +] + +[[provider.analytics.category]] +name = "Error and Performance Monitoring" +enabled = true +mapping = ["telemetry"] +source = [ + "sentry.io", + "bugsnag.com", + "rollbar.com", + "newrelic.com", + "datadoghq.com", + "elastic.co", + "dynatrace.com", + "appdynamics.com", +] + +[provider.tracking] +location = "inline" +type = "domain" +enabled = true + +[[provider.tracking.category]] +name = "Social Tracking Pixels" +enabled = true +mapping = ["tracking"] +source = [ + "facebook.net", + "connect.facebook.net", + "analytics.twitter.com", + "t.co", + "snapchat.com", + "sc-static.net", + "ads.pinterest.com", + "licdn.com", +] + +[[provider.tracking.category]] +name = "Ad Network Tracking" +enabled = true +mapping = ["tracking"] +source = [ + "bat.bing.com", + "taboola.com", + "outbrain.com", + "criteo.com", + "adroll.com", + "quantserve.com", + "scorecardresearch.com", + "bluekai.com", + "zemanta.com", + "doubleclick.net", +] + + +[provider.ads] +location = "inline" +type = "domain" +enabled = true + +[[provider.ads.category]] +name = "Ad Delivery" +enabled = true +mapping = ["ads"] +source = [ + "googlesyndication.com", + "googleadservices.com", + "doubleclick.net", + "amazon-adsystem.com", + "media.net", + "moatads.com", + "adsrvr.org", + "advertising.com", + "adnxs.com", + "rubiconproject.com", + "pubmatic.com", + "openx.net", + "smartadserver.com", +] + + + + +[provider.inlineip] +location = "inline" +type = "ip" +enabled = true + +[[provider.inlineip.category]] +name = "RFC1918" +enabled = true +mapping = ["local"] +source = [ + "192.168.0.0/16", + "10.0.0.0/8", + "172.16.0.0/12", +] + +[[provider.inlineip.category]] +name = "localhost" +enabled = true +mapping = ["local"] +source = [ + "127.0.0.0/8", + "::1/128" +] + +[[provider.inlineip.category]] +name = "multicast" +enabled = true +mapping = ["infrastructure"] +source = [ + "224.0.0.0/4", + "169.254.0.0/16", + "ff00::/8", + "fe80::/10", + "255.255.255.255/32", +] + +[[provider.inlineip.category]] +name = "carrier-grade-nat" +enabled = true +mapping = ["infrastructure"] +source = [ + "100.64.0.0/10", +] + + +[[provider.inlineip.category]] +name = "Example inline IP" +enabled = true +mapping = ["suspicious"] +source = [ + "2.57.122.210", + "46.101.74.113", + "81.192.46.45", + "92.118.39.56", + "92.118.39.72", + "92.118.39.76", + "102.88.137.80", + "118.193.36.205", + "162.223.91.130", + "193.32.162.151", + "197.5.145.102" +] + +# --------------------------------------------------------------------------- +# Remote Provider +# --------------------------------------------------------------------------- + +[provider.ipfire] +name = "IPFire" +location = "remote" +type = "domain" +enabled = false + +[[provider.ipfire.category]] +name = "Advertising" +enabled = false +tier = 3 +mapping = ["ads"] +source = ["https://dbl.ipfire.org/lists/ads/domains.txt"] + +[[provider.ipfire.category]] +name = "DNS-over-HTTPS" +enabled = false +mapping = ["telemetry"] +source = ["https://dbl.ipfire.org/lists/doh/domains.txt"] + +[[provider.ipfire.category]] +name = "Malware" +tier = 1 +enabled = true +mapping = ["malware"] +source = ["https://dbl.ipfire.org/lists/malware/domains.txt"] + +[[provider.ipfire.category]] +name = "Phishing" +tier = 1 +enabled = true +mapping = ["malware"] +source = ["https://dbl.ipfire.org/lists/phishing/domains.txt"] + +[[provider.ipfire.category]] +name = "Social Networks" +enabled = false +mapping = ["social"] +source = ["https://dbl.ipfire.org/lists/social/domains.txt"] + +[provider.mock-domain] + +location = "remote" +type = "domain" +enabled = true + +[[provider.mock-domain.category]] +enabled = true +mapping = ["ads"] +source = ["http://localhost:8080/ipfire-ads.txt"] + + +[provider.mock-ip] + +location = "remote" +type = "ip" +enabled = true + +[[provider.mock-ip.category]] +enabled = true +mapping = ["suspicious"] +source = ["http://localhost:8080/fail2ban.txt"] + + +# --------------------------------------------------------------------------- +# DNS Provider +# --------------------------------------------------------------------------- + +[provider.opendns] +name = "OpenDNS" +location = "dns" +type = "domain" +enabled = false + +[[provider.opendns.category]] +name = "Malware/Phishing" +mapping = ["malware"] +source = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"] +sinkhole = [ + "146.112.61.104", "146.112.61.105", "146.112.61.107", "146.112.61.108", + "::ffff:146.112.61.104", "::ffff:146.112.61.105", + "::ffff:146.112.61.107", "::ffff:146.112.61.108", +] + +[[provider.opendns.category]] +name = "Content/Adult" +mapping = ["social"] +source = ["208.67.222.123", "208.67.220.123"] +sinkhole = ["146.112.61.106", "::ffff:146.112.61.106"] + +[[provider.opendns.category]] +name = "Suspicious/DNS Tunneling" +enabled = false +mapping = ["suspicious"] +source = ["208.67.222.222", "208.67.220.220", "2620:119:35::35", "2620:119:53::53"] +sinkhole = ["146.112.61.110", "::ffff:146.112.61.110"] + + +[provider.pihole] +location = "dns" +type = "domain" +enabled = false + +[[provider.pihole.category]] +name = "PiHole local" +mapping = ["ads"] +source = ["192.168.178.2"] +sinkhole = ["0.0.0.0", "::", "::ffff:0.0.0.0"] diff --git a/tests/data/test.html b/tests/data/test.html new file mode 100644 index 0000000..3f0e022 --- /dev/null +++ b/tests/data/test.html @@ -0,0 +1,70 @@ + + + + + altbrow test page + + + + + + + + + + + + + + + + + + + + +

altbrow integration test page

+

This page contains known domains and IPs for testing provider classification.

+ + + Instagram + Facebook + + + Internal admin (RFC1918) + Router (RFC1918) + + + Suspicious IP + + + Unknown domain + + + Schema.org + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..cc2a35f --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,101 @@ +# tests/test_cache.py + +import pytest +from pathlib import Path +from altbrow.cache import get_or_build_cache, lookup_domain, lookup_ip + + +DATA_DIR = Path(__file__).parent / "data" +CONFIG_PATH = DATA_DIR / "altbrow.toml" +CACHE_PATH = DATA_DIR / ".altbrow.cache" + + +@pytest.fixture(scope="module") +def cache(tmp_path_factory): + """Build cache once for all tests in this module.""" + import tomllib + from altbrow.config import load_provider_config, load_toml + + cache_path = tmp_path_factory.mktemp("cache") / ".altbrow.cache" + config = load_toml(CONFIG_PATH) + provider = load_provider_config(CONFIG_PATH, config) + get_or_build_cache(cache_path, provider, CONFIG_PATH) + return cache_path + + +def test_cache_exists(cache): + """Cache file must exist after build.""" + assert cache.exists() + assert cache.stat().st_size > 0 + + +def test_lookup_infrastructure(cache): + """schema.org must be classified as infrastructure.""" + results = lookup_domain("schema.org", cache) + categories = [r["category"] for r in results] + assert "infrastructure" in categories + + +def test_lookup_cdn(cache): + """cdnjs.cloudflare.com must be classified as cdn.""" + results = lookup_domain("cdnjs.cloudflare.com", cache) + categories = [r["category"] for r in results] + assert "cdn" in categories + + +def test_lookup_analytics(cache): + """google-analytics.com must be classified as analytics.""" + results = lookup_domain("google-analytics.com", cache) + categories = [r["category"] for r in results] + assert "analytics" in categories + + +def test_lookup_multiple_categories(cache): + """doubleclick.net must match both tracking and ads.""" + results = lookup_domain("doubleclick.net", cache) + categories = [r["category"] for r in results] + assert "tracking" in categories + assert "ads" in categories + + +def test_lookup_subdomain_match(cache): + """Subdomain of known domain matches via registrable domain.""" + results = lookup_domain("sub.google-analytics.com", cache) + categories = [r["category"] for r in results] + assert "analytics" in categories + + +def test_lookup_unknown_domain(cache): + """Unknown domain returns empty list.""" + results = lookup_domain("totally-unknown-xyz123.example", cache) + assert results == [] + + +def test_lookup_ip_exact(cache): + """Exact IP match against RFC1918.""" + results = lookup_ip("192.168.1.1", cache) + categories = [r["category"] for r in results] + assert "local" in categories + + +def test_lookup_ip_cidr(cache): + """IP within CIDR block matches.""" + results = lookup_ip("10.0.0.1", cache) + categories = [r["category"] for r in results] + assert "local" in categories + + +def test_lookup_ip_unknown(cache): + """Public IP with no match returns empty list.""" + results = lookup_ip("1.2.3.4", cache) + assert results == [] + + +def test_no_duplicates(cache): + """Each (category, provider) pair appears only once per domain.""" + results = lookup_domain("doubleclick.net", cache) + seen = set() + for r in results: + key = (r["category"], r["provider"]) + assert key not in seen, f"Duplicate entry: {key}" + seen.add(key) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..a8f0bcf --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,11 @@ +# tests/test_cli.py +import sys +import subprocess + +def test_cli_validate(): + result = subprocess.run( + [sys.executable, "-m", "altbrow", "--validate-config"], + capture_output=True, + text=True, + ) + assert result.returncode == 0 diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 0000000..f71b979 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,4 @@ +# tests/test_import.py +def test_import(): + import altbrow + assert altbrow is not None diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..e79cd40 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,117 @@ +# tests/test_integration.py +# +# Integration tests for altbrow domain/IP classification. +# Uses the mock HTTP server from conftest.py serving tests/data/test.html. + +import pytest +from pathlib import Path + + +def _get_domains(extracted: dict) -> dict[str, dict]: + """Return external_domains as {value: result_dict}.""" + return { + d["value"]: d + for d in extracted.get("signals", {}).get("external_domains", []) + } + + +def _categories(domain_result: dict) -> list[str]: + """Return list of category names for a domain result.""" + return [c["category"] for c in domain_result.get("categories", [])] + + +@pytest.fixture(scope="module") +def extracted(mock_server, tmp_path_factory): + """Fetch and extract test.html from mock server.""" + from altbrow.fetch import fetch_url + from altbrow.extract import extract_data + from altbrow.cache import get_or_build_cache + from altbrow.config import load_toml, get_client_profile, load_provider_config + + # minimal config — no provider, no geoip + config = { + "meta": {"version": 1, "created": "2026-01-01", "use-provider": False}, + "client": { + "profile": "passive", + "defaults": { + "follow_redirects": True, "timeout": 5, + "fetch_subresources": 0, "use_session": False, + "use_header": False, "accept_cookies": False, "check_cert": True, + }, + "profiles": {"passive": {}}, + }, + "output": {"explicit_format": "json"}, + } + config["provider"] = False + + # use temp cache + cache_path = tmp_path_factory.mktemp("cache") / ".altbrow.cache" + get_or_build_cache(cache_path, None, Path(".")) + + client_profile = config["client"]["defaults"] + url = f"{mock_server}/test.html" + + fetched = fetch_url(url, client_profile) + extracted = extract_data(fetched, cache_path, config) + return extracted + + +# --------------------------------------------------------------------------- +# Basic structure +# --------------------------------------------------------------------------- + +def test_extracted_has_signals(extracted): + assert "signals" in extracted + + +def test_external_domains_present(extracted): + domains = extracted["signals"]["external_domains"] + assert len(domains) > 0 + + +# --------------------------------------------------------------------------- +# Domain classification — without provider (unknown expected) +# --------------------------------------------------------------------------- + +def test_google_analytics_present(extracted): + domains = _get_domains(extracted) + assert "www.google-analytics.com" in domains or "google-analytics.com" in domains + + +def test_facebook_present(extracted): + domains = _get_domains(extracted) + assert any("facebook" in k for k in domains) + + +def test_unknown_domain_present(extracted): + domains = _get_domains(extracted) + assert any("example-unknown-domain-xyz" in k for k in domains) + + +# --------------------------------------------------------------------------- +# IP classification — RFC1918 IPs should appear as external_ips or in domains +# --------------------------------------------------------------------------- + +def test_rfc1918_ip_in_output(extracted): + """RFC1918 IPs linked directly should appear in external_ips or domains.""" + signals = extracted["signals"] + ip_values = [ip["value"] for ip in signals.get("external_ips", [])] + domain_values = [d["value"] for d in signals.get("external_domains", [])] + all_values = ip_values + domain_values + # 10.0.0.1 or 192.168.1.1 should appear + assert any(v.startswith("10.") or v.startswith("192.168.") for v in all_values) + + +# --------------------------------------------------------------------------- +# JSON-LD +# --------------------------------------------------------------------------- + +def test_jsonld_detected(extracted): + jsonld = extracted.get("structured_data", {}).get("json-ld", []) + assert len(jsonld) > 0 + + +def test_jsonld_type_webpage(extracted): + jsonld = extracted.get("structured_data", {}).get("json-ld", []) + types = [b.get("@type") for b in jsonld] + assert "WebPage" in types diff --git a/tests/test_provider_config.py b/tests/test_provider_config.py new file mode 100644 index 0000000..c745fe2 --- /dev/null +++ b/tests/test_provider_config.py @@ -0,0 +1,44 @@ +import tomllib +import re +from pathlib import Path +from altbrow.config import ALLOWED_MAPPINGS + +DOMAIN_RE = re.compile(r"^[a-z0-9.-]+\.[a-z]{2,}$") + +PROVIDER_TOML = Path(__file__).parent / "data" / "provider.toml" + + +def test_provider_config_valid(): + """Validate structure and mapping values of all providers in provider.toml.""" + config = tomllib.loads(PROVIDER_TOML.read_text()) + assert "provider" in config + + for name, provider in config["provider"].items(): + assert "location" in provider, f"Provider '{name}' missing 'location'" + assert "type" in provider, f"Provider '{name}' missing 'type'" + assert "enabled" in provider, f"Provider '{name}' missing 'enabled'" + + if "category" not in provider: + continue + + for cat in provider.get("category", []): + assert isinstance(cat["mapping"], list), \ + f"Provider '{name}' mapping must be a list" + assert all(m in ALLOWED_MAPPINGS for m in cat["mapping"]), \ + f"Provider '{name}' unknown mapping: {cat['mapping']}" + + +def test_inline_domains(): + """Validate domain syntax for all inline domain providers.""" + config = tomllib.loads(PROVIDER_TOML.read_text()) + + for name, provider in config["provider"].items(): + if provider.get("location") != "inline": + continue + if provider.get("type") != "domain": + continue + + for cat in provider.get("category", []): + for d in cat.get("source", []): + assert DOMAIN_RE.match(d), \ + f"Provider '{name}' invalid domain '{d}'" \ No newline at end of file diff --git a/tests/test_resolve.py b/tests/test_resolve.py new file mode 100644 index 0000000..33bece1 --- /dev/null +++ b/tests/test_resolve.py @@ -0,0 +1,131 @@ +# tests/test_resolve_config.py +# +# Tests for [resolve] section validation in provider.toml + +import pytest +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from altbrow.config import validate_provider_config, RESOLVE_DEFAULTS, ConfigError + +BASE_CFG = { + "meta": {"version": 1, "created": "2026-01-01"}, + "provider": {}, +} + + +def cfg_with_resolve(resolve: dict) -> dict: + return {**BASE_CFG, "resolve": resolve} + + +# --------------------------------------------------------------------------- +# RESOLVE_DEFAULTS +# --------------------------------------------------------------------------- + +def test_resolve_defaults_keys(): + assert "resolve-domains" in RESOLVE_DEFAULTS + assert "resolver" in RESOLVE_DEFAULTS + assert "resolver-timeout" in RESOLVE_DEFAULTS + + +def test_resolve_defaults_values(): + assert RESOLVE_DEFAULTS["resolve-domains"] is False + assert RESOLVE_DEFAULTS["resolver"] == ["os"] + assert RESOLVE_DEFAULTS["resolver-timeout"] == 2 + + +# --------------------------------------------------------------------------- +# Missing [resolve] section — should pass with defaults +# --------------------------------------------------------------------------- + +def test_no_resolve_section(): + validate_provider_config(BASE_CFG) # no exception + + +# --------------------------------------------------------------------------- +# resolve-domains +# --------------------------------------------------------------------------- + +def test_resolve_domains_true(): + validate_provider_config(cfg_with_resolve({"resolve-domains": True})) + + +def test_resolve_domains_false(): + validate_provider_config(cfg_with_resolve({"resolve-domains": False})) + + +def test_resolve_domains_invalid(): + with pytest.raises(ConfigError, match="resolve.resolve-domains must be boolean"): + validate_provider_config(cfg_with_resolve({"resolve-domains": "yes"})) + + +# --------------------------------------------------------------------------- +# resolver +# --------------------------------------------------------------------------- + +def test_resolver_os(): + validate_provider_config(cfg_with_resolve({"resolver": ["os"]})) + + +def test_resolver_ip(): + validate_provider_config(cfg_with_resolve({"resolver": ["1.1.1.1", "8.8.8.8"]})) + + +def test_resolver_mixed(): + validate_provider_config(cfg_with_resolve({"resolver": ["os", "1.1.1.1"]})) + + +def test_resolver_ipv6(): + validate_provider_config(cfg_with_resolve({"resolver": ["2620:119:35::35"]})) + + +def test_resolver_empty_list(): + with pytest.raises(ConfigError, match="non-empty list"): + validate_provider_config(cfg_with_resolve({"resolver": []})) + + +def test_resolver_not_list(): + with pytest.raises(ConfigError, match="non-empty list"): + validate_provider_config(cfg_with_resolve({"resolver": "1.1.1.1"})) + + +def test_resolver_invalid_entry(): + with pytest.raises(ConfigError, match="invalid entry"): + validate_provider_config(cfg_with_resolve({"resolver": ["not-an-ip"]})) + + +# --------------------------------------------------------------------------- +# resolver-timeout +# --------------------------------------------------------------------------- + +def test_resolver_timeout_valid(): + validate_provider_config(cfg_with_resolve({"resolver-timeout": 5})) + + +def test_resolver_timeout_zero(): + with pytest.raises(ConfigError, match="positive integer"): + validate_provider_config(cfg_with_resolve({"resolver-timeout": 0})) + + +def test_resolver_timeout_negative(): + with pytest.raises(ConfigError, match="positive integer"): + validate_provider_config(cfg_with_resolve({"resolver-timeout": -1})) + + +def test_resolver_timeout_string(): + with pytest.raises(ConfigError, match="positive integer"): + validate_provider_config(cfg_with_resolve({"resolver-timeout": "2000"})) + + +# --------------------------------------------------------------------------- +# Full valid [resolve] section +# --------------------------------------------------------------------------- + +def test_full_resolve_section(): + validate_provider_config(cfg_with_resolve({ + "resolve-domains": True, + "resolver": ["os", "1.1.1.1"], + "resolver-timeout": 3, + }))