diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fae575..d52dd78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,17 @@ jobs: python-version: "3.13" - name: Install contract dependencies - run: python -m pip install -r service/requirements.txt + run: python -m pip install -r service/requirements-dev.txt - name: Validate shared detection contract run: PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v + - name: Audit Python dependencies + run: service/audit-dependencies.sh + + - name: Static security analysis + run: bandit -q -r service/detlab + website: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 8fa13e1..58429c0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: setup dev build start test service-test check +.PHONY: setup dev build start service-run test service-test check setup: cd web && npm install @@ -14,6 +14,9 @@ build: start: cd web && npm run start +service-run: + PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 + test: cd web && npm test diff --git a/README.md b/README.md index aa13462..7085197 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ What is confirmed in the repository today: - side-by-side support for Sigma, Splunk SPL, Microsoft Sentinel KQL, Elastic EQL, and Elastic ES|QL - repository content areas for detections, examples, knowledge, and reports - a shared [DetLab Detection Content Specification v1](docs/schema/detection-content-spec-v1.md) adapter with source hashing and generated-artifact provenance +- an optional server-side pySigma conversion API with an explicit backend registry, input limits, timeouts, and structured errors +- workbench conversion controls with loading, error, provenance, and stale-output states - Node-based tests for web copy/config behavior - Python contract tests covering every authored detection YAML file @@ -107,6 +109,7 @@ examples/ Example packs and sample artifacts knowledge/ Supporting documentation / authored briefs reports/ Generated or review-oriented outputs scripts/ Supporting repo scripts +service/ Optional FastAPI + pySigma conversion service web/ Next.js frontend and workbench app/ Routes and pages components/ UI components @@ -122,6 +125,7 @@ web/ Next.js frontend and workbench - Node built-in test runner (`node --test`) - Markdown/YAML-based detection artifacts - GitHub API-backed save flow in the workbench +- FastAPI and pinned pySigma backends for optional server-side conversion ## Quick start @@ -135,6 +139,17 @@ npm run dev Open `http://localhost:3000`. +### Optional conversion service + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r service/requirements.txt +PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +Enter `http://localhost:8000` in the workbench conversion panel. See [`service/README.md`](service/README.md) before exposing the API beyond localhost/LAN. + ### Repository-level helpers ```bash diff --git a/service/README.md b/service/README.md new file mode 100644 index 0000000..9d1ce5d --- /dev/null +++ b/service/README.md @@ -0,0 +1,69 @@ +# DetLab Sigma conversion service + +This optional service executes pySigma conversion outside the static browser application. The workbench stays compatible with GitHub Pages and calls this API only when an operator configures an absolute HTTP(S) origin. + +## Security model + +- explicit backend registry; request data cannot import Python modules or select arbitrary classes +- safe YAML parsing with maximum depth 20, maximum 20 aliases, and a 10,000-node post-load structure limit +- 256 KiB source limit +- five-second response timeout by default, enforced in a disposable worker process that is terminated on timeout +- conversion results are fully serialized in the worker before publication, so partial generated output is never returned +- narrow CORS allowlist from `DETLAB_CORS_ORIGINS` +- versioned converter provenance and canonical-source SHA-256 in each response +- no credentials accepted in the browser's API-origin field + +The service does not provide authentication or global rate limiting. Bind it to localhost/LAN by default and place authentication, TLS, request-rate controls, and stricter body limits at the reverse proxy before any broader exposure. + +## Local setup + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r service/requirements.txt +PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +Configure the workbench with `http://localhost:8000`, or set the build-time default: + +```bash +NEXT_PUBLIC_DETLAB_CONVERSION_API=https://conversion.example.test npm run build +``` + +Set allowed browser origins on the service: + +```bash +DETLAB_CORS_ORIGINS=https://detlab.example.test,http://localhost:3000 \ + PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +## API + +- `GET /healthz` +- `GET /v1/backends` +- `POST /v1/convert` + +Request: + +```json +{ + "source": "title: ...", + "target": "splunk" +} +``` + +Registered targets: + +- `splunk` +- `elastic-eql` +- `elastic-esql` +- `microsoft-kusto` + +## Tests + +```bash +PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v +cd web && npm test && npm run build +``` + +No endpoint in this service deploys a query to a SIEM or claims live validation. diff --git a/service/SECURITY.md b/service/SECURITY.md new file mode 100644 index 0000000..e8b14c1 --- /dev/null +++ b/service/SECURITY.md @@ -0,0 +1,23 @@ +# Service dependency security + +## DiskCache advisory (`PYSEC-2026-2447`) + +`pySigma==1.4.0` declares `diskcache==5.6.3` as a dependency, although pySigma's +runtime code does not import it and DetLab does not create or read a DiskCache. +The advisory requires both attacker write access to a cache directory and a +subsequent application read that invokes pickle deserialization. DetLab has no +such cache path, so the vulnerable operation is unreachable. + +The service's dependency audit ignores only `PYSEC-2026-2447` and must fail for +all other findings. `service/tests/test_dependency_security.py` also starts the +service with imports of `diskcache` blocked, proving the API and converter +registry do not depend on it. Do not add DiskCache use unless it uses a +non-pickle serializer, a service-private non-attacker-writable directory, and a +new security review removes this exception or updates its rationale. + +Run the policy-enforcing audit with: + +```bash +python3 -m pip install pip-audit +./service/audit-dependencies.sh +``` diff --git a/service/audit-dependencies.sh b/service/audit-dependencies.sh new file mode 100755 index 0000000..f2dcfa3 --- /dev/null +++ b/service/audit-dependencies.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +set -eu + +# pySigma 1.4.0 declares DiskCache but does not import or use it in the +# conversion service. PYSEC-2026-2447 requires an application to deserialize +# an attacker-written cache entry; this service never creates or reads one. +python3 -m pip_audit \ + --requirement "$(dirname "$0")/requirements.txt" \ + --ignore-vuln PYSEC-2026-2447 diff --git a/service/detlab/api.py b/service/detlab/api.py new file mode 100644 index 0000000..319ecdf --- /dev/null +++ b/service/detlab/api.py @@ -0,0 +1,137 @@ +"""HTTP API for bounded server-side Sigma conversion.""" +from __future__ import annotations + +import asyncio +import json +import multiprocessing +import os +import queue +import time +from typing import Any + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +from .converter import ConversionError, ConverterService + +MAX_SOURCE_BYTES = 262_144 + + +class ConversionWorkerError(RuntimeError): + """Raised when an isolated conversion worker cannot return a complete result.""" + + +def _conversion_worker(result_queue: Any, service: Any, source: str, target: str) -> None: + try: + # Serialize before publishing so callers can never observe partial output. + payload = json.dumps(service.convert(source, target), separators=(",", ":")) + result_queue.put(("ok", payload)) + except ConversionError as exc: + result_queue.put(("conversion_error", str(exc))) + except Exception: + result_queue.put(("worker_error", "Conversion worker failed")) + + +def _convert_isolated(service: Any, source: str, target: str, timeout: float) -> dict[str, Any]: + context = multiprocessing.get_context("spawn") + result_queue = context.Queue(maxsize=1) + process = context.Process( + target=_conversion_worker, + args=(result_queue, service, source, target), + daemon=True, + ) + process.start() + deadline = time.monotonic() + timeout + try: + try: + status, payload = result_queue.get(timeout=max(0.0, deadline - time.monotonic())) + except queue.Empty as exc: + if process.is_alive(): + raise TimeoutError("conversion timed out") from exc + raise ConversionWorkerError("Conversion worker returned no result") from exc + + process.join(max(0.0, deadline - time.monotonic())) + if process.is_alive(): + raise TimeoutError("conversion timed out") + finally: + if process.is_alive(): + process.terminate() + process.join(0.5) + if process.is_alive(): + process.kill() + process.join() + result_queue.close() + result_queue.join_thread() + if status == "conversion_error": + raise ConversionError(payload) + if status != "ok": + raise ConversionWorkerError(payload) + try: + result = json.loads(payload) + except (TypeError, json.JSONDecodeError) as exc: + raise ConversionWorkerError("Conversion worker returned an invalid result") from exc + if not isinstance(result, dict): + raise ConversionWorkerError("Conversion worker returned an invalid result") + return result + + +class ConvertRequest(BaseModel): + source: str + target: str + + +def _error(status_code: int, code: str, message: str) -> HTTPException: + return HTTPException(status_code=status_code, detail={"code": code, "message": message}) + + +def create_app( + *, + converter: Any | None = None, + conversion_timeout_seconds: float = 5.0, +) -> FastAPI: + service = converter or ConverterService() + app = FastAPI(title="DetLab Sigma Conversion API", version="1.0.0") + origins = [value.strip() for value in os.environ.get("DETLAB_CORS_ORIGINS", "http://localhost:3000").split(",") if value.strip()] + app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=False, + allow_methods=["GET", "POST"], + allow_headers=["Content-Type"], + ) + + @app.get("/healthz") + async def healthz() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/v1/backends") + async def backends() -> dict[str, Any]: + return {"backends": service.backends()} + + @app.post("/v1/convert") + async def convert(request: ConvertRequest) -> dict[str, Any]: + if len(request.source.encode("utf-8")) > MAX_SOURCE_BYTES: + raise _error(413, "source_too_large", f"Sigma source exceeds {MAX_SOURCE_BYTES} bytes") + known_targets = {backend["id"] for backend in service.backends()} + if request.target not in known_targets: + raise _error(422, "unsupported_backend", "Requested conversion backend is not registered") + try: + return await asyncio.to_thread( + _convert_isolated, + service, + request.source, + request.target, + conversion_timeout_seconds, + ) + except TimeoutError as exc: + raise _error(504, "conversion_timeout", "Conversion exceeded the configured timeout") from exc + except ConversionError as exc: + raise _error(422, "invalid_sigma", str(exc)) from exc + except ConversionWorkerError as exc: + raise _error(500, "conversion_failed", "Conversion worker failed safely") from exc + + return app + + +app = create_app() diff --git a/service/detlab/converter.py b/service/detlab/converter.py new file mode 100644 index 0000000..5a77721 --- /dev/null +++ b/service/detlab/converter.py @@ -0,0 +1,125 @@ +"""Pinned, explicit pySigma backend registry and conversion service.""" +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +from dataclasses import dataclass +from typing import Any, Callable + +import yaml +from yaml.events import AliasEvent, CollectionEndEvent, CollectionStartEvent + +from sigma.backends.elasticsearch import EqlBackend, ESQLBackend +from sigma.backends.kusto import KustoBackend +from sigma.backends.splunk import SplunkBackend +from sigma.collection import SigmaCollection + + +class ConversionError(ValueError): + """Raised when authored Sigma cannot be parsed or converted safely.""" + + +MAX_YAML_ALIASES = 20 +MAX_YAML_DEPTH = 20 +MAX_YAML_NODES = 10_000 + + +def _validate_loaded_structure(value: Any) -> None: + nodes = 0 + stack = [value] + while stack: + current = stack.pop() + nodes += 1 + if nodes > MAX_YAML_NODES: + raise ConversionError(f"Sigma YAML structure exceeds {MAX_YAML_NODES} nodes") + if isinstance(current, dict): + stack.extend(current.keys()) + stack.extend(current.values()) + elif isinstance(current, (list, tuple, set)): + stack.extend(current) + + +def validate_sigma_yaml(source: str) -> None: + aliases = 0 + depth = 0 + try: + for event in yaml.parse(source, Loader=yaml.SafeLoader): + if isinstance(event, AliasEvent): + aliases += 1 + if aliases > MAX_YAML_ALIASES: + raise ConversionError(f"Sigma YAML exceeds {MAX_YAML_ALIASES} aliases") + elif isinstance(event, CollectionStartEvent): + depth += 1 + if depth > MAX_YAML_DEPTH: + raise ConversionError(f"Sigma YAML exceeds depth {MAX_YAML_DEPTH}") + elif isinstance(event, CollectionEndEvent): + depth -= 1 + loaded = yaml.safe_load(source) + except ConversionError: + raise + except yaml.YAMLError as exc: + raise ConversionError("Sigma source is invalid") from exc + _validate_loaded_structure(loaded) + + +@dataclass(frozen=True) +class BackendDefinition: + id: str + language: str + package: str + factory: Callable[[], Any] + + def public(self) -> dict[str, str]: + return { + "id": self.id, + "language": self.language, + "package": self.package, + "version": importlib.metadata.version(self.package), + } + + +BACKENDS = ( + BackendDefinition("splunk", "spl", "pysigma-backend-splunk", SplunkBackend), + BackendDefinition("elastic-eql", "eql", "pysigma-backend-elasticsearch", EqlBackend), + BackendDefinition("elastic-esql", "esql", "pysigma-backend-elasticsearch", ESQLBackend), + BackendDefinition("microsoft-kusto", "kql", "pysigma-backend-kusto", KustoBackend), +) + + +class ConverterService: + def __init__(self) -> None: + self._registry = {definition.id: definition for definition in BACKENDS} + + def backends(self) -> list[dict[str, str]]: + return [definition.public() for definition in BACKENDS] + + def convert(self, source: str, target: str) -> dict[str, Any]: + definition = self._registry.get(target) + if definition is None: + raise KeyError(target) + validate_sigma_yaml(source) + try: + collection = SigmaCollection.from_yaml(source) + except Exception as exc: + raise ConversionError("Sigma source is invalid") from exc + try: + rendered = definition.factory().convert(collection) + except Exception as exc: + raise ConversionError("Sigma source is unsupported by the selected backend") from exc + outputs = [item if isinstance(item, str) else json.dumps(item, sort_keys=True) for item in rendered] + if not outputs or not all(item.strip() for item in outputs): + raise ConversionError("Selected backend produced no query output") + source_sha256 = hashlib.sha256(source.encode("utf-8")).hexdigest() + backend = definition.public() + return { + "target": definition.id, + "language": definition.language, + "outputs": outputs, + "source_sha256": source_sha256, + "provenance": { + "spec_version": "1.0.0", + "source_sha256": source_sha256, + "converter": {"name": backend["package"], "version": backend["version"]}, + }, + } diff --git a/service/requirements-dev.txt b/service/requirements-dev.txt new file mode 100644 index 0000000..d0c39ea --- /dev/null +++ b/service/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt +bandit==1.8.6 +pip-audit==2.10.1 +pytest==9.1.1 diff --git a/service/requirements.txt b/service/requirements.txt index 083ac9b..5a9ba49 100644 --- a/service/requirements.txt +++ b/service/requirements.txt @@ -1,2 +1,11 @@ +fastapi==0.140.7 +httpx==0.28.1 jsonschema==4.26.0 PyYAML==6.0.3 +pySigma==1.4.0 +pysigma-backend-elasticsearch==2.1.0 +pysigma-backend-kusto==1.0.1 +pysigma-backend-splunk==2.1.0 +pysigma-pipeline-windows==2.0.0 +starlette==1.3.1 +uvicorn==0.35.0 diff --git a/service/tests/test_ci_security_policy.py b/service/tests/test_ci_security_policy.py new file mode 100644 index 0000000..1e52faa --- /dev/null +++ b/service/tests/test_ci_security_policy.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import re +import unittest +from pathlib import Path + + +class CiSecurityPolicyTests(unittest.TestCase): + def test_contract_job_runs_dependency_policy_script(self) -> None: + workflow = ( + Path(__file__).resolve().parents[2] / ".github" / "workflows" / "ci.yml" + ).read_text(encoding="utf-8") + + self.assertRegex( + workflow, + re.compile( + r"- name: Audit Python dependencies\s+" + r"run: (?:sh )?service/audit-dependencies\.sh" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/service/tests/test_conversion_api.py b/service/tests/test_conversion_api.py new file mode 100644 index 0000000..25a16a6 --- /dev/null +++ b/service/tests/test_conversion_api.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import time +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from fastapi.testclient import TestClient + +from detlab.api import create_app +from detlab.converter import ConverterService + + +VALID_SIGMA = """title: Suspicious Encoded PowerShell +id: 11111111-1111-4111-8111-111111111111 +status: experimental +description: Detect encoded PowerShell command lines. +author: mell0wx +date: 2026-07-27 +logsource: + product: windows + category: process_creation +detection: + selection: + Image|endswith: '\\powershell.exe' + CommandLine|contains: '-enc' + condition: selection +falsepositives: + - Administrative automation +level: high +tags: + - attack.execution + - attack.t1059.001 +""" + + +class SlowConverter: + def backends(self): + return [{"id": "splunk", "language": "spl", "package": "test", "version": "1"}] + + def convert(self, source: str, target: str): + time.sleep(0.05) + return {} + + +class DelayedSideEffectConverter(SlowConverter): + def __init__(self, marker: str) -> None: + self.marker = marker + + def convert(self, source: str, target: str): + time.sleep(0.1) + Path(self.marker).write_text("worker survived timeout", encoding="utf-8") + return {} + + +class IncompleteResultConverter(SlowConverter): + def convert(self, source: str, target: str): + return {"outputs": ["complete", object()]} + + +class LargeResultConverter(SlowConverter): + def convert(self, source: str, target: str): + return {"target": target, "outputs": ["A" * 255_206]} + + +class ConversionApiTests(unittest.TestCase): + def setUp(self) -> None: + self.client = TestClient(create_app(conversion_timeout_seconds=2.0)) + + def test_backend_registry_is_explicit_and_versioned(self) -> None: + response = self.client.get("/v1/backends") + self.assertEqual(response.status_code, 200) + payload = response.json() + ids = {item["id"] for item in payload["backends"]} + self.assertEqual(ids, {"splunk", "elastic-eql", "elastic-esql", "microsoft-kusto"}) + for backend in payload["backends"]: + self.assertTrue(backend["package"]) + self.assertTrue(backend["version"]) + + def test_convert_returns_query_and_reproducible_provenance(self) -> None: + response = self.client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + self.assertEqual(response.status_code, 200, response.text) + payload = response.json() + self.assertEqual(payload["target"], "splunk") + self.assertEqual(payload["language"], "spl") + self.assertTrue(payload["outputs"]) + self.assertRegex(payload["source_sha256"], r"^[a-f0-9]{64}$") + self.assertEqual(payload["provenance"]["spec_version"], "1.0.0") + self.assertEqual(payload["provenance"]["source_sha256"], payload["source_sha256"]) + + def test_unsupported_backend_is_rejected_without_dynamic_loading(self) -> None: + response = self.client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "python:os.system"}) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "unsupported_backend") + + def test_malformed_and_unsafe_yaml_are_rejected(self) -> None: + for source in ("title: [", "!!python/object/apply:os.system ['id']"): + with self.subTest(source=source): + response = self.client.post("/v1/convert", json={"source": source, "target": "splunk"}) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "invalid_sigma") + + def test_source_size_limit_is_enforced(self) -> None: + response = self.client.post("/v1/convert", json={"source": "A" * 262145, "target": "splunk"}) + self.assertEqual(response.status_code, 413) + self.assertEqual(response.json()["detail"]["code"], "source_too_large") + + def test_conversion_timeout_is_bounded(self) -> None: + client = TestClient(create_app(converter=SlowConverter(), conversion_timeout_seconds=0.01)) + response = client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + self.assertEqual(response.status_code, 504) + self.assertEqual(response.json()["detail"]["code"], "conversion_timeout") + + def test_conversion_timeout_terminates_worker_before_side_effect(self) -> None: + with TemporaryDirectory() as directory: + marker = Path(directory) / "late-write" + client = TestClient( + create_app( + converter=DelayedSideEffectConverter(str(marker)), + conversion_timeout_seconds=0.01, + ) + ) + response = client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + time.sleep(0.15) + + self.assertEqual(response.status_code, 504) + self.assertFalse(marker.exists(), "timed-out worker continued running") + + def test_yaml_alias_limit_is_enforced_before_conversion(self) -> None: + aliases = "\n".join(f" alias_{index}: *shared" for index in range(21)) + source = f"title: aliases\nshared: &shared value\nitems:\n{aliases}\n" + response = self.client.post("/v1/convert", json={"source": source, "target": "splunk"}) + + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "invalid_sigma") + self.assertIn("alias", response.json()["detail"]["message"].lower()) + + def test_yaml_depth_limit_is_enforced_before_conversion(self) -> None: + source = "value" + for _ in range(21): + source = f"- {source}" + response = self.client.post("/v1/convert", json={"source": source, "target": "splunk"}) + + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "invalid_sigma") + self.assertIn("depth", response.json()["detail"]["message"].lower()) + + def test_post_load_structure_limit_is_enforced(self) -> None: + source = "items:\n" + "".join(f" - {index}\n" for index in range(10001)) + response = self.client.post("/v1/convert", json={"source": source, "target": "splunk"}) + + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "invalid_sigma") + self.assertIn("structure", response.json()["detail"]["message"].lower()) + + def test_generation_is_published_only_after_complete_serialization(self) -> None: + client = TestClient(create_app(converter=IncompleteResultConverter())) + response = client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()["detail"]["code"], "conversion_failed") + + def test_large_complete_result_is_received_before_worker_join(self) -> None: + client = TestClient(create_app(converter=LargeResultConverter(), conversion_timeout_seconds=2.0)) + + response = client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(response.json()["outputs"][0]), 255_206) + + +class ConverterServiceTests(unittest.TestCase): + def test_all_registered_backends_convert_the_same_sigma_source(self) -> None: + service = ConverterService() + for backend in service.backends(): + with self.subTest(target=backend["id"]): + result = service.convert(VALID_SIGMA, backend["id"]) + self.assertTrue(result["outputs"]) + self.assertEqual(result["target"], backend["id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/service/tests/test_dependency_security.py b/service/tests/test_dependency_security.py new file mode 100644 index 0000000..2c02beb --- /dev/null +++ b/service/tests/test_dependency_security.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import subprocess +import sys +import unittest + + +class DependencySecurityTests(unittest.TestCase): + def test_service_operates_with_diskcache_imports_blocked(self) -> None: + script = """ +import sys +sys.modules['diskcache'] = None +sys.modules['diskcache.core'] = None +from detlab.api import app +from detlab.converter import ConverterService +assert app.title == 'DetLab Sigma Conversion API' +service = ConverterService() +assert service.backends() +result = service.convert(''' +title: DiskCache-free conversion +logsource: + product: windows +detection: + selection: + EventID: 1 + condition: selection +''', 'splunk') +assert result['outputs'] +""" + completed = subprocess.run( + [sys.executable, "-c", script], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/app/content/[slug]/page.tsx b/web/app/content/[slug]/page.tsx index fb039e5..3a9b5d8 100644 --- a/web/app/content/[slug]/page.tsx +++ b/web/app/content/[slug]/page.tsx @@ -7,8 +7,9 @@ export function generateStaticParams() { return contentLanes.map((lane) => ({ slug: lane.slug })) } -export default function ContentLanePage({ params }: { params: { slug: string } }) { - const lane = getLaneBySlug(params.slug) +export default async function ContentLanePage({ params }: { params: Promise<{ slug: string }> }) { + const { slug } = await params + const lane = getLaneBySlug(slug) if (!lane) { notFound() diff --git a/web/components/lane-workbench.tsx b/web/components/lane-workbench.tsx index 428ce7e..1a5ebf3 100644 --- a/web/components/lane-workbench.tsx +++ b/web/components/lane-workbench.tsx @@ -1,12 +1,17 @@ 'use client' import type { ReactNode } from 'react' -import { useEffect, useMemo, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import { buildLaneArtifact, buildRepoFilePath, getWorkbenchConfig, } from '../data/workbench-config.mjs' +import { + conversionFieldForTarget, + requestSigmaConversion, +} from '../data/conversion-client.mjs' +import { runConversionRequest } from '../data/conversion-request.mjs' type LaneSlug = 'detections' @@ -17,6 +22,12 @@ type SaveResult = { sha: string } | null +type ConversionResult = { + sourceSha256: string + converter: string + target: string +} | null + type FormState = { repoOwner: string repoName: string @@ -120,12 +131,22 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La const [saveState, setSaveState] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle') const [statusMessage, setStatusMessage] = useState('') const [saveResult, setSaveResult] = useState(null) + const [conversionApiUrl, setConversionApiUrl] = useState(process.env.NEXT_PUBLIC_DETLAB_CONVERSION_API ?? '') + const [conversionTarget, setConversionTarget] = useState('splunk') + const [conversionState, setConversionState] = useState<'idle' | 'converting' | 'converted' | 'stale' | 'error'>('idle') + const [conversionMessage, setConversionMessage] = useState('') + const [conversionResult, setConversionResult] = useState(null) + const latestSigmaRef = useRef(formState.sigma) + const requestGenerationRef = useRef(0) + + latestSigmaRef.current = formState.sigma const config = useMemo(() => getWorkbenchConfig(laneSlug), [laneSlug]) useEffect(() => { const savedToken = window.localStorage.getItem('detlab:github-token') const savedState = window.localStorage.getItem('detlab:detection-workbench') + const savedConversionApi = window.localStorage.getItem('detlab:conversion-api') if (savedToken) { setToken(savedToken) @@ -136,6 +157,10 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La } else { setFormState(getDefaultFormState()) } + + if (savedConversionApi) { + setConversionApiUrl(savedConversionApi) + } }, []) useEffect(() => { @@ -146,6 +171,10 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La window.localStorage.setItem('detlab:detection-workbench', JSON.stringify(formState)) }, [formState]) + useEffect(() => { + window.localStorage.setItem('detlab:conversion-api', conversionApiUrl) + }, [conversionApiUrl]) + const artifact = useMemo( () => buildLaneArtifact({ @@ -256,6 +285,53 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La } } + async function handleConvert() { + if (!conversionApiUrl.trim()) { + setConversionState('error') + setConversionMessage('Configure the server-side conversion API URL before converting.') + return + } + if (!formState.sigma.trim()) { + setConversionState('error') + setConversionMessage('Add authored Sigma YAML before converting.') + return + } + const submittedSource = formState.sigma + const requestGeneration = ++requestGenerationRef.current + setConversionState('converting') + setConversionMessage('Converting authored Sigma with the selected pinned backend…') + await runConversionRequest({ + source: submittedSource, + generation: requestGeneration, + request: () => requestSigmaConversion({ + baseUrl: conversionApiUrl, + source: submittedSource, + target: conversionTarget, + }), + isCurrent: (generation: number, source: string) => ( + requestGenerationRef.current === generation && latestSigmaRef.current === source + ), + publishSuccess: (result: any) => { + const field = conversionFieldForTarget(result.target) + if (!field) { + throw new Error('Conversion response target is not supported by this workbench.') + } + setFormState((current) => ({ ...current, [field]: result.outputs.join('\n\n') })) + setConversionState('converted') + setConversionResult({ + sourceSha256: result.source_sha256, + converter: `${result.provenance.converter.name}@${result.provenance.converter.version}`, + target: result.target, + }) + setConversionMessage(`Generated ${result.target} from the current Sigma source.`) + }, + publishError: (error: unknown) => { + setConversionState('error') + setConversionMessage(error instanceof Error ? error.message : 'Unknown conversion failure.') + }, + }) + } + return (
@@ -354,11 +430,69 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La