From bf1babd349196d7860a95eaaa17a191763fa01ea Mon Sep 17 00:00:00 2001 From: mell0wx <21230024+egrexsec@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:14:43 +0000 Subject: [PATCH 1/7] feat: add server-side Sigma conversion workbench --- .github/workflows/ci.yml | 5 +- Makefile | 5 +- README.md | 15 ++++ service/README.md | 68 +++++++++++++++ service/detlab/api.py | 70 ++++++++++++++++ service/detlab/converter.py | 78 ++++++++++++++++++ service/requirements-dev.txt | 2 + service/requirements.txt | 8 ++ service/tests/test_conversion_api.py | 104 +++++++++++++++++++++++ web/components/lane-workbench.tsx | 119 ++++++++++++++++++++++++++- web/data/conversion-client.mjs | 47 +++++++++++ web/tests/conversion-client.test.mjs | 68 +++++++++++++++ 12 files changed, 586 insertions(+), 3 deletions(-) create mode 100644 service/README.md create mode 100644 service/detlab/api.py create mode 100644 service/detlab/converter.py create mode 100644 service/requirements-dev.txt create mode 100644 service/tests/test_conversion_api.py create mode 100644 web/data/conversion-client.mjs create mode 100644 web/tests/conversion-client.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fae575..293535b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,14 @@ jobs: python-version: "3.13" - name: Install contract dependencies - run: python -m pip install -r service/requirements.txt + run: python -m pip install -r service/requirements-dev.txt - name: Validate shared detection contract run: PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v + - name: Static security analysis + run: bandit -q -r service/detlab + website: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 8fa13e1..58429c0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: setup dev build start test service-test check +.PHONY: setup dev build start service-run test service-test check setup: cd web && npm install @@ -14,6 +14,9 @@ build: start: cd web && npm run start +service-run: + PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 + test: cd web && npm test diff --git a/README.md b/README.md index aa13462..7085197 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ What is confirmed in the repository today: - side-by-side support for Sigma, Splunk SPL, Microsoft Sentinel KQL, Elastic EQL, and Elastic ES|QL - repository content areas for detections, examples, knowledge, and reports - a shared [DetLab Detection Content Specification v1](docs/schema/detection-content-spec-v1.md) adapter with source hashing and generated-artifact provenance +- an optional server-side pySigma conversion API with an explicit backend registry, input limits, timeouts, and structured errors +- workbench conversion controls with loading, error, provenance, and stale-output states - Node-based tests for web copy/config behavior - Python contract tests covering every authored detection YAML file @@ -107,6 +109,7 @@ examples/ Example packs and sample artifacts knowledge/ Supporting documentation / authored briefs reports/ Generated or review-oriented outputs scripts/ Supporting repo scripts +service/ Optional FastAPI + pySigma conversion service web/ Next.js frontend and workbench app/ Routes and pages components/ UI components @@ -122,6 +125,7 @@ web/ Next.js frontend and workbench - Node built-in test runner (`node --test`) - Markdown/YAML-based detection artifacts - GitHub API-backed save flow in the workbench +- FastAPI and pinned pySigma backends for optional server-side conversion ## Quick start @@ -135,6 +139,17 @@ npm run dev Open `http://localhost:3000`. +### Optional conversion service + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r service/requirements.txt +PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +Enter `http://localhost:8000` in the workbench conversion panel. See [`service/README.md`](service/README.md) before exposing the API beyond localhost/LAN. + ### Repository-level helpers ```bash diff --git a/service/README.md b/service/README.md new file mode 100644 index 0000000..381911f --- /dev/null +++ b/service/README.md @@ -0,0 +1,68 @@ +# DetLab Sigma conversion service + +This optional service executes pySigma conversion outside the static browser application. The workbench stays compatible with GitHub Pages and calls this API only when an operator configures an absolute HTTP(S) origin. + +## Security model + +- explicit backend registry; request data cannot import Python modules or select arbitrary classes +- safe pySigma/YAML parsing path +- 256 KiB source limit +- five-second response timeout by default +- narrow CORS allowlist from `DETLAB_CORS_ORIGINS` +- versioned converter provenance and canonical-source SHA-256 in each response +- no credentials accepted in the browser's API-origin field + +The service does not provide authentication or global rate limiting. Bind it to localhost/LAN by default and place authentication, TLS, request-rate controls, and stricter body limits at the reverse proxy before any broader exposure. + +## Local setup + +```bash +python3 -m venv .venv +. .venv/bin/activate +pip install -r service/requirements.txt +PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +Configure the workbench with `http://localhost:8000`, or set the build-time default: + +```bash +NEXT_PUBLIC_DETLAB_CONVERSION_API=https://conversion.example.test npm run build +``` + +Set allowed browser origins on the service: + +```bash +DETLAB_CORS_ORIGINS=https://detlab.example.test,http://localhost:3000 \ + PYTHONPATH=service uvicorn detlab.api:app --host 127.0.0.1 --port 8000 +``` + +## API + +- `GET /healthz` +- `GET /v1/backends` +- `POST /v1/convert` + +Request: + +```json +{ + "source": "title: ...", + "target": "splunk" +} +``` + +Registered targets: + +- `splunk` +- `elastic-eql` +- `elastic-esql` +- `microsoft-kusto` + +## Tests + +```bash +PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v +cd web && npm test && npm run build +``` + +No endpoint in this service deploys a query to a SIEM or claims live validation. diff --git a/service/detlab/api.py b/service/detlab/api.py new file mode 100644 index 0000000..92d3770 --- /dev/null +++ b/service/detlab/api.py @@ -0,0 +1,70 @@ +"""HTTP API for bounded server-side Sigma conversion.""" +from __future__ import annotations + +import asyncio +import os +from typing import Any + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +from .converter import ConversionError, ConverterService + +MAX_SOURCE_BYTES = 262_144 + + +class ConvertRequest(BaseModel): + source: str + target: str + + +def _error(status_code: int, code: str, message: str) -> HTTPException: + return HTTPException(status_code=status_code, detail={"code": code, "message": message}) + + +def create_app( + *, + converter: Any | None = None, + conversion_timeout_seconds: float = 5.0, +) -> FastAPI: + service = converter or ConverterService() + app = FastAPI(title="DetLab Sigma Conversion API", version="1.0.0") + origins = [value.strip() for value in os.environ.get("DETLAB_CORS_ORIGINS", "http://localhost:3000").split(",") if value.strip()] + app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=False, + allow_methods=["GET", "POST"], + allow_headers=["Content-Type"], + ) + + @app.get("/healthz") + async def healthz() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/v1/backends") + async def backends() -> dict[str, Any]: + return {"backends": service.backends()} + + @app.post("/v1/convert") + async def convert(request: ConvertRequest) -> dict[str, Any]: + if len(request.source.encode("utf-8")) > MAX_SOURCE_BYTES: + raise _error(413, "source_too_large", f"Sigma source exceeds {MAX_SOURCE_BYTES} bytes") + known_targets = {backend["id"] for backend in service.backends()} + if request.target not in known_targets: + raise _error(422, "unsupported_backend", "Requested conversion backend is not registered") + try: + return await asyncio.wait_for( + asyncio.to_thread(service.convert, request.source, request.target), + timeout=conversion_timeout_seconds, + ) + except TimeoutError as exc: + raise _error(504, "conversion_timeout", "Conversion exceeded the configured timeout") from exc + except ConversionError as exc: + raise _error(422, "invalid_sigma", str(exc)) from exc + + return app + + +app = create_app() diff --git a/service/detlab/converter.py b/service/detlab/converter.py new file mode 100644 index 0000000..e88e975 --- /dev/null +++ b/service/detlab/converter.py @@ -0,0 +1,78 @@ +"""Pinned, explicit pySigma backend registry and conversion service.""" +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +from dataclasses import dataclass +from typing import Any, Callable + +from sigma.backends.elasticsearch import EqlBackend, ESQLBackend +from sigma.backends.kusto import KustoBackend +from sigma.backends.splunk import SplunkBackend +from sigma.collection import SigmaCollection + + +class ConversionError(ValueError): + """Raised when authored Sigma cannot be parsed or converted safely.""" + + +@dataclass(frozen=True) +class BackendDefinition: + id: str + language: str + package: str + factory: Callable[[], Any] + + def public(self) -> dict[str, str]: + return { + "id": self.id, + "language": self.language, + "package": self.package, + "version": importlib.metadata.version(self.package), + } + + +BACKENDS = ( + BackendDefinition("splunk", "spl", "pysigma-backend-splunk", SplunkBackend), + BackendDefinition("elastic-eql", "eql", "pysigma-backend-elasticsearch", EqlBackend), + BackendDefinition("elastic-esql", "esql", "pysigma-backend-elasticsearch", ESQLBackend), + BackendDefinition("microsoft-kusto", "kql", "pysigma-backend-kusto", KustoBackend), +) + + +class ConverterService: + def __init__(self) -> None: + self._registry = {definition.id: definition for definition in BACKENDS} + + def backends(self) -> list[dict[str, str]]: + return [definition.public() for definition in BACKENDS] + + def convert(self, source: str, target: str) -> dict[str, Any]: + definition = self._registry.get(target) + if definition is None: + raise KeyError(target) + try: + collection = SigmaCollection.from_yaml(source) + except Exception as exc: + raise ConversionError("Sigma source is invalid") from exc + try: + rendered = definition.factory().convert(collection) + except Exception as exc: + raise ConversionError("Sigma source is unsupported by the selected backend") from exc + outputs = [item if isinstance(item, str) else json.dumps(item, sort_keys=True) for item in rendered] + if not outputs or not all(item.strip() for item in outputs): + raise ConversionError("Selected backend produced no query output") + source_sha256 = hashlib.sha256(source.encode("utf-8")).hexdigest() + backend = definition.public() + return { + "target": definition.id, + "language": definition.language, + "outputs": outputs, + "source_sha256": source_sha256, + "provenance": { + "spec_version": "1.0.0", + "source_sha256": source_sha256, + "converter": {"name": backend["package"], "version": backend["version"]}, + }, + } diff --git a/service/requirements-dev.txt b/service/requirements-dev.txt new file mode 100644 index 0000000..fa046dc --- /dev/null +++ b/service/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +bandit==1.8.6 diff --git a/service/requirements.txt b/service/requirements.txt index 083ac9b..5b7bace 100644 --- a/service/requirements.txt +++ b/service/requirements.txt @@ -1,2 +1,10 @@ +fastapi==0.116.1 +httpx==0.28.1 jsonschema==4.26.0 PyYAML==6.0.3 +pySigma==1.4.0 +pysigma-backend-elasticsearch==2.1.0 +pysigma-backend-kusto==1.0.1 +pysigma-backend-splunk==2.1.0 +pysigma-pipeline-windows==2.0.0 +uvicorn==0.35.0 diff --git a/service/tests/test_conversion_api.py b/service/tests/test_conversion_api.py new file mode 100644 index 0000000..de77ea1 --- /dev/null +++ b/service/tests/test_conversion_api.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import time +import unittest + +from fastapi.testclient import TestClient + +from detlab.api import create_app +from detlab.converter import ConverterService + + +VALID_SIGMA = """title: Suspicious Encoded PowerShell +id: 11111111-1111-4111-8111-111111111111 +status: experimental +description: Detect encoded PowerShell command lines. +author: mell0wx +date: 2026-07-27 +logsource: + product: windows + category: process_creation +detection: + selection: + Image|endswith: '\\powershell.exe' + CommandLine|contains: '-enc' + condition: selection +falsepositives: + - Administrative automation +level: high +tags: + - attack.execution + - attack.t1059.001 +""" + + +class SlowConverter: + def backends(self): + return [{"id": "splunk", "language": "spl", "package": "test", "version": "1"}] + + def convert(self, source: str, target: str): + time.sleep(0.05) + return {} + + +class ConversionApiTests(unittest.TestCase): + def setUp(self) -> None: + self.client = TestClient(create_app(conversion_timeout_seconds=2.0)) + + def test_backend_registry_is_explicit_and_versioned(self) -> None: + response = self.client.get("/v1/backends") + self.assertEqual(response.status_code, 200) + payload = response.json() + ids = {item["id"] for item in payload["backends"]} + self.assertEqual(ids, {"splunk", "elastic-eql", "elastic-esql", "microsoft-kusto"}) + for backend in payload["backends"]: + self.assertTrue(backend["package"]) + self.assertTrue(backend["version"]) + + def test_convert_returns_query_and_reproducible_provenance(self) -> None: + response = self.client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + self.assertEqual(response.status_code, 200, response.text) + payload = response.json() + self.assertEqual(payload["target"], "splunk") + self.assertEqual(payload["language"], "spl") + self.assertTrue(payload["outputs"]) + self.assertRegex(payload["source_sha256"], r"^[a-f0-9]{64}$") + self.assertEqual(payload["provenance"]["spec_version"], "1.0.0") + self.assertEqual(payload["provenance"]["source_sha256"], payload["source_sha256"]) + + def test_unsupported_backend_is_rejected_without_dynamic_loading(self) -> None: + response = self.client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "python:os.system"}) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "unsupported_backend") + + def test_malformed_and_unsafe_yaml_are_rejected(self) -> None: + for source in ("title: [", "!!python/object/apply:os.system ['id']"): + with self.subTest(source=source): + response = self.client.post("/v1/convert", json={"source": source, "target": "splunk"}) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.json()["detail"]["code"], "invalid_sigma") + + def test_source_size_limit_is_enforced(self) -> None: + response = self.client.post("/v1/convert", json={"source": "A" * 262145, "target": "splunk"}) + self.assertEqual(response.status_code, 413) + self.assertEqual(response.json()["detail"]["code"], "source_too_large") + + def test_conversion_timeout_is_bounded(self) -> None: + client = TestClient(create_app(converter=SlowConverter(), conversion_timeout_seconds=0.01)) + response = client.post("/v1/convert", json={"source": VALID_SIGMA, "target": "splunk"}) + self.assertEqual(response.status_code, 504) + self.assertEqual(response.json()["detail"]["code"], "conversion_timeout") + + +class ConverterServiceTests(unittest.TestCase): + def test_all_registered_backends_convert_the_same_sigma_source(self) -> None: + service = ConverterService() + for backend in service.backends(): + with self.subTest(target=backend["id"]): + result = service.convert(VALID_SIGMA, backend["id"]) + self.assertTrue(result["outputs"]) + self.assertEqual(result["target"], backend["id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/components/lane-workbench.tsx b/web/components/lane-workbench.tsx index 428ce7e..816c08f 100644 --- a/web/components/lane-workbench.tsx +++ b/web/components/lane-workbench.tsx @@ -7,6 +7,10 @@ import { buildRepoFilePath, getWorkbenchConfig, } from '../data/workbench-config.mjs' +import { + conversionFieldForTarget, + requestSigmaConversion, +} from '../data/conversion-client.mjs' type LaneSlug = 'detections' @@ -17,6 +21,12 @@ type SaveResult = { sha: string } | null +type ConversionResult = { + sourceSha256: string + converter: string + target: string +} | null + type FormState = { repoOwner: string repoName: string @@ -120,12 +130,18 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La const [saveState, setSaveState] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle') const [statusMessage, setStatusMessage] = useState('') const [saveResult, setSaveResult] = useState(null) + const [conversionApiUrl, setConversionApiUrl] = useState(process.env.NEXT_PUBLIC_DETLAB_CONVERSION_API ?? '') + const [conversionTarget, setConversionTarget] = useState('splunk') + const [conversionState, setConversionState] = useState<'idle' | 'converting' | 'converted' | 'stale' | 'error'>('idle') + const [conversionMessage, setConversionMessage] = useState('') + const [conversionResult, setConversionResult] = useState(null) const config = useMemo(() => getWorkbenchConfig(laneSlug), [laneSlug]) useEffect(() => { const savedToken = window.localStorage.getItem('detlab:github-token') const savedState = window.localStorage.getItem('detlab:detection-workbench') + const savedConversionApi = window.localStorage.getItem('detlab:conversion-api') if (savedToken) { setToken(savedToken) @@ -136,6 +152,10 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La } else { setFormState(getDefaultFormState()) } + + if (savedConversionApi) { + setConversionApiUrl(savedConversionApi) + } }, []) useEffect(() => { @@ -146,6 +166,10 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La window.localStorage.setItem('detlab:detection-workbench', JSON.stringify(formState)) }, [formState]) + useEffect(() => { + window.localStorage.setItem('detlab:conversion-api', conversionApiUrl) + }, [conversionApiUrl]) + const artifact = useMemo( () => buildLaneArtifact({ @@ -256,6 +280,43 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La } } + async function handleConvert() { + if (!conversionApiUrl.trim()) { + setConversionState('error') + setConversionMessage('Configure the server-side conversion API URL before converting.') + return + } + if (!formState.sigma.trim()) { + setConversionState('error') + setConversionMessage('Add authored Sigma YAML before converting.') + return + } + setConversionState('converting') + setConversionMessage('Converting authored Sigma with the selected pinned backend…') + try { + const result = await requestSigmaConversion({ + baseUrl: conversionApiUrl, + source: formState.sigma, + target: conversionTarget, + }) + const field = conversionFieldForTarget(result.target) + if (!field) { + throw new Error('Conversion response target is not supported by this workbench.') + } + setFormState((current) => ({ ...current, [field]: result.outputs.join('\n\n') })) + setConversionState('converted') + setConversionResult({ + sourceSha256: result.source_sha256, + converter: `${result.provenance.converter.name}@${result.provenance.converter.version}`, + target: result.target, + }) + setConversionMessage(`Generated ${result.target} from the current Sigma source.`) + } catch (error) { + setConversionState('error') + setConversionMessage(error instanceof Error ? error.message : 'Unknown conversion failure.') + } + } + return (
@@ -354,11 +415,67 @@ export default function LaneWorkbench({ initialLaneSlug }: { initialLaneSlug: La