diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd26b65..5fae575 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,24 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: + contract: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install contract dependencies + run: python -m pip install -r service/requirements.txt + + - name: Validate shared detection contract + run: PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v + website: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 28fe4b4..4a6a4dc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ web/node_modules/ web/.next/ web/.vercel/ web/out/ +__pycache__/ +*.py[cod] .env .hermes/ diff --git a/Makefile b/Makefile index ea065d7..8fa13e1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: setup dev build start test check +.PHONY: setup dev build start test service-test check setup: cd web && npm install @@ -17,4 +17,7 @@ start: test: cd web && npm test -check: test build +service-test: + PYTHONPATH=service python3 -m unittest discover -s service/tests -p 'test_*.py' -v + +check: test service-test build diff --git a/README.md b/README.md index 9e0877b..aa13462 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ What is confirmed in the repository today: - canonical detection artifact generation with frontmatter - side-by-side support for Sigma, Splunk SPL, Microsoft Sentinel KQL, Elastic EQL, and Elastic ES|QL - repository content areas for detections, examples, knowledge, and reports +- a shared [DetLab Detection Content Specification v1](docs/schema/detection-content-spec-v1.md) adapter with source hashing and generated-artifact provenance - Node-based tests for web copy/config behavior +- Python contract tests covering every authored detection YAML file ## Features diff --git a/docs/schema/detection-content-spec-v1.md b/docs/schema/detection-content-spec-v1.md new file mode 100644 index 0000000..2b3440d --- /dev/null +++ b/docs/schema/detection-content-spec-v1.md @@ -0,0 +1,49 @@ +# DetLab Detection Content Specification v1 + +This repository implements the shared **DetLab Detection Content Specification v1** at: + +- `schemas/detlab-detection-content-v1.schema.json` + +The schema identifier is `https://schemas.detlab.dev/detection-content/v1.0.0/schema.json`. + +## Contract boundary + +The v1 contract is a portable, normalized interchange model. It does **not** replace the authored source format: + +- Rich DetLab detection YAML remains canonical in `DetLab-DAC`. +- Sigma YAML remains canonical in `cybersecurity-playbook`. +- Adapters normalize either source into the same v1 shape. +- Splunk, Elastic, Kusto, and other target queries are derived artifacts, never competing authored sources. + +The existing DetLab authored schema (`schema_version: 2.0.0`) can therefore evolve without changing the stable cross-repository v1 interchange contract. + +## Required normalized fields + +- identity: `spec_version`, `kind`, `id`, `title`, `description` +- lifecycle: `status`, `severity`, `authors` +- behavior: `platforms`, `attack`, `logsource`, `logic` +- canonical source provenance: `source.format`, `source.path`, `source.sha256`, `source.canonical` + +`logic.body` contains selections. `logic.condition` remains explicit so consumers do not need format-specific condition discovery. + +## Generated artifact provenance + +Every generated artifact records: + +- target and query language +- rendered content and its SHA-256 +- canonical source SHA-256 +- contract version +- converter package name and version + +An artifact is stale when its recorded source hash or contract version no longer matches the normalized canonical source. + +## Repository adapter + +`service/detlab/contract.py` normalizes rich DetLab YAML and provides provenance/staleness helpers. + +Tests: + +```bash +PYTHONPATH=service python3 -m unittest discover -s service/tests -p 'test_*.py' -v +``` diff --git a/schemas/detlab-detection-content-v1.schema.json b/schemas/detlab-detection-content-v1.schema.json new file mode 100644 index 0000000..fb90b1e --- /dev/null +++ b/schemas/detlab-detection-content-v1.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.detlab.dev/detection-content/v1.0.0/schema.json", + "title": "DetLab Detection Content Specification v1", + "description": "Portable normalized detection contract. Authored source remains canonical; target queries are derived artifacts with provenance.", + "type": "object", + "additionalProperties": false, + "required": ["spec_version", "kind", "id", "title", "description", "status", "severity", "authors", "platforms", "attack", "logsource", "logic", "source"], + "properties": { + "spec_version": {"const": "1.0.0"}, + "kind": {"const": "detection"}, + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"}, + "title": {"type": "string", "minLength": 1, "maxLength": 300}, + "description": {"type": "string", "minLength": 1, "maxLength": 4000}, + "status": {"enum": ["draft", "experimental", "test", "testing", "available", "validated", "stable", "deprecated", "unsupported"]}, + "severity": {"enum": ["informational", "low", "medium", "high", "critical", "unknown"]}, + "authors": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1, "maxLength": 200}}, + "platforms": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}}, + "attack": { + "type": "object", + "additionalProperties": false, + "required": ["techniques", "tactics"], + "properties": { + "techniques": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^T[0-9]{4}(\\.[0-9]{3})?$"}}, + "tactics": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z][a-z0-9-]*$"}} + } + }, + "logsource": { + "type": "object", + "additionalProperties": false, + "required": ["product"], + "properties": { + "product": {"type": "string", "minLength": 1, "maxLength": 100}, + "category": {"type": "string", "minLength": 1, "maxLength": 100}, + "service": {"type": "string", "minLength": 1, "maxLength": 100} + }, + "anyOf": [{"required": ["category"]}, {"required": ["service"]}] + }, + "logic": { + "type": "object", + "additionalProperties": false, + "required": ["format", "body", "condition"], + "properties": { + "format": {"enum": ["sigma", "detlab-canonical"]}, + "body": {"type": "object", "minProperties": 1}, + "condition": {"type": "string", "minLength": 1, "maxLength": 1000} + } + }, + "source": {"$ref": "#/$defs/source"}, + "generated_artifacts": {"type": "array", "items": {"$ref": "#/$defs/generatedArtifact"}} + }, + "$defs": { + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["format", "path", "sha256", "canonical"], + "properties": { + "format": {"enum": ["sigma", "detlab-canonical"]}, + "path": {"type": "string", "minLength": 1, "maxLength": 500, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "canonical": {"const": true} + } + }, + "converter": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version"], + "properties": { + "name": {"type": "string", "minLength": 1, "maxLength": 200}, + "version": {"type": "string", "minLength": 1, "maxLength": 100} + } + }, + "generatedArtifact": { + "type": "object", + "additionalProperties": false, + "required": ["target", "language", "content", "content_sha256", "provenance"], + "properties": { + "target": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}, + "language": {"type": "string", "pattern": "^[a-z0-9][a-z0-9+|._-]*$"}, + "content": {"type": "string", "minLength": 1}, + "content_sha256": {"$ref": "#/$defs/sha256"}, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["source_sha256", "spec_version", "converter"], + "properties": { + "source_sha256": {"$ref": "#/$defs/sha256"}, + "spec_version": {"const": "1.0.0"}, + "converter": {"$ref": "#/$defs/converter"} + } + } + } + } + } +} diff --git a/service/detlab/__init__.py b/service/detlab/__init__.py new file mode 100644 index 0000000..8302a03 --- /dev/null +++ b/service/detlab/__init__.py @@ -0,0 +1 @@ +"""DetLab service package.""" diff --git a/service/detlab/contract.py b/service/detlab/contract.py new file mode 100644 index 0000000..6770f8e --- /dev/null +++ b/service/detlab/contract.py @@ -0,0 +1,82 @@ +"""Adapters for DetLab Detection Content Specification v1.""" +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any, Mapping + +SPEC_VERSION = "1.0.0" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _as_authors(value: Any) -> list[str]: + values = value if isinstance(value, list) else [value] + return [str(item).strip() for item in values if str(item).strip()] + + +def normalize_detlab_detection(detection: Mapping[str, Any], source_path: Path | str, source_bytes: bytes) -> dict[str, Any]: + canonical = detection.get("canonical_detection") or {} + analytics = canonical.get("analytics") or {} + logic = dict(analytics.get("logic") or {}) + condition = str(analytics.get("condition") or logic.pop("condition", "")).strip() + attack = canonical.get("attack") or {} + primary = attack.get("primary") or {} + related = attack.get("related") or [] + techniques = sorted({str(item.get("technique")) for item in [primary, *related] if item.get("technique")}) + tactics = sorted({str(item.get("tactic")).lower().replace("_", "-") for item in [primary, *related] if item.get("tactic")}) + logsource = {key: str(value) for key, value in (canonical.get("logsource") or {}).items() if key in {"product", "category", "service"} and value} + platforms = [str(value).lower().replace(" ", "-") for value in detection.get("platforms", []) if str(value).strip()] + if not platforms: + platforms = [str(logsource.get("product", "unknown")).lower().replace(" ", "-")] + return { + "spec_version": SPEC_VERSION, + "kind": "detection", + "id": str(detection.get("id", "")).strip(), + "title": str(detection.get("title", "")).strip(), + "description": str(detection.get("description", "")).strip(), + "status": str(detection.get("status", "draft")).lower(), + "severity": str(detection.get("severity", "unknown")).lower(), + "authors": _as_authors(detection.get("author", "unknown")), + "platforms": sorted(set(platforms)), + "attack": {"techniques": techniques, "tactics": tactics}, + "logsource": logsource, + "logic": {"format": "detlab-canonical", "body": logic, "condition": condition}, + "source": { + "format": "detlab-canonical", + "path": Path(source_path).as_posix(), + "sha256": _sha256(source_bytes), + "canonical": True, + }, + } + + +def build_generated_artifact( + normalized: Mapping[str, Any], + *, + target: str, + language: str, + content: str, + converter: Mapping[str, str], +) -> dict[str, Any]: + return { + "target": target, + "language": language, + "content": content, + "content_sha256": _sha256(content.encode("utf-8")), + "provenance": { + "source_sha256": str(normalized["source"]["sha256"]), + "spec_version": SPEC_VERSION, + "converter": {"name": str(converter["name"]), "version": str(converter["version"])}, + }, + } + + +def generated_artifact_is_stale(normalized: Mapping[str, Any], artifact: Mapping[str, Any]) -> bool: + provenance = artifact.get("provenance") or {} + return ( + provenance.get("source_sha256") != normalized.get("source", {}).get("sha256") + or provenance.get("spec_version") != SPEC_VERSION + ) diff --git a/service/requirements.txt b/service/requirements.txt new file mode 100644 index 0000000..083ac9b --- /dev/null +++ b/service/requirements.txt @@ -0,0 +1,2 @@ +jsonschema==4.26.0 +PyYAML==6.0.3 diff --git a/service/tests/test_contract.py b/service/tests/test_contract.py new file mode 100644 index 0000000..3bd34c4 --- /dev/null +++ b/service/tests/test_contract.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import hashlib +import unittest +from pathlib import Path + +import yaml +from jsonschema import Draft202012Validator + +from detlab.contract import ( + build_generated_artifact, + generated_artifact_is_stale, + normalize_detlab_detection, +) + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DETECTION_PATH = REPO_ROOT / "detections" / "windows" / "suspicious_encoded_powershell.yaml" +SCHEMA_PATH = REPO_ROOT / "schemas" / "detlab-detection-content-v1.schema.json" + + +class DetectionContentContractTests(unittest.TestCase): + def setUp(self) -> None: + self.source_bytes = DETECTION_PATH.read_bytes() + self.detection = yaml.safe_load(self.source_bytes) + + def test_rich_detection_normalizes_to_shared_v1_contract(self) -> None: + normalized = normalize_detlab_detection( + self.detection, + DETECTION_PATH.relative_to(REPO_ROOT), + self.source_bytes, + ) + schema = yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8")) + Draft202012Validator(schema).validate(normalized) + + self.assertEqual(normalized["spec_version"], "1.0.0") + self.assertEqual(normalized["source"]["format"], "detlab-canonical") + self.assertEqual(normalized["source"]["sha256"], hashlib.sha256(self.source_bytes).hexdigest()) + self.assertEqual(normalized["logic"]["condition"], "selection") + self.assertIn("T1059.001", normalized["attack"]["techniques"]) + + def test_all_authored_detections_normalize_to_shared_v1_contract(self) -> None: + schema = yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema) + paths = sorted((REPO_ROOT / "detections").rglob("*.yaml")) + sorted((REPO_ROOT / "detections").rglob("*.yml")) + self.assertGreater(len(paths), 0) + for path in paths: + with self.subTest(path=path.relative_to(REPO_ROOT)): + source_bytes = path.read_bytes() + normalized = normalize_detlab_detection( + yaml.safe_load(source_bytes), + path.relative_to(REPO_ROOT), + source_bytes, + ) + validator.validate(normalized) + + def test_generated_artifact_staleness_uses_source_hash(self) -> None: + normalized = normalize_detlab_detection( + self.detection, + DETECTION_PATH.relative_to(REPO_ROOT), + self.source_bytes, + ) + artifact = build_generated_artifact( + normalized, + target="splunk", + language="spl", + content="index=win powershell", + converter={"name": "pysigma-backend-splunk", "version": "2.1.0"}, + ) + self.assertFalse(generated_artifact_is_stale(normalized, artifact)) + artifact["provenance"]["source_sha256"] = "0" * 64 + self.assertTrue(generated_artifact_is_stale(normalized, artifact)) + + +if __name__ == "__main__": + unittest.main()