feat: add shared detection content specification v1

.github/workflows/ci.yml

@@ -12,6 +12,24 @@ env:
   FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
 
 jobs:
+  contract:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+
+      - name: Install contract dependencies
+        run: python -m pip install -r service/requirements.txt
+
+      - name: Validate shared detection contract
+        run: PYTHONPATH=service python -m unittest discover -s service/tests -p 'test_*.py' -v
+
   website:
     runs-on: ubuntu-latest
 

.gitignore

@@ -2,5 +2,7 @@ web/node_modules/
 web/.next/
 web/.vercel/
 web/out/
+__pycache__/
+*.py[cod]
 .env
 .hermes/

Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 
-.PHONY: setup dev build start test check
+.PHONY: setup dev build start test service-test check
 
 setup:
 	cd web && npm install
@@ -17,4 +17,7 @@ start:
 test:
 	cd web && npm test
 
-check: test build
+service-test:
+	PYTHONPATH=service python3 -m unittest discover -s service/tests -p 'test_*.py' -v
+
+check: test service-test build

README.md

@@ -69,7 +69,9 @@ What is confirmed in the repository today:
 - canonical detection artifact generation with frontmatter
 - side-by-side support for Sigma, Splunk SPL, Microsoft Sentinel KQL, Elastic EQL, and Elastic ES|QL
 - repository content areas for detections, examples, knowledge, and reports
+- a shared [DetLab Detection Content Specification v1](docs/schema/detection-content-spec-v1.md) adapter with source hashing and generated-artifact provenance
 - Node-based tests for web copy/config behavior
+- Python contract tests covering every authored detection YAML file
 
 ## Features
 

docs/schema/detection-content-spec-v1.md

@@ -0,0 +1,49 @@
+# DetLab Detection Content Specification v1
+
+This repository implements the shared **DetLab Detection Content Specification v1** at:
+
+- `schemas/detlab-detection-content-v1.schema.json`
+
+The schema identifier is `https://schemas.detlab.dev/detection-content/v1.0.0/schema.json`.
+
+## Contract boundary
+
+The v1 contract is a portable, normalized interchange model. It does **not** replace the authored source format:
+
+- Rich DetLab detection YAML remains canonical in `DetLab-DAC`.
+- Sigma YAML remains canonical in `cybersecurity-playbook`.
+- Adapters normalize either source into the same v1 shape.
+- Splunk, Elastic, Kusto, and other target queries are derived artifacts, never competing authored sources.
+
+The existing DetLab authored schema (`schema_version: 2.0.0`) can therefore evolve without changing the stable cross-repository v1 interchange contract.
+
+## Required normalized fields
+
+- identity: `spec_version`, `kind`, `id`, `title`, `description`
+- lifecycle: `status`, `severity`, `authors`
+- behavior: `platforms`, `attack`, `logsource`, `logic`
+- canonical source provenance: `source.format`, `source.path`, `source.sha256`, `source.canonical`
+
+`logic.body` contains selections. `logic.condition` remains explicit so consumers do not need format-specific condition discovery.
+
+## Generated artifact provenance
+
+Every generated artifact records:
+
+- target and query language
+- rendered content and its SHA-256
+- canonical source SHA-256
+- contract version
+- converter package name and version
+
+An artifact is stale when its recorded source hash or contract version no longer matches the normalized canonical source.
+
+## Repository adapter
+
+`service/detlab/contract.py` normalizes rich DetLab YAML and provides provenance/staleness helpers.
+
+Tests:
+
+```bash
+PYTHONPATH=service python3 -m unittest discover -s service/tests -p 'test_*.py' -v
+```

schemas/detlab-detection-content-v1.schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://schemas.detlab.dev/detection-content/v1.0.0/schema.json",
+  "title": "DetLab Detection Content Specification v1",
+  "description": "Portable normalized detection contract. Authored source remains canonical; target queries are derived artifacts with provenance.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["spec_version", "kind", "id", "title", "description", "status", "severity", "authors", "platforms", "attack", "logsource", "logic", "source"],
+  "properties": {
+    "spec_version": {"const": "1.0.0"},
+    "kind": {"const": "detection"},
+    "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"},
+    "title": {"type": "string", "minLength": 1, "maxLength": 300},
+    "description": {"type": "string", "minLength": 1, "maxLength": 4000},
+    "status": {"enum": ["draft", "experimental", "test", "testing", "available", "validated", "stable", "deprecated", "unsupported"]},
+    "severity": {"enum": ["informational", "low", "medium", "high", "critical", "unknown"]},
+    "authors": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1, "maxLength": 200}},
+    "platforms": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}},
+    "attack": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["techniques", "tactics"],
+      "properties": {
+        "techniques": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^T[0-9]{4}(\\.[0-9]{3})?$"}},
+        "tactics": {"type": "array", "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z][a-z0-9-]*$"}}
+      }
+    },
+    "logsource": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["product"],
+      "properties": {
+        "product": {"type": "string", "minLength": 1, "maxLength": 100},
+        "category": {"type": "string", "minLength": 1, "maxLength": 100},
+        "service": {"type": "string", "minLength": 1, "maxLength": 100}
+      },
+      "anyOf": [{"required": ["category"]}, {"required": ["service"]}]
+    },
+    "logic": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["format", "body", "condition"],
+      "properties": {
+        "format": {"enum": ["sigma", "detlab-canonical"]},
+        "body": {"type": "object", "minProperties": 1},
+        "condition": {"type": "string", "minLength": 1, "maxLength": 1000}
+      }
+    },
+    "source": {"$ref": "#/$defs/source"},
+    "generated_artifacts": {"type": "array", "items": {"$ref": "#/$defs/generatedArtifact"}}
+  },
+  "$defs": {
+    "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["format", "path", "sha256", "canonical"],
+      "properties": {
+        "format": {"enum": ["sigma", "detlab-canonical"]},
+        "path": {"type": "string", "minLength": 1, "maxLength": 500, "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$"},
+        "sha256": {"$ref": "#/$defs/sha256"},
+        "canonical": {"const": true}
+      }
+    },
+    "converter": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": {"type": "string", "minLength": 1, "maxLength": 200},
+        "version": {"type": "string", "minLength": 1, "maxLength": 100}
+      }
+    },
+    "generatedArtifact": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["target", "language", "content", "content_sha256", "provenance"],
+      "properties": {
+        "target": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"},
+        "language": {"type": "string", "pattern": "^[a-z0-9][a-z0-9+|._-]*$"},
+        "content": {"type": "string", "minLength": 1},
+        "content_sha256": {"$ref": "#/$defs/sha256"},
+        "provenance": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["source_sha256", "spec_version", "converter"],
+          "properties": {
+            "source_sha256": {"$ref": "#/$defs/sha256"},
+            "spec_version": {"const": "1.0.0"},
+            "converter": {"$ref": "#/$defs/converter"}
+          }
+        }
+      }
+    }
+  }
+}

service/detlab/__init__.py

@@ -0,0 +1 @@
+"""DetLab service package."""

service/detlab/contract.py

@@ -0,0 +1,82 @@
+"""Adapters for DetLab Detection Content Specification v1."""
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any, Mapping
+
+SPEC_VERSION = "1.0.0"
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _as_authors(value: Any) -> list[str]:
+    values = value if isinstance(value, list) else [value]
+    return [str(item).strip() for item in values if str(item).strip()]
+
+
+def normalize_detlab_detection(detection: Mapping[str, Any], source_path: Path | str, source_bytes: bytes) -> dict[str, Any]:
+    canonical = detection.get("canonical_detection") or {}
+    analytics = canonical.get("analytics") or {}
+    logic = dict(analytics.get("logic") or {})
+    condition = str(analytics.get("condition") or logic.pop("condition", "")).strip()
+    attack = canonical.get("attack") or {}
+    primary = attack.get("primary") or {}
+    related = attack.get("related") or []
+    techniques = sorted({str(item.get("technique")) for item in [primary, *related] if item.get("technique")})
+    tactics = sorted({str(item.get("tactic")).lower().replace("_", "-") for item in [primary, *related] if item.get("tactic")})
+    logsource = {key: str(value) for key, value in (canonical.get("logsource") or {}).items() if key in {"product", "category", "service"} and value}
+    platforms = [str(value).lower().replace(" ", "-") for value in detection.get("platforms", []) if str(value).strip()]
+    if not platforms:
+        platforms = [str(logsource.get("product", "unknown")).lower().replace(" ", "-")]
+    return {
+        "spec_version": SPEC_VERSION,
+        "kind": "detection",
+        "id": str(detection.get("id", "")).strip(),
+        "title": str(detection.get("title", "")).strip(),
+        "description": str(detection.get("description", "")).strip(),
+        "status": str(detection.get("status", "draft")).lower(),
+        "severity": str(detection.get("severity", "unknown")).lower(),
+        "authors": _as_authors(detection.get("author", "unknown")),
+        "platforms": sorted(set(platforms)),
+        "attack": {"techniques": techniques, "tactics": tactics},
+        "logsource": logsource,
+        "logic": {"format": "detlab-canonical", "body": logic, "condition": condition},
+        "source": {
+            "format": "detlab-canonical",
+            "path": Path(source_path).as_posix(),
+            "sha256": _sha256(source_bytes),
+            "canonical": True,
+        },
+    }
+
+
+def build_generated_artifact(
+    normalized: Mapping[str, Any],
+    *,
+    target: str,
+    language: str,
+    content: str,
+    converter: Mapping[str, str],
+) -> dict[str, Any]:
+    return {
+        "target": target,
+        "language": language,
+        "content": content,
+        "content_sha256": _sha256(content.encode("utf-8")),
+        "provenance": {
+            "source_sha256": str(normalized["source"]["sha256"]),
+            "spec_version": SPEC_VERSION,
+            "converter": {"name": str(converter["name"]), "version": str(converter["version"])},
+        },
+    }
+
+
+def generated_artifact_is_stale(normalized: Mapping[str, Any], artifact: Mapping[str, Any]) -> bool:
+    provenance = artifact.get("provenance") or {}
+    return (
+        provenance.get("source_sha256") != normalized.get("source", {}).get("sha256")
+        or provenance.get("spec_version") != SPEC_VERSION
+    )

service/requirements.txt

@@ -0,0 +1,2 @@
+jsonschema==4.26.0
+PyYAML==6.0.3

service/tests/test_contract.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import hashlib
+import unittest
+from pathlib import Path
+
+import yaml
+from jsonschema import Draft202012Validator
+
+from detlab.contract import (
+    build_generated_artifact,
+    generated_artifact_is_stale,
+    normalize_detlab_detection,
+)
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+DETECTION_PATH = REPO_ROOT / "detections" / "windows" / "suspicious_encoded_powershell.yaml"
+SCHEMA_PATH = REPO_ROOT / "schemas" / "detlab-detection-content-v1.schema.json"
+
+
+class DetectionContentContractTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.source_bytes = DETECTION_PATH.read_bytes()
+        self.detection = yaml.safe_load(self.source_bytes)
+
+    def test_rich_detection_normalizes_to_shared_v1_contract(self) -> None:
+        normalized = normalize_detlab_detection(
+            self.detection,
+            DETECTION_PATH.relative_to(REPO_ROOT),
+            self.source_bytes,
+        )
+        schema = yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8"))
+        Draft202012Validator(schema).validate(normalized)
+
+        self.assertEqual(normalized["spec_version"], "1.0.0")
+        self.assertEqual(normalized["source"]["format"], "detlab-canonical")
+        self.assertEqual(normalized["source"]["sha256"], hashlib.sha256(self.source_bytes).hexdigest())
+        self.assertEqual(normalized["logic"]["condition"], "selection")
+        self.assertIn("T1059.001", normalized["attack"]["techniques"])
+
+    def test_all_authored_detections_normalize_to_shared_v1_contract(self) -> None:
+        schema = yaml.safe_load(SCHEMA_PATH.read_text(encoding="utf-8"))
+        validator = Draft202012Validator(schema)
+        paths = sorted((REPO_ROOT / "detections").rglob("*.yaml")) + sorted((REPO_ROOT / "detections").rglob("*.yml"))
+        self.assertGreater(len(paths), 0)
+        for path in paths:
+            with self.subTest(path=path.relative_to(REPO_ROOT)):
+                source_bytes = path.read_bytes()
+                normalized = normalize_detlab_detection(
+                    yaml.safe_load(source_bytes),
+                    path.relative_to(REPO_ROOT),
+                    source_bytes,
+                )
+                validator.validate(normalized)
+
+    def test_generated_artifact_staleness_uses_source_hash(self) -> None:
+        normalized = normalize_detlab_detection(
+            self.detection,
+            DETECTION_PATH.relative_to(REPO_ROOT),
+            self.source_bytes,
+        )
+        artifact = build_generated_artifact(
+            normalized,
+            target="splunk",
+            language="spl",
+            content="index=win powershell",
+            converter={"name": "pysigma-backend-splunk", "version": "2.1.0"},
+        )
+        self.assertFalse(generated_artifact_is_stale(normalized, artifact))
+        artifact["provenance"]["source_sha256"] = "0" * 64
+        self.assertTrue(generated_artifact_is_stale(normalized, artifact))
+
+
+if __name__ == "__main__":
+    unittest.main()