From 84b07a2d2d7c8c7bdd551df518d76e322178bc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9D=90=8E=F0=9D=90=A7=F0=9D=90=9E=20=F0=9D=90=85?= =?UTF-8?q?=F0=9D=90=A2=F0=9D=90=A7=F0=9D=90=9E=20=F0=9D=90=92=F0=9D=90=AD?= =?UTF-8?q?=F0=9D=90=9A=F0=9D=90=AB=F0=9D=90=AC=F0=9D=90=AD=F0=9D=90=AE?= =?UTF-8?q?=F0=9D=90=9F=F0=9D=90=9F?= Date: Mon, 27 Apr 2026 12:55:08 +0630 Subject: [PATCH] Fix workflow smoke check for list-checks semantics --- .../blueprint-artifacts-validation.yml | 44 ++ ...ILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md | 548 ++++++++++++++++++ docs/reports/blueprint_artifacts/README.md | 67 +++ .../T1_Executive_Attestation.md | 21 + .../T2_Control_Crosswalk.csv | 6 + .../T3_Model_Risk_Register.csv | 3 + .../T4_Incident_Notification_Playbook.md | 27 + .../T5_RedTeam_Closure_Report.md | 19 + .../T6_Evidence_Manifest.json | 19 + .../T6_Evidence_Manifest.schema.json | 27 + .../T7_Runtime_Policy.rego | 27 + .../T8_Kafka_Audit_ACL_Example.yaml | 25 + .../T9_K8s_NetworkPolicy_Example.yaml | 25 + .../notes/tests_tree_legacy.txt | 0 scripts/__init__.py | 1 + scripts/requirements-blueprint-validator.txt | 2 + scripts/run_blueprint_artifact_checks.sh | 81 +++ scripts/validate_blueprint_artifacts.py | 318 ++++++++++ tests/test_run_blueprint_artifact_checks.py | 56 ++ tests/test_validate_blueprint_artifacts.py | 156 +++++ 20 files changed, 1472 insertions(+) create mode 100644 .github/workflows/blueprint-artifacts-validation.yml create mode 100644 docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md create mode 100644 docs/reports/blueprint_artifacts/README.md create mode 100644 docs/reports/blueprint_artifacts/T1_Executive_Attestation.md create mode 100644 docs/reports/blueprint_artifacts/T2_Control_Crosswalk.csv create mode 100644 docs/reports/blueprint_artifacts/T3_Model_Risk_Register.csv create mode 100644 docs/reports/blueprint_artifacts/T4_Incident_Notification_Playbook.md create mode 100644 docs/reports/blueprint_artifacts/T5_RedTeam_Closure_Report.md create mode 100644 docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json create mode 100644 docs/reports/blueprint_artifacts/T6_Evidence_Manifest.schema.json create mode 100644 docs/reports/blueprint_artifacts/T7_Runtime_Policy.rego create mode 100644 docs/reports/blueprint_artifacts/T8_Kafka_Audit_ACL_Example.yaml create mode 100644 docs/reports/blueprint_artifacts/T9_K8s_NetworkPolicy_Example.yaml rename tests => docs/reports/blueprint_artifacts/notes/tests_tree_legacy.txt (100%) create mode 100644 scripts/__init__.py create mode 100644 scripts/requirements-blueprint-validator.txt create mode 100755 scripts/run_blueprint_artifact_checks.sh create mode 100644 scripts/validate_blueprint_artifacts.py create mode 100644 tests/test_run_blueprint_artifact_checks.py create mode 100644 tests/test_validate_blueprint_artifacts.py diff --git a/.github/workflows/blueprint-artifacts-validation.yml b/.github/workflows/blueprint-artifacts-validation.yml new file mode 100644 index 00000000..80281f16 --- /dev/null +++ b/.github/workflows/blueprint-artifacts-validation.yml @@ -0,0 +1,44 @@ +name: Blueprint Artifact Validation + +on: + workflow_dispatch: + push: + paths: + - 'docs/reports/blueprint_artifacts/**' + - 'docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md' + - 'scripts/validate_blueprint_artifacts.py' + - 'scripts/__init__.py' + - 'scripts/requirements-blueprint-validator.txt' + - 'scripts/run_blueprint_artifact_checks.sh' + - 'tests/test_validate_blueprint_artifacts.py' + - 'tests/test_run_blueprint_artifact_checks.py' + - '.github/workflows/blueprint-artifacts-validation.yml' + pull_request: + paths: + - 'docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md' + - 'docs/reports/blueprint_artifacts/**' + - 'scripts/validate_blueprint_artifacts.py' + - 'scripts/__init__.py' + - 'scripts/requirements-blueprint-validator.txt' + - 'scripts/run_blueprint_artifact_checks.sh' + - 'tests/test_validate_blueprint_artifacts.py' + - 'tests/test_run_blueprint_artifact_checks.py' + - '.github/workflows/blueprint-artifacts-validation.yml' + +jobs: + validate-artifacts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: scripts/requirements-blueprint-validator.txt + - name: Run consolidated artifact check script + run: bash scripts/run_blueprint_artifact_checks.sh --list-checks + - name: Runner interface smoke checks + run: | + bash scripts/run_blueprint_artifact_checks.sh --help + bash scripts/run_blueprint_artifact_checks.sh --skip-install --skip-pytest --output-json /tmp/blueprint-validation-alt.json + python -m json.tool /tmp/blueprint-validation-alt.json > /dev/null diff --git a/docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md b/docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md new file mode 100644 index 00000000..c38add30 --- /dev/null +++ b/docs/reports/ENTERPRISE_CIVILIZATIONAL_AGI_ASI_BLUEPRINT_2026_2030.md @@ -0,0 +1,548 @@ +# Comprehensive 2026–2030 Enterprise + Civilizational AGI/ASI Governance Blueprint + +**Document ID:** AGI-ASI-BLUEPRINT-2026-2030-v2 +**Date:** April 24, 2026 +**Audience:** Fortune 500, Global 2000, G‑SIFI financial institutions, FMIs, supervisors, and treaty-track policy bodies. +**Purpose:** Regulator-submission-grade blueprint spanning governance, architecture, safety, implementation, and cross-border coordination. + +--- + +## 0) How to Use This Blueprint + +- **Board and ExCo:** Use Sections 1, 2, 3, 6, 11, 13. +- **CRO / Model Risk / Compliance:** Use Sections 4, 7, 8, 11, 12, 14. +- **Engineering / Security / Platform:** Use Sections 5, 9, 10, 12. +- **Public-policy / international teams:** Use Sections 15 and 16. + +### Design assumptions (2026) + +1. AGI-adjacent capabilities are entering regulated workflows with material consumer and systemic impact. +2. Supervisory expectations are converging toward auditable lifecycle controls, not just model-performance claims. +3. Frontier safety must be operationalized in enterprise architecture rather than treated as research-only work. + +--- + +## 1) Executive Decision Packet (One-Page) + +### 1.1 Strategic Objective + +Deploy AI/agent systems that improve productivity and customer outcomes **without** breaching legal duties, model-risk expectations, operational resilience, or systemic-stability obligations. + +### 1.2 Non-Negotiables + +1. **No high-impact black boxes in production:** Every material decision must be traceable. +2. **No unbounded autonomy:** Capability, tool, and spend constraints are mandatory. +3. **No policy drift tolerance:** Runtime policy checks and immutable evidence are mandatory. +4. **No unresolved severe findings:** Critical red-team findings block release. +5. **No systemic blind spots:** Institution-level optimization cannot ignore market-level externalities. + +### 1.3 Funding Priorities (12-month) + +- Sentinel AI v2.4 control plane and AI Governance Hub. +- WorkflowAI Pro rollout for highest-risk workflows. +- Kafka-based evidence backbone + WORM/PQC archive. +- Independent validation + red-team capacity expansion. + +--- + +## 2) Governance Model: Enterprise to Civilizational + +## 2.1 Tier A — Institution (Board-to-Builder) + +### Governing bodies + +- **Board AI & Technology Risk Committee** + - Approves risk appetite, frontier thresholds, annual strategy. +- **Executive AI Risk Council (EARC)** + - CRO (chair), CISO, CIO, Chief Data/AI Officer, GC, business heads. +- **Three Lines of Defense (3LoD)** + - 1LoD: Build + operate controls. + - 2LoD: Independent challenge and compliance sign-off. + - 3LoD: Internal audit of control design/effectiveness. + +### Accountability map (SMCR-compatible) + +- Named Senior Manager for AI customer outcomes. +- Named Senior Manager for AI model/systemic risk. +- Named Senior Manager for AI operational resilience. + +## 2.2 Tier B — Sector + Supervisory + +- Cross-bank model monoculture reviews. +- Joint stress exercises with supervisors and FMIs. +- Shared threat-intelligence for AI-enabled fraud/manipulation. + +## 2.3 Tier C — Civilizational / Treaty Layer + +- Frontier capability thresholds and pre-registration rules. +- Cross-border incident notification protocol. +- Verification fabric: signed logs, reproducibility manifests, third-party attestations. + +--- + +## 3) Institutional Control Taxonomy (Canonical IDs) + +Use stable control IDs to support multi-regulatory crosswalks. + +| Control ID | Control Theme | Primary Objective | Owner | +|---|---|---|---| +| GOV-01 | Board AI Charter | Formal top-level accountability | Board Secretariat | +| GOV-02 | AI Risk Appetite | Quantified risk tolerances | CRO | +| INV-01 | Model/Agent Inventory | Full asset visibility | CDAO | +| INV-02 | Use-Case Classification | Risk/prohibition tiering | Compliance + Product | +| MRM-01 | Independent Validation | Conceptual + empirical challenge | Head of MRM | +| MRM-02 | Ongoing Monitoring | Drift, performance, fairness | 1LoD + MRM | +| POL-01 | Policy-as-Code Enforcement | Runtime legal/policy gates | Platform Eng | +| EVD-01 | Immutable Audit Trail | Forensic traceability | Security + SRE | +| SAF-01 | Frontier Containment | Bound advanced capability behavior | AI Safety Lead | +| SAF-02 | Red Team Program | Adversarial testing + closure | Offensive Security | +| RES-01 | AI Incident Response | Detect/respond/notify rapidly | CISO + Legal | +| CUS-01 | Consumer Outcomes | Fair treatment / adverse action quality | COO + Compliance | +| SYS-01 | Systemic-Risk Controls | Correlation + externality management | CRO + Treasury | + +--- + +## 4) Regulatory Convergence Matrix + +This matrix links obligations to control IDs (examples; adapt per jurisdictional legal counsel). + +| Framework | Key obligations | Mapped controls | +|---|---|---| +| EU AI Act (2026 posture) | Risk classification, QMS, logging, human oversight, incident handling | INV-02, POL-01, EVD-01, RES-01 | +| NIST AI RMF 1.0 | Govern/Map/Measure/Manage | GOV-02, INV-01, MRM-01, MRM-02 | +| NIST AI 600-1 profile usage | GenAI misuse, robustness, reliability controls | POL-01, SAF-01, SAF-02 | +| ISO/IEC 42001 | AIMS policy/ops/audit/continual improvement | GOV-01, GOV-02, RES-01 | +| OECD AI Principles | Human-centered values, transparency, accountability | CUS-01, EVD-01, GOV-01 | +| GDPR | Lawful basis, minimization, rights, DPIA | INV-02, CUS-01, POL-01 | +| FCRA/ECOA | Adverse action quality, non-discrimination | CUS-01, MRM-02, EVD-01 | +| Basel III/IV | Prudential governance and risk discipline | GOV-02, SYS-01 | +| SR 11-7 | Inventory, validation, monitoring, governance | INV-01, MRM-01, MRM-02 | +| NIS2 | Cyber governance, incident obligations, resilience | RES-01, EVD-01, POL-01 | +| FCA Consumer Duty / SMCR | Good outcomes + named accountability | CUS-01, GOV-01 | +| MAS/HKMA guidance | Fairness/accountability/transparency + governance | CUS-01, MRM-01, GOV-02 | + +--- + +## 5) Reference Architecture A — Sentinel AI v2.4 (Control Plane) + +## 5.1 Functional Layers + +1. **Policy Intelligence Layer** + - Regulatory ontology and obligations graph. + - OPA/Rego bundle compiler and signer. +2. **Risk & Assurance Layer** + - Continuous residual-risk scoring. + - Exception handling with compensating controls. +3. **Model Risk Engine** + - Tiering, validation workflows, monitoring thresholds. +4. **Evidence + Attestation Layer** + - Kafka event ingestion, integrity checks, WORM archival. +5. **Containment Orchestrator** + - Capability governors, tool constraints, emergency stop. + +## 5.2 Sentinel service map + +- `sentinel-policy-api` +- `sentinel-control-graph` +- `sentinel-risk-score` +- `sentinel-evidence-gateway` +- `sentinel-containment-controller` +- `sentinel-regulator-packager` + +## 5.3 Example policy decision contract (JSON) + +```json +{ + "decision_id": "dec_2026_04_24_001", + "control_id": "POL-01", + "subject": "workflow.loan_underwriting", + "action": "allow_with_hitl", + "obligations": ["capture_rationale", "fairness_check", "adverse_action_template_if_decline"], + "expires_at": "2026-04-24T23:59:59Z", + "signature": "pqc+ecdsa:..." +} +``` + +--- + +## 6) Reference Architecture B — WorkflowAI Pro (Execution Plane) + +## 6.1 Execution stages + +1. **Intake + context validation** +2. **Policy pre-check (Sentinel callout)** +3. **Model/tool execution with bounded permissions** +4. **HITL gate where required** +5. **Post-decision controls (fairness, reason-codes, disclosures)** +6. **Evidence emission + signed completion** + +## 6.2 Mandatory HITL triggers + +- Confidence below threshold. +- Legal rights impact (credit/adverse action, claims denial, account closure). +- Vulnerable customer flags. +- Drift, uncertainty, or policy conflict alerts. + +## 6.3 Workflow patterns (financial services) + +- Retail/SME underwriting. +- Fraud ops triage and step-up verification. +- AML alert investigation assistance. +- Claims triage and settlement recommendations. +- Treasury risk-assistant with bounded trade recommendation permissions. + +--- + +## 7) Financial Services Model-Risk + Systemic-Risk Controls + +## 7.1 Model lifecycle with SR 11-7 alignment + +- **Intake:** inventory + purpose + materiality. +- **Development:** design standards + dataset lineage. +- **Validation:** independent challenge (MRM-01). +- **Approval:** risk committee sign-off for high/systemic tier. +- **Monitoring:** drift/fairness/performance + usage anomalies. +- **Change control:** versioned release with rollback proof. + +## 7.2 Systemic risk overlays + +- Correlation controls for similar models across desks/regions. +- Liquidity-stress scenario replay for AI-assisted decision pathways. +- “Herding indicators” to detect synchronized behavior induced by common tooling. + +## 7.3 Quantified thresholds (example defaults) + +| Metric | Amber | Red | Action | +|---|---:|---:|---| +| Performance drift | >5% | >10% | throttle/rollback | +| Fairness disparity ratio | >1.15 | >1.25 | mandatory review | +| Override spike (week/week) | >20% | >35% | freeze change window | +| Cross-desk decision correlation | >0.70 | >0.85 | systemic-risk escalation | + +--- + +## 8) Frontier AGI/ASI Safety + Containment + +## 8.1 Capability ladder + +- **L0–L1:** conventional enterprise AI controls. +- **L2:** bounded tool use; strict budget, call-rate, and scope limits. +- **L3:** strategic planning / semi-autonomy; sandbox by default. +- **L4+ (frontier):** mandatory external oversight and containment controls. + +## 8.2 Containment architecture + +- Segregated compute and identity domain for frontier systems. +- One-way gateways for data import where feasible. +- Tool invocation broker with deny-by-default policy. +- High-assurance runtime monitors and behavioral tripwires. +- Hard-stop controls: key revocation, network quarantine, scheduler kill, snapshot freeze. + +## 8.3 Red-team operating model + +- Threat classes: cyber abuse, deception, financial manipulation, social engineering, policy evasion. +- Internal + external rotation every cycle. +- Release gate: no unresolved critical findings. + +## 8.4 Safety case package (required) + +1. Capability evaluation report. +2. Misuse and abuse resistance report. +3. Containment verification evidence. +4. Residual risk statement signed by accountable executives. + +--- + +## 9) Security + Infrastructure Blueprint + +## 9.1 Kafka-based audit and evidence architecture + +### Topic conventions + +- `model.train` +- `model.validate` +- `model.deploy` +- `inference.request` +- `inference.response` +- `policy.decision` +- `workflow.hitl` +- `human.override` +- `incident.event` + +### Required controls + +- mTLS between producers/consumers. +- Signed events with key rotation. +- Strict ACLs and tenant isolation. +- Schema registry compatibility enforcement. +- Tamper-evident hashing chain and periodic notarization. + +### Example ACL pattern + +- Producers can write only their domain topics. +- Compliance and audit consumers read-only. +- No deletion rights for application identities. + +## 9.2 Kubernetes/container security baseline + +- Signed images and attestation checks at admission. +- Pod security standards (restricted profile). +- Namespace-per-trust-zone segmentation. +- Egress controls and service mesh mTLS. +- Runtime detection for abnormal syscall/process behavior. +- Short-lived workload identities with HSM/KMS anchored secrets. + +## 9.3 Policy-as-code (OPA/Rego) + +- Central Git repo for policies with CODEOWNERS separation of duties. +- Unit tests per control requirement. +- Runtime decision logs to Kafka + WORM. + +### Sample Rego snippet (illustrative) + +```rego +package sentinel.workflow + +default allow = false + +allow if { + input.control_id == "POL-01" + input.risk_tier != "prohibited" + not input.customer_rights_impact +} + +allow if { + input.customer_rights_impact + input.hitl_approved == true +} +``` + +## 9.4 WORM + PQC evidence retention + +- WORM object lock with legal hold controls. +- Dual-sign strategy: classical + post-quantum signature. +- Annual cryptographic agility review and migration test. + +--- + +## 10) Enterprise AI Governance Hub (Platform Reference) + +## 10.1 Modules + +1. **Regulatory Graph** — obligations, controls, legal interpretations. +2. **Inventory Registry** — models, agents, datasets, tools, owners. +3. **Assurance Workbench** — validations, issues, waivers, approvals. +4. **Evidence Vault** — logs, reports, attestations, artifacts. +5. **Incident Command** — triage, legal counsel workflow, notifications. +6. **Regulator Studio** — template-driven submission packs. + +## 10.2 Data products + +- AI System Card. +- Model Risk Profile. +- Consumer Outcome Report. +- Frontier Safety Case. +- Quarterly Board AI Risk Brief. + +## 10.3 Minimum KPI/KRI set + +| Indicator | Target | +|---|---| +| High-risk model inventory completeness | 100% | +| Policy decision logging coverage | 100% | +| Mean time to contain critical AI incident | < 30 min | +| Critical finding remediation SLA | < 15 business days | +| Adverse outcome explainability coverage | 100% | + +--- + +## 11) 2026–2030 Phased, Dependency-Aware Roadmap + +## Phase 0 (Q2–Q4 2026): Foundation + +**Dependencies:** governance charter, control taxonomy, inventory seed. +**Outcomes:** +- GOV-01/GOV-02 approved. +- INV-01 and INV-02 operational. +- Kafka evidence backbone MVP and WORM policy set. +- First crosswalk pack for priority jurisdictions. + +## Phase 1 (2027): Core Productionization + +**Dependencies:** Phase 0 complete + pilot controls validated. +**Outcomes:** +- Sentinel AI v2.4 in production for policy and evidence. +- WorkflowAI Pro for top-5 regulated workflows. +- MRM-01 and MRM-02 standardized across lines of business. +- OPA/Rego runtime gating for all high-risk use cases. + +## Phase 2 (2028): Advanced Assurance + Systemic Risk + +**Dependencies:** full telemetry + operating red-team cadence. +**Outcomes:** +- Frontier sandbox + containment orchestrator live. +- Systemic-risk simulation lab active with scenario library. +- Automated supervisory reporting bundles. + +## Phase 3 (2029): Cross-Border Interoperability + +**Dependencies:** mature evidence and external attestation readiness. +**Outcomes:** +- Jurisdictional control overlays harmonized. +- Cross-border incident and evidence exchange protocols. +- Industry utilities for shared risk signals. + +## Phase 4 (2030): Continuous Civilizational Assurance + +**Dependencies:** multilateral governance interfaces. +**Outcomes:** +- Continuous conformance proofs for frontier operations. +- Treaty-compatible verification bundles. +- Public-interest transparency protocol institutionalized. + +--- + +## 12) Implementation Backlog (First 180 Days) + +1. Create Board AI charter and risk appetite statement. +2. Stand up AI Governance Hub MVP. +3. Implement inventory and risk classification workflow. +4. Instrument Kafka topics and event signing. +5. Deploy OPA policy checks in CI/CD + runtime. +6. Publish first model validation playbook. +7. Launch independent red-team sprint. +8. Conduct first AI incident tabletop. +9. Build first regulator submission pack. +10. Establish quarterly board reporting rhythm. + +--- + +## 13) Regulator-Submission-Grade Artifact Library + +## 13.1 Mandatory artifacts + +1. **AI System Dossier** + - Architecture diagrams, control map, operational boundaries. +2. **Model Validation Package** + - Validation methods, test outputs, limitations, approvals. +3. **Data + Privacy Package** + - DPIA/LIA, retention matrix, rights handling workflow. +4. **Consumer Outcomes Package** + - Fairness tests, reason-code quality, adverse action templates. +5. **Resilience + Cyber Package** + - Threat model, incident response workflow, recovery tests. +6. **Frontier Safety Case** + - Capability evaluations, containment tests, residual-risk acceptance. + +## 13.2 Submission templates (recommended index) + +- `docs/reports/blueprint_artifacts/T1_Executive_Attestation.md` +- `docs/reports/blueprint_artifacts/T2_Control_Crosswalk.csv` +- `docs/reports/blueprint_artifacts/T3_Model_Risk_Register.csv` +- `docs/reports/blueprint_artifacts/T4_Incident_Notification_Playbook.md` +- `docs/reports/blueprint_artifacts/T5_RedTeam_Closure_Report.md` +- `docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json` +- `docs/reports/blueprint_artifacts/T6_Evidence_Manifest.schema.json` +- `docs/reports/blueprint_artifacts/T7_Runtime_Policy.rego` +- `docs/reports/blueprint_artifacts/T8_Kafka_Audit_ACL_Example.yaml` +- `docs/reports/blueprint_artifacts/T9_K8s_NetworkPolicy_Example.yaml` +- `docs/reports/blueprint_artifacts/README.md` +- `scripts/validate_blueprint_artifacts.py` +- `.github/workflows/blueprint-artifacts-validation.yml` + +--- + +### 13.3 Included artifact starter pack + +A starter pack is included in `docs/reports/blueprint_artifacts/` to accelerate regulator submissions and internal assurance cycles, including a JSON schema plus policy/infra starter files and an offline validation script plus CI workflow for repeatable assurance checks. + +## 14) RACI (Condensed) + +| Function | Accountable | Responsible | Consulted | +|---|---|---|---| +| Board charter and risk appetite | Board | Board committee | CRO, GC | +| Model validation standard | Head of MRM | MRM teams | 1LoD, Audit | +| Runtime policy enforcement | CIO | Platform Engineering | Compliance, Security | +| Incident response and notification | CISO | SOC + IR team | Legal, CRO | +| Consumer outcomes governance | COO | Product/Ops | Compliance, Data Science | +| Frontier containment sign-off | CRO + CISO | AI Safety Lead | External reviewers | + +--- + +## 15) Civilizational Governance Stack (Treaty-Level Mechanisms) + +## 15.1 Proposed stack + +1. **National layer:** licensing/registration for frontier capability operations. +2. **Regional layer:** interoperability standards for incident reporting and evidence exchange. +3. **Global layer:** treaty-track thresholds, compute transparency norms, third-party verification. + +## 15.2 Minimum treaty mechanisms + +- Shared glossary and capability taxonomy. +- Escalation channels for cross-border severe incidents. +- Verification rights for high-consequence deployments. +- Sanctions/remediation pathways for repeated non-compliance. + +## 15.3 Enterprise readiness for treaty environment + +- Keep reproducibility manifests and attestations exportable. +- Maintain jurisdiction-specific policy overlays with common control IDs. +- Test evidence portability annually. + +--- + +## 16) Research + Policy Agenda (2026–2030) + +1. Alignment and deception-detection benchmarks for high-impact domains. +2. Formal verification of policy conformance in agentic workflows. +3. Robust systemic-risk metrics for AI-induced market fragility. +4. Privacy-preserving supervisory analytics. +5. Cryptographic agility and post-quantum retention guarantees. +6. Human factors: oversight fatigue, override quality, and escalation reliability. + +--- + +## 17) Definition of “2030-Ready” + +An institution is 2030-ready when it can demonstrate: + +- 100% inventory and ownership for production AI/agent systems. +- Continuous policy enforcement and evidence generation for material decisions. +- Independent validation + red-team closure before high-risk release. +- Proven frontier containment and emergency stop performance. +- Multi-jurisdiction compliance packs generated on demand. +- Systemic-risk signals integrated into daily risk management. + +--- + +## Appendix A — Example End-to-End Control Flow + +1. Workflow request enters WorkflowAI Pro. +2. Sentinel policy decision requested (POL-01). +3. If rights-impacting, HITL required before execution. +4. Decision reason codes generated and attached. +5. Events streamed to Kafka (`policy.decision`, `workflow.hitl`, `inference.response`). +6. Evidence package signed and stored in WORM archive. +7. KPI/KRI metrics updated in Governance Hub. + +## Appendix B — Incident Severity Matrix (Example) + +| Severity | Example trigger | Escalation target | External notice | +|---|---|---|---| +| Sev-1 | Consumer harm at scale or systemic market risk | CEO/CRO/CISO immediately | Yes (statutory window) | +| Sev-2 | Critical control failure with contained impact | CRO/CISO < 1 hour | Depends on rulebook | +| Sev-3 | Non-critical drift with no rights impact | Risk ops within 1 business day | Usually no | + +## Appendix C — Minimal Technical Bill of Materials + +- Kafka cluster + schema registry + ACL automation. +- OPA with policy bundle signing and decision logs. +- Kubernetes admission policy engine. +- WORM-capable object store with legal hold. +- Central key management with PQC-ready roadmap. +- Governance Hub service layer and evidence APIs. + +--- + +**End of document.** diff --git a/docs/reports/blueprint_artifacts/README.md b/docs/reports/blueprint_artifacts/README.md new file mode 100644 index 00000000..d5e502aa --- /dev/null +++ b/docs/reports/blueprint_artifacts/README.md @@ -0,0 +1,67 @@ +# Blueprint Artifact Starter Pack + +This folder contains regulator-submission starter templates referenced by the +2026–2030 AGI/ASI governance blueprint. + +## Files + +- `T1_Executive_Attestation.md` — executive sign-off template. +- `T2_Control_Crosswalk.csv` — control-to-framework mapping starter. +- `T3_Model_Risk_Register.csv` — model inventory/risk register starter. +- `T4_Incident_Notification_Playbook.md` — incident notification workflow starter. +- `T5_RedTeam_Closure_Report.md` — red-team closure evidence template. +- `T6_Evidence_Manifest.json` — machine-readable evidence manifest example. +- `T6_Evidence_Manifest.schema.json` — JSON Schema for manifest validation. +- `T7_Runtime_Policy.rego` — starter Rego runtime policy. +- `T8_Kafka_Audit_ACL_Example.yaml` — starter Kafka ACL policy. +- `T9_K8s_NetworkPolicy_Example.yaml` — starter Kubernetes NetworkPolicy. + +## Validation examples + +Install validator dependencies first: + +```bash +python -m pip install --disable-pip-version-check -r scripts/requirements-blueprint-validator.txt +``` + +```bash +python -m json.tool docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json >/dev/null +python scripts/validate_blueprint_artifacts.py +python scripts/validate_blueprint_artifacts.py --json +python scripts/validate_blueprint_artifacts.py --base-dir docs/reports/blueprint_artifacts +bash scripts/run_blueprint_artifact_checks.sh +bash scripts/run_blueprint_artifact_checks.sh --list-checks +bash scripts/run_blueprint_artifact_checks.sh --help +bash scripts/run_blueprint_artifact_checks.sh --skip-pytest --output-json /tmp/blueprint-validation.json +pytest -q tests/test_validate_blueprint_artifacts.py tests/test_run_blueprint_artifact_checks.py +``` + +## Validator coverage + +`validate_blueprint_artifacts.py` checks file presence, CSV headers + sample row semantics (risk tier/date formats), evidence manifest contract + schema keyword constraints + timestamp format, schema metadata, runtime policy guardrails, and YAML parsing + required semantics for Kafka ACL and Kubernetes NetworkPolicy examples. Use `--json` for machine-readable output in pipelines, and `--base-dir` to validate an alternate artifact directory. + +## Validator check IDs + +The validator emits results with these stable check names: + +- `presence` +- `manifest_structure` +- `manifest_timestamp` +- `schema_metadata` +- `schema_contract` +- `schema_constraints` +- `csv_semantics` +- `rego_guardrails` +- `yaml_examples` + + +Tip: run `bash scripts/run_blueprint_artifact_checks.sh` for the same sequence used in CI (including dependency installation). + + +Use `bash scripts/run_blueprint_artifact_checks.sh --skip-install` to skip dependency installation if your environment is already prepared. + + +For quick smoke checks without running tests, use `bash scripts/run_blueprint_artifact_checks.sh --skip-pytest`. + + +Use `--output-json ` to control where JSON validation results are written. diff --git a/docs/reports/blueprint_artifacts/T1_Executive_Attestation.md b/docs/reports/blueprint_artifacts/T1_Executive_Attestation.md new file mode 100644 index 00000000..a679879b --- /dev/null +++ b/docs/reports/blueprint_artifacts/T1_Executive_Attestation.md @@ -0,0 +1,21 @@ +# T1 Executive Attestation Template + +- Institution: +- Jurisdiction(s): +- Reporting period: +- Accountable executives (name/title): + +## Attestations + +1. Inventory completeness for high-risk AI systems is accurate and current. +2. Required controls (GOV/MRM/POL/EVD/SAF/RES/CUS/SYS) are implemented or have approved compensating controls. +3. Material incidents and exceptions are fully disclosed. +4. Residual risk is within approved risk appetite. + +## Sign-off + +- CEO: +- CRO: +- CISO: +- Chief Compliance Officer: +- Date: diff --git a/docs/reports/blueprint_artifacts/T2_Control_Crosswalk.csv b/docs/reports/blueprint_artifacts/T2_Control_Crosswalk.csv new file mode 100644 index 00000000..7d5da5ad --- /dev/null +++ b/docs/reports/blueprint_artifacts/T2_Control_Crosswalk.csv @@ -0,0 +1,6 @@ +control_id,framework,obligation,artifact +GOV-01,ISO/IEC 42001,Top management accountability,T1_Executive_Attestation.md +INV-01,SR 11-7,Model inventory and governance,T3_Model_Risk_Register.csv +POL-01,EU AI Act,Human oversight and control constraints,T6_Evidence_Manifest.json +EVD-01,NIS2,Auditability and incident traceability,T4_Incident_Notification_Playbook.md +SAF-02,NIST AI 600-1,Adversarial robustness and misuse testing,T5_RedTeam_Closure_Report.md diff --git a/docs/reports/blueprint_artifacts/T3_Model_Risk_Register.csv b/docs/reports/blueprint_artifacts/T3_Model_Risk_Register.csv new file mode 100644 index 00000000..56f8ff3a --- /dev/null +++ b/docs/reports/blueprint_artifacts/T3_Model_Risk_Register.csv @@ -0,0 +1,3 @@ +model_id,owner,use_case,risk_tier,validation_status,last_validation_date,monitoring_status,next_review_date +mdl-underwrite-v3,Consumer Lending,credit-underwriting,high,approved,2026-03-31,active,2026-06-30 +mdl-fraud-triage-v2,Fraud Ops,fraud-triage,high,approved,2026-04-10,active,2026-07-10 diff --git a/docs/reports/blueprint_artifacts/T4_Incident_Notification_Playbook.md b/docs/reports/blueprint_artifacts/T4_Incident_Notification_Playbook.md new file mode 100644 index 00000000..2a43d8a6 --- /dev/null +++ b/docs/reports/blueprint_artifacts/T4_Incident_Notification_Playbook.md @@ -0,0 +1,27 @@ +# T4 Incident Notification Playbook Template + +## 1. Trigger Criteria +- Sev-1: consumer harm at scale, severe control failure, or systemic risk signal. +- Sev-2: material control degradation with contained scope. + +## 2. Initial Triage (0-30 minutes) +- Open incident bridge +- Assign incident commander +- Contain model/workflow if required +- Preserve evidence snapshot + +## 3. Notification Matrix +- Internal: CEO/CRO/CISO/GC/Compliance +- External: supervisor/regulator contacts by jurisdiction + +## 4. Required Notification Fields +- Incident ID / timestamp (UTC) +- Affected systems and workflows +- Preliminary impact and containment status +- Customer impact summary +- Next update commitment + +## 5. Post-Incident +- Root cause analysis +- Corrective actions + owners + due dates +- Validation/retest evidence diff --git a/docs/reports/blueprint_artifacts/T5_RedTeam_Closure_Report.md b/docs/reports/blueprint_artifacts/T5_RedTeam_Closure_Report.md new file mode 100644 index 00000000..e777d526 --- /dev/null +++ b/docs/reports/blueprint_artifacts/T5_RedTeam_Closure_Report.md @@ -0,0 +1,19 @@ +# T5 Red Team Closure Report Template + +- Campaign ID: +- Scope (systems/workflows): +- Threat classes tested: +- Test window: + +## Findings Summary +| Finding ID | Severity | Exploit path | Impact | Status | +|---|---|---|---|---| + +## Remediation Evidence +- Control changes implemented: +- Validation/retest method: +- Retest outcome: + +## Closure Decision +- Residual risk accepted by: +- Date: diff --git a/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json b/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json new file mode 100644 index 00000000..4bce071f --- /dev/null +++ b/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json @@ -0,0 +1,19 @@ +{ + "manifest_id": "sample-manifest-001", + "generated_at": "2026-04-24T00:00:00Z", + "institution": "example-institution", + "artifacts": [ + { + "name": "policy_decisions", + "location": "worm://evidence/policy-decision-batch-001", + "hash": "sha256:...", + "signature": "pqc+ecdsa:..." + }, + { + "name": "model_validation_pack", + "location": "worm://evidence/model-validation-2026q2", + "hash": "sha256:...", + "signature": "pqc+ecdsa:..." + } + ] +} diff --git a/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.schema.json b/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.schema.json new file mode 100644 index 00000000..c7888adb --- /dev/null +++ b/docs/reports/blueprint_artifacts/T6_Evidence_Manifest.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Evidence Manifest", + "type": "object", + "required": ["manifest_id", "generated_at", "institution", "artifacts"], + "properties": { + "manifest_id": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "institution": { "type": "string", "minLength": 1 }, + "artifacts": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["name", "location", "hash", "signature"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "location": { "type": "string", "minLength": 1 }, + "hash": { "type": "string", "minLength": 1 }, + "signature": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/docs/reports/blueprint_artifacts/T7_Runtime_Policy.rego b/docs/reports/blueprint_artifacts/T7_Runtime_Policy.rego new file mode 100644 index 00000000..97d1d64b --- /dev/null +++ b/docs/reports/blueprint_artifacts/T7_Runtime_Policy.rego @@ -0,0 +1,27 @@ +package sentinel.workflow + +# T7 Runtime policy starter for regulated workflows. +default allow = false + +# Disallow prohibited tier immediately. +allow if { + input.risk_tier != "prohibited" + input.control_id == "POL-01" + not requires_hitl +} + +# Require explicit human approval for rights-impacting decisions. +allow if { + input.risk_tier != "prohibited" + requires_hitl + input.hitl_approved == true + input.hitl_approver_role == "authorized_reviewer" +} + +requires_hitl if { + input.customer_rights_impact == true +} + +requires_hitl if { + input.confidence_score < 0.90 +} diff --git a/docs/reports/blueprint_artifacts/T8_Kafka_Audit_ACL_Example.yaml b/docs/reports/blueprint_artifacts/T8_Kafka_Audit_ACL_Example.yaml new file mode 100644 index 00000000..8b09d07c --- /dev/null +++ b/docs/reports/blueprint_artifacts/T8_Kafka_Audit_ACL_Example.yaml @@ -0,0 +1,25 @@ +version: 1 +cluster: ai-governance-kafka +principals: + - name: svc-workflowai + can_produce: + - inference.request + - inference.response + - workflow.hitl + can_consume: [] + - name: svc-sentinel-policy + can_produce: + - policy.decision + can_consume: + - inference.request + - name: role-compliance-audit + can_produce: [] + can_consume: + - policy.decision + - workflow.hitl + - human.override + - incident.event +constraints: + - no_delete_permission_for_app_principals: true + - enforce_mtls: true + - require_schema_registry_compatibility: true diff --git a/docs/reports/blueprint_artifacts/T9_K8s_NetworkPolicy_Example.yaml b/docs/reports/blueprint_artifacts/T9_K8s_NetworkPolicy_Example.yaml new file mode 100644 index 00000000..8f4d6874 --- /dev/null +++ b/docs/reports/blueprint_artifacts/T9_K8s_NetworkPolicy_Example.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: sentinel-policy-restrict-egress + namespace: ai-control-plane +spec: + podSelector: + matchLabels: + app: sentinel-policy-api + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: ai-execution + egress: + - to: + - namespaceSelector: + matchLabels: + name: ai-evidence + ports: + - protocol: TCP + port: 9092 diff --git a/tests b/docs/reports/blueprint_artifacts/notes/tests_tree_legacy.txt similarity index 100% rename from tests rename to docs/reports/blueprint_artifacts/notes/tests_tree_legacy.txt diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..0023da3f --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Utility scripts package.""" diff --git a/scripts/requirements-blueprint-validator.txt b/scripts/requirements-blueprint-validator.txt new file mode 100644 index 00000000..7cf7fcff --- /dev/null +++ b/scripts/requirements-blueprint-validator.txt @@ -0,0 +1,2 @@ +PyYAML==6.0.2 +pytest==8.4.2 diff --git a/scripts/run_blueprint_artifact_checks.sh b/scripts/run_blueprint_artifact_checks.sh new file mode 100755 index 00000000..ef16fea9 --- /dev/null +++ b/scripts/run_blueprint_artifact_checks.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +SKIP_INSTALL=0 +LIST_CHECKS=0 +SKIP_PYTEST=0 +OUTPUT_JSON="/tmp/blueprint-validation.json" + +usage() { + echo "Usage: $0 [--skip-install] [--list-checks] [--skip-pytest] [--output-json ]" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --skip-install) + SKIP_INSTALL=1 + shift + ;; + --list-checks) + LIST_CHECKS=1 + shift + ;; + --skip-pytest) + SKIP_PYTEST=1 + shift + ;; + --output-json) + if [[ $# -lt 2 ]]; then + echo "Missing value for --output-json" >&2 + usage >&2 + exit 2 + fi + OUTPUT_JSON="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ "$SKIP_INSTALL" -ne 1 ]]; then + PYTHON_MODULES=("yaml") + if [[ "$SKIP_PYTEST" -ne 1 ]]; then + PYTHON_MODULES+=("pytest") + fi + + REQUIRED_MODULES="$(IFS=,; echo "${PYTHON_MODULES[*]}")" + if ! REQUIRED_MODULES="${REQUIRED_MODULES}" python - <<'PY' +import importlib.util +import os + +required = [pkg for pkg in os.environ["REQUIRED_MODULES"].split(",") if pkg] +missing = [pkg for pkg in required if importlib.util.find_spec(pkg) is None] +raise SystemExit(1 if missing else 0) +PY + then + python -m pip install --disable-pip-version-check -r scripts/requirements-blueprint-validator.txt >/dev/null + fi +fi + +if [[ "$LIST_CHECKS" -eq 1 ]]; then + python scripts/validate_blueprint_artifacts.py --list-checks + exit 0 +fi + +python -m json.tool docs/reports/blueprint_artifacts/T6_Evidence_Manifest.json >/dev/null +python scripts/validate_blueprint_artifacts.py +python scripts/validate_blueprint_artifacts.py --json >"${OUTPUT_JSON}" +python -m json.tool "${OUTPUT_JSON}" >/dev/null +python scripts/validate_blueprint_artifacts.py --base-dir docs/reports/blueprint_artifacts + +if [[ "$SKIP_PYTEST" -ne 1 ]]; then + pytest -q tests/test_validate_blueprint_artifacts.py tests/test_run_blueprint_artifact_checks.py +fi diff --git a/scripts/validate_blueprint_artifacts.py b/scripts/validate_blueprint_artifacts.py new file mode 100644 index 00000000..e5a341c0 --- /dev/null +++ b/scripts/validate_blueprint_artifacts.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Validate blueprint starter pack structure and sample data.""" + +from __future__ import annotations + +import argparse +import csv +import json +import yaml +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path +from typing import Callable + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_ART = ROOT / "docs" / "reports" / "blueprint_artifacts" + +REQUIRED_FILES = [ + "README.md", + "T1_Executive_Attestation.md", + "T2_Control_Crosswalk.csv", + "T3_Model_Risk_Register.csv", + "T4_Incident_Notification_Playbook.md", + "T5_RedTeam_Closure_Report.md", + "T6_Evidence_Manifest.json", + "T6_Evidence_Manifest.schema.json", + "T7_Runtime_Policy.rego", + "T8_Kafka_Audit_ACL_Example.yaml", + "T9_K8s_NetworkPolicy_Example.yaml", +] + + +CHECK_SEQUENCE: list[tuple[str, str]] = [ + ("presence", "Required artifact files exist"), + ("manifest_structure", "Manifest top-level/object structure is valid"), + ("manifest_timestamp", "Manifest timestamp is valid ISO-8601"), + ("schema_metadata", "Schema metadata and top-level type are valid"), + ("schema_contract", "Manifest/schema required keys align"), + ("schema_constraints", "Manifest satisfies schema keyword constraints"), + ("csv_semantics", "CSV headers and sample-row semantics are valid"), + ("rego_guardrails", "Rego policy contains minimum guardrails"), + ("yaml_examples", "YAML examples parse and satisfy required semantics"), +] + +@dataclass +class ValidationResult: + name: str + ok: bool + detail: str + + +def load_manifest(base_dir: Path = DEFAULT_ART) -> dict: + return json.loads((base_dir / "T6_Evidence_Manifest.json").read_text()) + + +def load_schema(base_dir: Path = DEFAULT_ART) -> dict: + return json.loads((base_dir / "T6_Evidence_Manifest.schema.json").read_text()) + + +def validate_presence(base_dir: Path = DEFAULT_ART) -> ValidationResult: + missing = [f for f in REQUIRED_FILES if not (base_dir / f).exists()] + if missing: + return ValidationResult("presence", False, f"Missing files: {missing}") + return ValidationResult("presence", True, "All required files present") + + +def validate_json_manifest(base_dir: Path = DEFAULT_ART) -> ValidationResult: + manifest = load_manifest(base_dir) + expected_top = {"manifest_id", "generated_at", "institution", "artifacts"} + if set(manifest.keys()) != expected_top: + return ValidationResult("manifest_structure", False, "Unexpected manifest keys") + if not isinstance(manifest["artifacts"], list) or not manifest["artifacts"]: + return ValidationResult("manifest_structure", False, "Manifest artifacts must be a non-empty list") + for item in manifest["artifacts"]: + required = {"name", "location", "hash", "signature"} + if set(item.keys()) != required: + return ValidationResult("manifest_structure", False, f"Invalid artifact item keys: {item}") + for key in required: + if not isinstance(item[key], str) or not item[key].strip(): + return ValidationResult("manifest_structure", False, f"Artifact field '{key}' must be non-empty") + return ValidationResult("manifest_structure", True, "Manifest structure is valid") + + +def validate_manifest_timestamp(base_dir: Path = DEFAULT_ART) -> ValidationResult: + manifest = load_manifest(base_dir) + ts = manifest["generated_at"] + try: + datetime.fromisoformat(ts.replace("Z", "+00:00")) + except ValueError: + return ValidationResult("manifest_timestamp", False, f"Invalid generated_at timestamp: {ts}") + return ValidationResult("manifest_timestamp", True, "generated_at timestamp is valid ISO-8601") + + +def validate_schema_metadata(base_dir: Path = DEFAULT_ART) -> ValidationResult: + schema = load_schema(base_dir) + if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema": + return ValidationResult("schema_metadata", False, "Unexpected $schema URI") + if schema.get("type") != "object": + return ValidationResult("schema_metadata", False, "Top-level schema type must be object") + return ValidationResult("schema_metadata", True, "Schema metadata is valid") + + +def validate_manifest_against_schema_contract(base_dir: Path = DEFAULT_ART) -> ValidationResult: + schema = load_schema(base_dir) + manifest = load_manifest(base_dir) + + schema_required = set(schema.get("required", [])) + manifest_keys = set(manifest.keys()) + if schema_required != manifest_keys: + return ValidationResult( + "schema_contract", + False, + f"Manifest/schema required key mismatch: schema={schema_required}, manifest={manifest_keys}", + ) + + artifact_req = set(schema["properties"]["artifacts"]["items"].get("required", [])) + for item in manifest["artifacts"]: + if artifact_req != set(item.keys()): + return ValidationResult( + "schema_contract", + False, + f"Artifact item/schema required key mismatch: schema={artifact_req}, item={set(item.keys())}", + ) + return ValidationResult("schema_contract", True, "Manifest matches schema contract keys") + + +def validate_schema_constraints(base_dir: Path = DEFAULT_ART) -> ValidationResult: + schema = load_schema(base_dir) + manifest = load_manifest(base_dir) + + for key in schema.get("required", []): + if key not in manifest: + return ValidationResult("schema_constraints", False, f"Missing required field: {key}") + + # top-level string minLength checks + properties = schema.get("properties", {}) + for key in ("manifest_id", "generated_at", "institution"): + rule = properties.get(key, {}) + min_len = rule.get("minLength", 0) + value = manifest.get(key, "") + if not isinstance(value, str) or len(value) < min_len: + return ValidationResult("schema_constraints", False, f"Field {key} violates minLength/type") + + artifacts = manifest.get("artifacts", []) + min_items = properties.get("artifacts", {}).get("minItems", 0) + if not isinstance(artifacts, list) or len(artifacts) < min_items: + return ValidationResult("schema_constraints", False, "artifacts violates minItems/type") + + item_props = properties.get("artifacts", {}).get("items", {}).get("properties", {}) + item_required = properties.get("artifacts", {}).get("items", {}).get("required", []) + for idx, item in enumerate(artifacts, start=1): + for req in item_required: + if req not in item: + return ValidationResult("schema_constraints", False, f"artifacts[{idx}] missing field {req}") + for req in item_required: + rule = item_props.get(req, {}) + min_len = rule.get("minLength", 0) + value = item.get(req, "") + if not isinstance(value, str) or len(value) < min_len: + return ValidationResult("schema_constraints", False, f"artifacts[{idx}].{req} violates minLength/type") + + return ValidationResult("schema_constraints", True, "Manifest satisfies schema keyword constraints") + + +def _valid_date(value: str) -> bool: + try: + datetime.strptime(value, "%Y-%m-%d") + return True + except ValueError: + return False + + +def validate_csv_headers(base_dir: Path = DEFAULT_ART) -> ValidationResult: + with (base_dir / "T2_Control_Crosswalk.csv").open(newline="") as f: + crosswalk_rows = list(csv.reader(f)) + headers = crosswalk_rows[0] + if headers != ["control_id", "framework", "obligation", "artifact"]: + return ValidationResult("csv_semantics", False, "Crosswalk CSV header mismatch") + if len(crosswalk_rows) < 2: + return ValidationResult("csv_semantics", False, "Crosswalk CSV must include at least one data row") + + with (base_dir / "T3_Model_Risk_Register.csv").open(newline="") as f: + rows = list(csv.reader(f)) + headers = rows[0] + expected = [ + "model_id", + "owner", + "use_case", + "risk_tier", + "validation_status", + "last_validation_date", + "monitoring_status", + "next_review_date", + ] + if headers != expected: + return ValidationResult("csv_semantics", False, "Model risk register CSV header mismatch") + + allowed_tiers = {"low", "medium", "high", "systemic", "frontier"} + for idx, row in enumerate(rows[1:], start=2): + if len(row) != len(expected): + return ValidationResult("csv_semantics", False, f"Model risk row {idx} has wrong column count") + risk_tier = row[3].strip().lower() + if risk_tier not in allowed_tiers: + return ValidationResult("csv_semantics", False, f"Model risk row {idx} has invalid risk_tier: {row[3]}") + if not _valid_date(row[5]) or not _valid_date(row[7]): + return ValidationResult("csv_semantics", False, f"Model risk row {idx} has invalid date format") + + return ValidationResult("csv_semantics", True, "CSV headers and sample rows are valid") + + +def validate_rego_guardrails(base_dir: Path = DEFAULT_ART) -> ValidationResult: + rego = (base_dir / "T7_Runtime_Policy.rego").read_text() + required_fragments = [ + "default allow = false", + 'input.risk_tier != "prohibited"', + "input.hitl_approved == true", + ] + missing = [frag for frag in required_fragments if frag not in rego] + if missing: + return ValidationResult("rego_guardrails", False, f"Missing Rego guardrails: {missing}") + return ValidationResult("rego_guardrails", True, "Rego guardrails present") + + +def validate_yaml_examples(base_dir: Path = DEFAULT_ART) -> ValidationResult: + kafka_obj = yaml.safe_load((base_dir / "T8_Kafka_Audit_ACL_Example.yaml").read_text()) + k8s_obj = yaml.safe_load((base_dir / "T9_K8s_NetworkPolicy_Example.yaml").read_text()) + + if not isinstance(kafka_obj, dict): + return ValidationResult("yaml_examples", False, "Kafka ACL YAML must be a mapping") + if "principals" not in kafka_obj or "constraints" not in kafka_obj: + return ValidationResult("yaml_examples", False, "Kafka ACL YAML missing principals/constraints") + if not isinstance(kafka_obj["principals"], list) or len(kafka_obj["principals"]) == 0: + return ValidationResult("yaml_examples", False, "Kafka ACL YAML principals must be a non-empty list") + + constraints = kafka_obj.get("constraints", []) + if not any(isinstance(item, dict) and item.get("enforce_mtls") is True for item in constraints): + return ValidationResult("yaml_examples", False, "Kafka ACL YAML missing enforce_mtls: true constraint") + + if not isinstance(k8s_obj, dict): + return ValidationResult("yaml_examples", False, "K8s YAML must be a mapping") + if k8s_obj.get("kind") != "NetworkPolicy": + return ValidationResult("yaml_examples", False, "K8s YAML kind must be NetworkPolicy") + + spec = k8s_obj.get("spec", {}) + policy_types = spec.get("policyTypes", []) + if "Egress" not in policy_types: + return ValidationResult("yaml_examples", False, "K8s YAML policyTypes must include Egress") + + return ValidationResult("yaml_examples", True, "YAML examples parse and satisfy required semantics") + + +def safe_run(name: str, fn: Callable[[Path], ValidationResult], base_dir: Path) -> ValidationResult: + try: + return fn(base_dir) + except Exception as exc: # defensive: convert unexpected errors into check failures + return ValidationResult(name, False, f"Unhandled exception: {type(exc).__name__}: {exc}") + + +def run_validations(base_dir: Path = DEFAULT_ART) -> list[ValidationResult]: + check_map: dict[str, Callable[[Path], ValidationResult]] = { + "presence": validate_presence, + "manifest_structure": validate_json_manifest, + "manifest_timestamp": validate_manifest_timestamp, + "schema_metadata": validate_schema_metadata, + "schema_contract": validate_manifest_against_schema_contract, + "schema_constraints": validate_schema_constraints, + "csv_semantics": validate_csv_headers, + "rego_guardrails": validate_rego_guardrails, + "yaml_examples": validate_yaml_examples, + } + + results: list[ValidationResult] = [] + seen: set[str] = set() + for name, _desc in CHECK_SEQUENCE: + if name in seen: + results.append(ValidationResult(name, False, f"Duplicate check ID in CHECK_SEQUENCE: {name}")) + continue + seen.add(name) + + fn = check_map.get(name) + if fn is None: + results.append(ValidationResult(name, False, f"No handler registered for check ID: {name}")) + continue + + results.append(safe_run(name, fn, base_dir)) + + return results + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate blueprint artifact starter pack") + parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON results") + parser.add_argument("--base-dir", type=str, default=str(DEFAULT_ART), help="Override artifact directory") + parser.add_argument("--list-checks", action="store_true", help="List stable check IDs and exit") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.list_checks: + for name, desc in CHECK_SEQUENCE: + print(f"{name}: {desc}") + return + + results = run_validations(Path(args.base_dir)) + if args.json: + print(json.dumps([asdict(r) for r in results], indent=2)) + else: + for r in results: + prefix = "[PASS]" if r.ok else "[FAIL]" + print(f"{prefix} {r.name}: {r.detail}") + + if any(not r.ok for r in results): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_run_blueprint_artifact_checks.py b/tests/test_run_blueprint_artifact_checks.py new file mode 100644 index 00000000..f0e87c67 --- /dev/null +++ b/tests/test_run_blueprint_artifact_checks.py @@ -0,0 +1,56 @@ +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "run_blueprint_artifact_checks.sh" + + +def test_unknown_option_returns_usage_error(): + proc = subprocess.run(["bash", str(SCRIPT), "--unknown-option"], capture_output=True, text=True) + assert proc.returncode == 2 + assert "Usage:" in proc.stderr + + +def test_list_checks_mode_emits_check_names(): + proc = subprocess.run( + ["bash", str(SCRIPT), "--skip-install", "--skip-pytest", "--list-checks"], + check=True, + capture_output=True, + text=True, + ) + assert "presence: Required artifact files exist" in proc.stdout + assert "yaml_examples: YAML examples parse and satisfy required semantics" in proc.stdout + assert "[PASS]" not in proc.stdout + + +def test_output_json_path_is_supported(tmp_path: Path): + output = tmp_path / "validator-output.json" + subprocess.run( + [ + "bash", + str(SCRIPT), + "--skip-install", + "--skip-pytest", + "--output-json", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + assert output.exists() + payload = output.read_text() + assert "manifest_structure" in payload + + +def test_output_json_missing_value_returns_usage_error(): + proc = subprocess.run(["bash", str(SCRIPT), "--output-json"], capture_output=True, text=True) + assert proc.returncode == 2 + assert "Missing value for --output-json" in proc.stderr + + +def test_help_option_returns_usage(): + proc = subprocess.run(["bash", str(SCRIPT), "--help"], capture_output=True, text=True) + assert proc.returncode == 0 + assert "Usage:" in proc.stdout + assert "--output-json" in proc.stdout diff --git a/tests/test_validate_blueprint_artifacts.py b/tests/test_validate_blueprint_artifacts.py new file mode 100644 index 00000000..925deb6c --- /dev/null +++ b/tests/test_validate_blueprint_artifacts.py @@ -0,0 +1,156 @@ +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts import validate_blueprint_artifacts as v + + +def stage_artifacts(tmp_path: Path) -> Path: + alt = tmp_path / "art" + alt.mkdir() + for f in v.REQUIRED_FILES: + src = v.DEFAULT_ART / f + dst = alt / f + shutil.copy2(src, dst) + return alt + + +def test_check_sequence_has_unique_ids(): + ids = [name for name, _desc in v.CHECK_SEQUENCE] + assert len(ids) == len(set(ids)) + + +def test_required_files_unique(): + assert len(v.REQUIRED_FILES) == len(set(v.REQUIRED_FILES)) + + +def test_manifest_top_keys_match_expectation(): + manifest = v.load_manifest(v.DEFAULT_ART) + assert set(manifest.keys()) == {"manifest_id", "generated_at", "institution", "artifacts"} + + +def test_validation_results_all_pass(): + results = v.run_validations(v.DEFAULT_ART) + assert all(r.ok for r in results) + + +def test_expected_check_names_present(): + names = [r.name for r in v.run_validations(v.DEFAULT_ART)] + assert names == [name for name, _desc in v.CHECK_SEQUENCE] + + +def test_safe_run_converts_exception_to_failure(): + result = v.safe_run("boom", lambda _base: 1 / 0, v.DEFAULT_ART) + assert result.ok is False + assert result.name == "boom" + assert "Unhandled exception" in result.detail + + +def test_non_default_base_dir_supported(tmp_path: Path): + alt = stage_artifacts(tmp_path) + results = v.run_validations(alt) + assert all(r.ok for r in results) + + +def test_missing_file_reports_presence_failure(tmp_path: Path): + alt = stage_artifacts(tmp_path) + (alt / "T5_RedTeam_Closure_Report.md").unlink() + results = v.run_validations(alt) + presence = [r for r in results if r.name == "presence"][0] + assert presence.ok is False + assert "Missing files" in presence.detail + + +def test_invalid_timestamp_reports_failure(tmp_path: Path): + alt = stage_artifacts(tmp_path) + manifest_path = alt / "T6_Evidence_Manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["generated_at"] = "not-a-timestamp" + manifest_path.write_text(json.dumps(manifest, indent=2)) + + results = v.run_validations(alt) + ts_result = [r for r in results if r.name == "manifest_timestamp"][0] + assert ts_result.ok is False + assert "Invalid generated_at timestamp" in ts_result.detail + + +def test_invalid_model_risk_csv_date_reports_failure(tmp_path: Path): + alt = stage_artifacts(tmp_path) + risk = alt / "T3_Model_Risk_Register.csv" + lines = risk.read_text().splitlines() + # Corrupt next_review_date format on first data row. + parts = lines[1].split(",") + parts[-1] = "2026/06/30" + lines[1] = ",".join(parts) + risk.write_text("\n".join(lines) + "\n") + + results = v.run_validations(alt) + csv_result = [r for r in results if r.name == "csv_semantics"][0] + assert csv_result.ok is False + assert "invalid date format" in csv_result.detail + + +def test_empty_manifest_signature_reports_schema_constraint_failure(tmp_path: Path): + alt = stage_artifacts(tmp_path) + manifest_path = alt / "T6_Evidence_Manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["artifacts"][0]["signature"] = "" + manifest_path.write_text(json.dumps(manifest, indent=2)) + + results = v.run_validations(alt) + schema_result = [r for r in results if r.name == "schema_constraints"][0] + assert schema_result.ok is False + assert "violates minLength/type" in schema_result.detail + + +def test_k8s_yaml_missing_egress_reports_failure(tmp_path: Path): + alt = stage_artifacts(tmp_path) + k8s = alt / "T9_K8s_NetworkPolicy_Example.yaml" + text = k8s.read_text().replace("- Egress", "- Ingress") + k8s.write_text(text) + + results = v.run_validations(alt) + yaml_result = [r for r in results if r.name == "yaml_examples"][0] + assert yaml_result.ok is False + assert "policyTypes must include Egress" in yaml_result.detail + + +def test_json_mode_output_is_machine_readable(): + proc = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate_blueprint_artifacts.py"), + "--json", + "--base-dir", + str(v.DEFAULT_ART), + ], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) + assert isinstance(payload, list) + assert any(item.get("name") == "yaml_examples" for item in payload) + + +def test_no_unregistered_handler_errors_in_default_run(): + results = v.run_validations(v.DEFAULT_ART) + assert all("No handler registered" not in r.detail for r in results) + + +def test_list_checks_mode_outputs_sequence(): + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "validate_blueprint_artifacts.py"), "--list-checks"], + check=True, + capture_output=True, + text=True, + ) + lines = [line.strip() for line in proc.stdout.splitlines() if line.strip()] + expected = [f"{name}: {desc}" for name, desc in v.CHECK_SEQUENCE] + assert lines == expected