fix(workflow): keep RC connector extras and fail sentinel on missing extras

CHANGELOG.md

@@ -16,6 +16,12 @@ Empty while the working tree carries **`1.8.0-rc`** (see section below).
 
 - **`filesystem_credit_card`** in `benchmark-rc-v2.sentinel.yaml` and `benchmark-rc-v3.sentinel.yaml` fails the post-smoke checker when `CREDIT_CARD` is absent from `filesystem_findings`. Corpus file: `tests/data/homelab_synthetic/synthetic_pan_luhn.txt` (Visa test PAN).
 
+### RC smoke extras and missing-extra failures (maestro#91)
+
+- **Host smoke prepare** syncs `--extra compressed` plus the extras the `--bench-config` targets need (`scripts/rc_bench_extras.py list`, from `optional_extra_for_target`; RC v2 → `nosql`, v3 → `mysql nosql postgres shares`). `uv sync` no longer prunes `pymongo` before the scan. `rc_bench_extras.py verify` fails the prepare when a distribution of those extras is missing.
+- **MongoDB / Redis connectors** record `scan_failures.reason=missing_optional_dependency` (not `unreachable`) when the `nosql` extra is absent.
+- **RC sentinel** fails on any `missing_optional_dependency` in the latest session before optional probes run, so a closed lab port cannot turn a missing extra into `SKIP` + `SENTINEL_OK`.
+
 ### Trust anchor (#1992)
 
 - **Key rotation** is accepted only when both embedded anchors (Ed25519 and ML-DSA-65) sign `data-boar/license-key-rotation/v1` plus the epoch. A raw `DATA_BOAR_LICENSE_PUBLIC_KEY_*`, `DATA_BOAR_LICENSE_MLDSA_PUBLIC_KEY_*`, or YAML public-key path fails closed (`untrusted_key_override`). Hybrid `dbmldsa_sig` still verifies, against the packaged or rotated ML-DSA key.

connectors/mongodb_connector.py

@@ -135,6 +135,17 @@ def run(self) -> None:
 
         target_name = self.config.get("name", "mongodb")
         audit_name = audit_log_target_label(self.config, default="mongodb")
+        if not _MONGO_AVAILABLE:
+            from core.extras_runtime import missing_optional_message
+
+            # Not "unreachable": the RC sentinel must not read a missing extra as
+            # a lab target that is down (maestro#91).
+            self.db_manager.save_failure(
+                target_name,
+                "missing_optional_dependency",
+                missing_optional_message(subject="MongoDB connector", extra="nosql"),
+            )
+            return
         try:
             self.connect()
         except Exception as e:

connectors/redis_connector.py

@@ -150,6 +150,17 @@ def run(self) -> None:
 
         target_name = self.config.get("name", "redis")
         audit_name = audit_log_target_label(self.config, default="redis")
+        if not _REDIS_AVAILABLE:
+            from core.extras_runtime import missing_optional_message
+
+            # Not "unreachable": the RC sentinel must not read a missing extra as
+            # a lab target that is down (maestro#91).
+            self.db_manager.save_failure(
+                target_name,
+                "missing_optional_dependency",
+                missing_optional_message(subject="Redis connector", extra="nosql"),
+            )
+            return
         try:
             self.connect()
         except Exception as e:

scripts/benchmark_rc_sentinel_check.py

@@ -3,9 +3,11 @@
 
 Validates SQLite findings for required patterns (min_count / max_count), optional
 connector probes, forbidden pattern substrings, and static negative SSRF/auth cases.
+Any ``scan_failures.reason=missing_optional_dependency`` in the session fails before
+optional probes run, so a closed lab port cannot mask a missing extra (maestro#91).
 Exit 0 = pass, 1 = sentinel fail, 2 = usage/config error.
 
-Refs: maestro#82, maestro#86, data-boar#1980 (gap report §E), data-boar#1985.
+Refs: maestro#82, maestro#86, maestro#91, data-boar#1980 (gap report §E), data-boar#1985.
 """
 
 from __future__ import annotations
@@ -259,6 +261,33 @@ def _count_scan_failures(
     return int(row[0]) if row else 0
 
 
+# A connector whose optional extra is absent is a tool/prepare defect, never a lab
+# outage: it fails the sentinel even when the optional probe port is closed.
+_MISSING_EXTRA_REASON = "missing_optional_dependency"
+
+
+def _missing_extra_failures(
+    conn: sqlite3.Connection, session_id: str
+) -> list[tuple[str, str]]:
+    # Minimal fixture sqlite files may lack scan_failures or its details column;
+    # the app schema (core.database.ScanFailure) always has both.
+    cols = {str(r[1]) for r in conn.execute("PRAGMA table_info(scan_failures)")}
+    if "reason" not in cols:
+        return []
+    if "details" in cols:
+        query = (
+            "SELECT target_name, details FROM scan_failures "
+            "WHERE session_id = ? AND reason = ? ORDER BY target_name"
+        )
+    else:
+        query = (
+            "SELECT target_name, '' FROM scan_failures "
+            "WHERE session_id = ? AND reason = ? ORDER BY target_name"
+        )
+    rows = conn.execute(query, (session_id, _MISSING_EXTRA_REASON)).fetchall()
+    return [(str(r[0] or ""), str(r[1] or "")) for r in rows]
+
+
 def _latest_session_id(conn: sqlite3.Connection) -> tuple[str | None, str | None]:
     """Latest scan by started_at; only status=completed is acceptable evidence."""
     row = conn.execute(
@@ -292,6 +321,12 @@ def _check_findings_sentinel(
         if session_err:
             return [session_err]
 
+        for target_name, details in _missing_extra_failures(conn, session_id):
+            errors.append(
+                f"scan_failures reason={_MISSING_EXTRA_REASON} target={target_name!r}: "
+                f"optional extra absent in the scan venv, not a lab outage ({details[:200]})"
+            )
+
         total = sum(
             _count_rows(conn, t, session_id)
             for t in (

scripts/lab-completao-host-smoke.sh

@@ -189,17 +189,55 @@ _lc_install_prebuilt_rust_wheel() {
   return 1
 }
 
+# Print the optional extras the bench config's targets need (one per line; empty when
+# there is no bench config or helper). Needs an existing venv (PyYAML); non-zero exit
+# means the caller must bootstrap first.
+_lc_rc_bench_extras() {
+  local cfg="${LC_BENCH_CONFIG:-tests/config/benchmark-rc-v3.yaml}"
+  if [[ ! -f "$LC_REPO_ROOT/$cfg" || ! -f "$LC_REPO_ROOT/scripts/rc_bench_extras.py" ]]; then
+    return 0
+  fi
+  (cd "$LC_REPO_ROOT" && uv run --no-sync python scripts/rc_bench_extras.py list --config "$cfg")
+}
+
 _lc_prepare_baremetal_runtime() {
   if [[ ! -f "$LC_REPO_ROOT/pyproject.toml" ]] || ! _lc_cmd uv; then
     return 1
   fi
+  # `uv sync` is exact: every extra not named here is pruned from the venv.
   # --extra compressed pulls py7zr so .7z archives are scannable in the completao
-  # flow; without it `uv sync` prunes py7zr and .7z stays archive_unsupported (#931).
-  echo "Preparing baremetal venv (uv sync --extra compressed)..."
-  if ! (cd "$LC_REPO_ROOT" && uv sync --extra compressed); then
+  # flow (#931). The bench config's targets add theirs (mongodb -> nosql, smb -> shares,
+  # ...) so a connector extra is never pruned before the scan (maestro#91). This runs
+  # for both the engine import probe and the RC scan with the same config default as
+  # CONFIG_RC, so the second sync does not prune what the first installed.
+  local cfg="${LC_BENCH_CONFIG:-tests/config/benchmark-rc-v3.yaml}"
+  local extras="" extra=""
+  local -a extra_args=(--extra compressed)
+  if ! extras="$(_lc_rc_bench_extras 2>/dev/null)"; then
+    echo "Bootstrapping baremetal venv (uv sync --extra compressed)..."
+    if ! (cd "$LC_REPO_ROOT" && uv sync --extra compressed); then
+      echo "uv sync: FAILED"
+      return 1
+    fi
+    if ! extras="$(_lc_rc_bench_extras)"; then
+      echo "rc_bench_extras list: FAILED ($cfg)"
+      return 1
+    fi
+  fi
+  while IFS= read -r extra; do
+    [[ -n "$extra" ]] && extra_args+=(--extra "$extra")
+  done <<<"$extras"
+  echo "Preparing baremetal venv (uv sync ${extra_args[*]})..."
+  if ! (cd "$LC_REPO_ROOT" && uv sync "${extra_args[@]}"); then
     echo "uv sync: FAILED"
     return 1
   fi
+  if [[ -f "$LC_REPO_ROOT/$cfg" && -f "$LC_REPO_ROOT/scripts/rc_bench_extras.py" ]]; then
+    if ! (cd "$LC_REPO_ROOT" && uv run --no-sync python scripts/rc_bench_extras.py verify --config "$cfg" --extra compressed); then
+      echo "rc_bench_extras verify: FAILED (connector extras missing after uv sync; $cfg)"
+      return 1
+    fi
+  fi
   # Prefer the prebuilt Build-Once wheel (no Rust toolchain per host, #937); only
   # build from source via maturin when no usable wheel is available on this host.
   if _lc_install_prebuilt_rust_wheel; then

scripts/rc_bench_extras.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Optional extras a Maestro RC bench config needs in the baremetal scan venv.
+
+``uv sync`` is exact: any extra not named on the command line is pruned. The host
+smoke used a fixed ``--extra compressed`` and lost ``pymongo`` (extra ``nosql``),
+so the Mongo target failed while the sentinel read it as a lab outage (maestro#91).
+
+``list``   — print the extras the config's targets need, one per line, from
+             ``core.extras_runtime.optional_extra_for_target`` (type/driver map).
+``verify`` — after the sync, confirm every distribution declared by those extras
+             (plus any ``--extra``) in ``pyproject.toml`` is installed. Exit 1 with
+             ``EXTRA_MISSING`` lines otherwise, so the prepare fails loud before
+             the scan instead of recording connector failures.
+
+Exit 0 = ok, 1 = missing distribution, 2 = usage/config error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.metadata
+import re
+import sys
+import tomllib
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(_REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(_REPO_ROOT))
+
+from core.extras_runtime import optional_extra_for_target  # noqa: E402
+
+_PROJECT_NAME = "data-boar"
+_REQ_NAME = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[([^\]]*)\])?")
+
+
+def _normalize(name: str) -> str:
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def extras_for_config(config: dict[str, Any]) -> list[str]:
+    found: set[str] = set()
+    for target in config.get("targets") or []:
+        if isinstance(target, dict):
+            extra = optional_extra_for_target(target)
+            if extra:
+                found.add(extra)
+    return sorted(found)
+
+
+def _optional_dependencies(pyproject: Path) -> dict[str, list[str]]:
+    data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
+    return dict((data.get("project") or {}).get("optional-dependencies") or {})
+
+
+def distributions_for_extras(
+    extras: list[str], optional_deps: dict[str, list[str]]
+) -> dict[str, list[str]]:
+    """Map extra -> distribution names; self-references (``data-boar[x]``) expand."""
+    out: dict[str, list[str]] = {}
+    for extra in extras:
+        if extra not in optional_deps:
+            raise KeyError(f"extra {extra!r} not declared in pyproject.toml")
+        dists: list[str] = []
+        pending = [extra]
+        seen: set[str] = set()
+        while pending:
+            current = pending.pop()
+            if current in seen:
+                continue
+            seen.add(current)
+            for req in optional_deps.get(current) or []:
+                m = _REQ_NAME.match(req)
+                if not m:
+                    continue
+                name, sub = m.group(1), m.group(2)
+                if _normalize(name) == _PROJECT_NAME:
+                    pending.extend(
+                        s.strip() for s in (sub or "").split(",") if s.strip()
+                    )
+                elif name not in dists:
+                    dists.append(name)
+        out[extra] = dists
+    return out
+
+
+def missing_distributions(dist_map: dict[str, list[str]]) -> list[tuple[str, str]]:
+    missing: list[tuple[str, str]] = []
+    for extra, dists in dist_map.items():
+        for dist in dists:
+            try:
+                importlib.metadata.distribution(dist)
+            except importlib.metadata.PackageNotFoundError:
+                missing.append((extra, dist))
+    return missing
+
+
+def _load_config(path: Path) -> dict[str, Any]:
+    data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    if not isinstance(data, dict):
+        raise ValueError(f"expected mapping in {path}")
+    return data
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+    parser.add_argument("command", choices=("list", "verify"))
+    parser.add_argument("--config", required=True, type=Path)
+    parser.add_argument(
+        "--extra",
+        action="append",
+        default=[],
+        help="Extra synced regardless of the config (repeatable; verify only).",
+    )
+    parser.add_argument("--pyproject", type=Path, default=_REPO_ROOT / "pyproject.toml")
+    args = parser.parse_args(argv)
+
+    config_path = args.config if args.config.is_absolute() else _REPO_ROOT / args.config
+    try:
+        extras = extras_for_config(_load_config(config_path))
+    except (OSError, ValueError, yaml.YAMLError) as exc:
+        print(
+            f"rc_bench_extras: cannot read config {config_path}: {exc}", file=sys.stderr
+        )
+        return 2
+
+    if args.command == "list":
+        for extra in extras:
+            print(extra)
+        return 0
+
+    wanted = sorted(set(extras) | set(args.extra))
+    try:
+        dist_map = distributions_for_extras(
+            wanted, _optional_dependencies(args.pyproject)
+        )
+    except (OSError, KeyError, tomllib.TOMLDecodeError) as exc:
+        print(f"rc_bench_extras: {exc}", file=sys.stderr)
+        return 2
+    missing = missing_distributions(dist_map)
+    for extra, dist in missing:
+        print(f"EXTRA_MISSING extra={extra} dist={dist}", file=sys.stderr)
+    if missing:
+        return 1
+    print(f"rc_bench_extras: OK ({', '.join(wanted) or 'none'})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())