Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 36 additions & 18 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ name: ci
# ruff is a pre-existing 347-error backlog — see step comment).
# - macos: proves the install path on macOS-14. Cannot be validated from the
# Linux dev box; needs a real macOS runner (see NOTE in the job).
# - wheel-canary: NON-BLOCKING honest red marker — builds + installs the wheel
# - wheel-canary: blocking install check — builds + installs the wheel
# and imports the console entrypoints.
# - integration: runs `pytest -m integration` so integration-marked tests
# can't silently rot. Most skip gracefully when their external dependency
Expand Down Expand Up @@ -46,13 +46,15 @@ jobs:
# Isolate all state under a throwaway dir so tests never touch a real
# ~/.cortex or the checkout's own state.
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
mkdir -p "$CORTEX_ROOT_DIR"
# synthetic/tests/ depends on [analytics] (xgboost, shap) which we
# don't install in smoke — excluded here (matches --collect-only
# policy). This runs the FULL test BODIES, not just collection.
set +e
python -m pytest -p no:cacheprovider --ignore=synthetic/tests -q \
python -m pytest -p tests.network_guard -p no:cacheprovider --ignore=synthetic/tests -q --junitxml=/tmp/pytest.xml \
| tee /tmp/pytest.out
code=${PIPESTATUS[0]}
set -e
Expand All @@ -62,25 +64,34 @@ jobs:
# branch. A floor of 1960 catches accidental mass-skips or a
# collection regression that silently drops tests while still
# "passing". Bump the floor deliberately as the suite grows.
passed="$(grep -oE '[0-9]+ passed' /tmp/pytest.out | tail -1 | grep -oE '[0-9]+')"
echo "pytest reported ${passed} passed (floor: 1960)"
test -n "$passed"
test "$passed" -ge 1960
python - <<'PYTEST_COUNTS'
import xml.etree.ElementTree as ET
root = ET.parse("/tmp/pytest.xml").getroot()
suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite"))
passed = sum(int(s.get("tests", 0)) - int(s.get("failures", 0))
- int(s.get("errors", 0)) - int(s.get("skipped", 0)) for s in suites)
print(f"JUnit reported {passed} passed (floor: 1960)")
assert passed >= 1960, f"Unexpected test count: {passed}"
PYTEST_COUNTS

- name: cortex demo must print the FK trail
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
output="$(cortex demo)"
echo "$output"
# Three synthesized prompts must each appear as a linked entry
# with score 0.80. If this assertion ever fails, the headline
# 'compounding intelligence' claim is broken on this build.
# synthetic linkage fixture is broken; this does not prove useful learning.
echo "$output" | grep -c 'score 0.80' | grep -q '^3$'

- name: cortex briefing must exit 2 without ANTHROPIC_API_KEY
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
# Pre-flight contract: missing key → exit 2 with actionable msg.
# Anything else (silent hang, exit 0, exit 1) regresses Phase 1.
Expand All @@ -94,6 +105,8 @@ jobs:
- name: MCP smoke drill — bridge DOWN (crash-proof headline)
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
# No bridge is started. The core-8 in-process MCP tools must still be
# green (crash-proof contract). Passthrough tools that need the bridge
Expand Down Expand Up @@ -133,11 +146,15 @@ jobs:

- name: ruff check (release-touched paths)
run: |
# Scoped to the Python files this release (Workstream F) touched.
# Both are ruff-clean; widen this list as backlog dirs reach zero.
# Core release paths plus the execution-integrity regression surface.
# Widen this list as other backlog directories reach zero.
python -m ruff check \
bridge_intelligence.py \
tests/contract/test_phase5_invariants.py
tests/contract/test_phase5_invariants.py \
supervisor/dispatch.py supervisor/collector.py supervisor/router.py \
store_registry.py tests/test_dispatch_integrity.py \
tests/test_orchestration_pipeline.py tests/test_model_router.py \
tests/test_doctor_json_reset.py tests/test_mcp_research_tools.py tests/network_guard.py

macos:
# macOS-14 install path. NOTE: this job CANNOT be validated from the Linux
Expand Down Expand Up @@ -166,6 +183,8 @@ jobs:
- name: cortex doctor must run without crashing (non-zero exit allowed)
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
mkdir -p "$CORTEX_ROOT_DIR"
# doctor reports env facts (API key, launchd, bridge) — a red env is
Expand All @@ -184,6 +203,8 @@ jobs:
- name: cortex demo must print the FK trail
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
output="$(cortex demo)"
echo "$output"
Expand All @@ -192,17 +213,14 @@ jobs:
- name: pytest (full suite, hermetic) — Python 3.12
env:
CORTEX_ROOT_DIR: ${{ runner.temp }}/cortex-home
CORTEX_STATE_DIR: ${{ runner.temp }}/cortex-state
CORTEX_HOME: ${{ runner.temp }}/cortex-state
run: |
python -m pytest -p no:cacheprovider --ignore=synthetic/tests -q
python -m pytest -p tests.network_guard -p no:cacheprovider --ignore=synthetic/tests -q

wheel-canary:
# NON-BLOCKING honest red marker for the git-clone-only beta. Builds the
# wheel, installs it into a clean venv, and imports the console entrypoints.
# Kept continue-on-error per the release plan even though the packaging was
# fixed on this branch and it is expected to pass locally — a green here is
# a bonus, a red here is a signal, neither blocks the release gate.
# The deployed artifact must contain the diagnostics as well as entrypoints.
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v4

Expand All @@ -226,7 +244,7 @@ jobs:
# The two console-script entrypoints must import from the installed
# wheel (not the checkout) — run from /tmp so cwd isn't the repo root.
cd /tmp
/tmp/wheelvenv/bin/python -c "import cli, mcp_server; print('wheel imports OK')"
/tmp/wheelvenv/bin/python -c "import cli, mcp_server, store_registry, recall_confidence; print('wheel imports OK')"
/tmp/wheelvenv/bin/cortex --help

integration:
Expand Down
76 changes: 76 additions & 0 deletions docs/design/truthful-execution.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Execution receipts and unverified outcomes

This change is a containment step, not a completed autonomous learning system.

## Contract

- `DispatchResult.success` and summary `succeeded` describe execution only.
- `execution_status` distinguishes completed, failed, incomplete, timed_out,
unsupported, and dry_run. Dry runs are neither successful nor failed execution.
- Truncation, refusal, missing final text, and tool-turn exhaustion cannot
produce successful Anthropic execution. Partial output and known token usage
survive incomplete responses. A batch API `succeeded` envelope is insufficient.
- Direct and message-batch adapters reject explicit shell commands; they do not
have a command-execution capability. Free-text capability inference is not solved.
- Persisted results and routing receipts say `verification_status=unverified`
and `learning_eligible=false`. A successful response without independent
verification is counted as unverified and flagged for review.
- `VerificationEvidence` is a legacy heuristic assessment. Even its maximum
score cannot verify a task or automatically requeue it. It is serialized under
`assessment`, not `verification`.
- Each dispatch has an attempt ID; batch IDs plus custom IDs provide stable
attempt identities on retrieval. Assessments attach to attempts. Ambiguous
work-item-only attachments fail rather than attach to the wrong retry.
- Repeated collection of the same receipt in one collector is idempotent;
conflicting receipts fail. Run directories cannot overwrite another run in
the same second. Cross-process journal deduplication is still pending.

## Routing behavior change

ModelRouter retains existing journals and appends execution observations, but
does not ingest them into adaptive outcome statistics or update those statistics
from new execution-only observations. Static task/complexity routing continues.
Batch completion no longer fabricates quality 1.0. Existing quality heuristics
remain advisory telemetry under `heuristic_quality_score`; measured `quality_score`
remains null, not a learning signal.

There is deliberately no switch to re-enable unverified history. Adaptive
outcome routing needs a separate ingestion contract that binds task, attempt,
artifact revision, check version, coverage and result, and supports corrections.
This pause concerns this supervisor router; it does not claim to disable every
other learning or retrieval heuristic in Cortex.

## Compatibility and limits

Consumers reading `verification` must migrate to the explicitly advisory
`assessment` field. Summary success counts are retained for compatibility but
must be labeled execution counts in downstream interfaces. Evidence attachment
before collection, or ambiguous attachment after retry, now raises ValueError.

Local providers currently return only text/token tuples, so their truncation
status cannot be established here. Nonempty local output remains unverified.
Timeout usage may be unknown, and cancelling an asyncio thread wrapper does not
prove an underlying network request stopped. No exactly-once external execution
guarantee is made.

Path validation resolves components instead of comparing string prefixes. It
rejects sibling-prefix and symlink escapes under a stable filesystem. This is
not an OS sandbox against a hostile concurrent filesystem mutation.

## Release and next gates

The focused regression suite is `tests/test_dispatch_integrity.py`. It exercises
real dispatcher/collector/router code with a synthetic provider and actual
temporary filesystem persistence. No live-model or installed-machine claim is
made by that suite. CI also gates touched files and installed-wheel imports.

Next: transactional event identity and correction, unified memory reads/writes,
atomic supersession, capture privacy, dependency/lease/cancellation enforcement,
and an independent exact-artifact verifier. Then prove a real interrupted work
loop and useful lesson reuse. Machine cutover requires a verified source/state
inventory, a consistent backup, and an explicit rollback path.

The offline pytest plugin also checks intended HTTP destinations before proxy
transport, permits in-memory test transports, and blocks external Python sockets.
Use a credential-free environment for broad test runs. This is not an OS-level
sandbox and does not constrain arbitrary subprocess network access.
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,12 @@ Repository = "https://github.com/jessekemp1/cortex"
# wheel (mcp_handlers, then state_paths, then config… ModuleNotFoundError one
# import deeper each time), so we package every root module. All 42 are
# import-safe (no top-level side effects), so unused ones are inert.
py-modules = ["agent_factory", "ai_intelligence", "alert_monitor", "bridge", "bridge_intelligence", "bridge_system", "briefing_resilient", "config", "context_intelligence", "data_migration", "deep_assessment", "env_loader", "feedback", "formatter", "goal_commands", "goal_parser", "goal_velocity", "heartbeat", "learning", "learning_config", "mcp_handlers", "mcp_server", "metrics_tracker", "orchestrator", "portfolio_analyzer", "portfolio_memory", "recommendation_engine", "recommendations", "reflection", "scheduler", "security", "self_audit", "session_cache", "session_delta", "session_manager", "session_snapshot", "session_watcher", "setup_ops", "spec_knowledge_base", "state_paths", "task_discovery", "validation", "watches", "weekly_planner"]
py-modules = ["agent_factory", "ai_intelligence", "alert_monitor", "bridge", "bridge_intelligence", "bridge_system", "briefing_resilient", "config", "context_intelligence", "data_migration", "deep_assessment", "env_loader", "feedback", "formatter", "goal_commands", "goal_parser", "goal_velocity", "heartbeat", "learning", "learning_config", "mcp_handlers", "mcp_server", "metrics_tracker", "orchestrator", "portfolio_analyzer", "portfolio_memory", "recommendation_engine", "recommendations", "reflection", "scheduler", "security", "self_audit", "session_cache", "session_delta", "session_manager", "session_snapshot", "session_watcher", "setup_ops", "spec_knowledge_base", "state_paths", "store_registry", "recall_confidence", "task_discovery", "validation", "watches", "weekly_planner"]

[tool.setuptools.packages.find]
exclude = [
"tests*",
"build*",
"docs*",
"examples*",
"reports*",
Expand Down
18 changes: 11 additions & 7 deletions store_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@
from pathlib import Path
from typing import List, Optional

from state_paths import get_cortex_dir

WRITER_NONE = "NONE"


def _cortex_dir() -> Path:
return Path.home() / ".cortex"
return get_cortex_dir()


def _portfolio_dir() -> Path:
Expand Down Expand Up @@ -134,8 +136,8 @@ class StoreStatus:
store: Store
exists: bool
age_days: Optional[float]
problems: List[str] = field(default_factory=list) # hard failures
warnings: List[str] = field(default_factory=list) # advisory, not a failure
problems: List[str] = field(default_factory=list) # hard failures
warnings: List[str] = field(default_factory=list) # advisory, not a failure

@property
def ok(self) -> bool:
Expand Down Expand Up @@ -166,9 +168,7 @@ def evaluate(stores: Optional[List[Store]] = None) -> List[StoreStatus]:
if not exists:
problems.append(f"missing at {s.path}")
elif not s.event_driven:
past_sla = (
s.max_age_days is not None and age is not None and age > s.max_age_days
)
past_sla = s.max_age_days is not None and age is not None and age > s.max_age_days
if past_sla and s.scheduled:
# A store its own scheduler is supposed to keep fresh is stale:
# the scheduler is broken. Hard fail — someone must fix it.
Expand All @@ -187,7 +187,11 @@ def evaluate(stores: Optional[List[Store]] = None) -> List[StoreStatus]:

out.append(
StoreStatus(
store=s, exists=exists, age_days=age, problems=problems, warnings=warnings
store=s,
exists=exists,
age_days=age,
problems=problems,
warnings=warnings,
)
)
return out
Loading
Loading