From 55a53cd596d8f1d8b15a5a4ab862788d3117b405 Mon Sep 17 00:00:00 2001 From: Ryan Nadel Date: Sun, 26 Jul 2026 12:49:52 -0700 Subject: [PATCH] Separate demo and real CogMap data Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a84aee78-f9c8-47ca-b640-2af5af136037 --- .agents/skills/build-cogmap/SKILL.md | 33 +++++++----- .../assets/demo/sources/README.md | 21 +++----- .../build-cogmap/scripts/cogmap_paths.py | 52 ++++++++++++++----- .../skills/build-cogmap/scripts/refresh.py | 22 +++++++- README.md | 9 ++-- cogmap-app/README.md | 18 ++++--- cogmap-app/skills/build-cogmap/SKILL.md | 33 +++++++----- .../assets/demo/sources/README.md | 21 +++----- .../build-cogmap/scripts/cogmap_paths.py | 52 ++++++++++++++----- .../skills/build-cogmap/scripts/refresh.py | 22 +++++++- tests/test_cogmap_pipeline.py | 46 +++++++++++++++- 11 files changed, 232 insertions(+), 97 deletions(-) diff --git a/.agents/skills/build-cogmap/SKILL.md b/.agents/skills/build-cogmap/SKILL.md index 93e4440..4420845 100644 --- a/.agents/skills/build-cogmap/SKILL.md +++ b/.agents/skills/build-cogmap/SKILL.md @@ -35,15 +35,15 @@ Everything else resolves automatically via `scripts/cogmap_paths.py` (using build-cogmap/ <- this skill SKILL.md scripts/ engine: refresh.py + stage scripts - assets/demo/ bundled demo corpus + prebuilt artifacts (seed data) + assets/demo/ bundled demo corpus + prebuilt artifacts (demo-only) requirements.txt ``` ## The workspace (where the user's data lives) Runtime data is written to a **project-local workspace**, never into the skill -folder. It defaults to `./cogmap` under the current working directory and is -auto-created + seeded from `assets/demo/` on first run: +folder. A real run defaults to an empty `./cogmap` under the current working +directory: ``` cogmap/ <- the workspace (in the user's working dir) @@ -52,6 +52,11 @@ cogmap/ <- the workspace (in the user's working dir) output/ knowledge-base-viz.html + knowledge-base-viz-data.json <- deliverable ``` +The synthetic corpus is never copied into a real workspace. To render it +explicitly, run `refresh.py --demo --no-open`; this creates and uses a separate +`./cogmap-demo` workspace. Never add user notes to `cogmap-demo/`, and never copy +`sample-notes.txt` into `cogmap/sources/`. + Run `refresh.py` from a stable directory (e.g. the user's project root) so the `./cogmap` workspace is predictable. On startup `refresh.py` prints the resolved workspace paths — use those when reporting the deliverable location. To relocate @@ -87,9 +92,9 @@ When the user wants to add notes, put them in the workspace `sources/` and refre - If they give a **path**, copy the file(s) into `/sources/`. - If they **paste** text, write it to a new `/sources/.md` (or `.txt`). - Encourage a date near the top of each note for a chronological timeline. -- Remove the seeded `sources/sample-notes.txt` once real notes are added (it's a - synthetic demo corpus, not user content). Adding real notes also triggers the - pipeline's foreign-corpus reset, which clears the demo's resolved/insight files. +- Use only the user's content in this real workspace. If the exact bundled + `sample-notes.txt` is present, the refresh stops instead of generating a mixed + graph. Then run the loop below. See `sources/README.md` in the workspace for the user-facing guide. @@ -110,6 +115,8 @@ PYTHONIOENCODING=utf-8 python /scripts/refresh.py --with-resolve --with-s ``` Flags: +- `--demo` — render the bundled synthetic corpus in a separate `./cogmap-demo` + workspace. This is for product evaluation only, never real note ingestion. - `--with-resolve` — on concept-set change, cluster new names into existing/new canonical concepts (best quality). Without it, assemble uses a lossless singleton fallback (new concepts appear un-merged). @@ -244,14 +251,12 @@ warning and the branch it pushed so the user can enable Pages manually. ## Notes -- First run **seeds the workspace** from the bundled demo (`assets/demo/`) and - seeds the extraction baseline from whatever extractions already cover the - current notes: a fresh install left unchanged is a no-op (renders the demo), but - if the user has swapped in their own notes those chunks aren't covered, so - extraction is triggered for them. -- The seeded `sources/sample-notes.txt` + its extractions/resolution/insights are - a **synthetic demo** so a fresh install renders immediately. They are replaced - the moment the user adds real notes and refreshes. +- A first real run creates an empty `cogmap/` workspace and exits with instructions + until user notes are present. +- `refresh.py --demo` creates a distinct `cogmap-demo/` workspace seeded from + `assets/demo/`. The demo and real corpus never share sources, state, or output. +- A real refresh rejects the exact bundled `sample-notes.txt` so sample concepts + cannot silently leak into a user's graph. - Correctness mechanism: `v3_aggregate.py` filters extractions by the current chunk IDs, so stale extractions for edited/removed chunks are auto-dropped. Chunk IDs are content-only (position-independent), so mid-file edits don't cascade. diff --git a/.agents/skills/build-cogmap/assets/demo/sources/README.md b/.agents/skills/build-cogmap/assets/demo/sources/README.md index b96548b..99208e7 100644 --- a/.agents/skills/build-cogmap/assets/demo/sources/README.md +++ b/.agents/skills/build-cogmap/assets/demo/sources/README.md @@ -1,7 +1,10 @@ -# Your notes go here +# Synthetic demo sources -Drop your notes into this `sources/` folder, then ask your agent to -**"refresh the knowledge base"** (or run `python ../pipeline/refresh.py`). +This folder belongs only to the explicit CogMap demo. Do not add user notes here. +Run `refresh.py --demo` to render it in a separate `cogmap-demo/` workspace. + +For a real CogMap, put user notes in `cogmap/sources/` and run `refresh.py` +without `--demo`. ## Supported formats @@ -29,15 +32,7 @@ current year (override with the `COGMAP_DEFAULT_YEAR` env var; legacy `OSLER_DEFAULT_YEAR` is also accepted). If nothing in the corpus is dated, the timeline collapses — so add at least a few date anchors. -## Adding notes with your coding agent - -You don't have to touch files manually. Just tell the agent, e.g.: - -- *"Add these notes to the knowledge base: <paste text>"* — it saves them here and refreshes. -- *"Import my journal.md into the knowledge base."* -- *"I have a OneNote export (.onex) — ingest it."* — it runs the `.onex` converter (best-effort ~66%; a clean `.txt`/`.md` export is always better). - -## Sample content +## Demo content `sample-notes.txt` is a **synthetic demo** (a research journal on urban mobility). -Delete it and add your own notes when you're ready — then refresh. +It is not a template or user content and must remain isolated from real CogMaps. diff --git a/.agents/skills/build-cogmap/scripts/cogmap_paths.py b/.agents/skills/build-cogmap/scripts/cogmap_paths.py index df651cf..46c709b 100644 --- a/.agents/skills/build-cogmap/scripts/cogmap_paths.py +++ b/.agents/skills/build-cogmap/scripts/cogmap_paths.py @@ -5,19 +5,19 @@ user's notes or generated output into a shared/global skill directory. Workspace root (APP) resolution order: - 1. `COGMAP_APP` env var, if set (advanced / explicit relocation). - 2. Legacy repo layout: if these scripts sit in a folder literally named + 1. In demo mode, `COGMAP_DEMO_APP` or a dedicated `./cogmap-demo`. + 2. `COGMAP_APP` env var, if set (advanced / explicit relocation). + 3. Legacy repo layout: if these scripts sit in a folder literally named `pipeline`, the app is its parent (backward compatible with the original `cogmap-app/` checkout that kept sources/work/output beside `pipeline/`). - 3. Otherwise a project-local `./cogmap` folder under the current working dir. + 4. Otherwise a project-local `./cogmap` folder under the current working dir. Individual folders can still be overridden with `COGMAP_SOURCES`, `COGMAP_WORK`, `COGMAP_OUTPUT` (legacy `OSLER_*` names are accepted as fallbacks). -On first use of a project-local workspace, the pipeline seeds it from the bundled -demo (`/assets/demo/`) so a fresh install renders immediately. Once the -user drops their own notes into `sources/` and refreshes, the pipeline's -foreign-corpus reset takes over and clears the demo artifacts. +Real workspaces are always created empty. The bundled synthetic corpus is copied +only when `COGMAP_DEMO=1` (normally set by `refresh.py --demo`), and the CLI puts +that explicit demo in a separate `./cogmap-demo` workspace by default. """ import os import pathlib @@ -32,6 +32,14 @@ def _env(*names): return None +def _env_enabled(*names): + for n in names: + value = os.environ.get(n, "").strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + return False + + # Where these scripts live (the skill's scripts/ dir). refresh.py runs its # sibling stage scripts from here; PIPELINE is kept as a backward-compatible alias. SCRIPTS = pathlib.Path(__file__).resolve().parent @@ -39,9 +47,12 @@ def _env(*names): # Bundled demo corpus + prebuilt artifacts (present only in the packaged skill). _DEMO = SCRIPTS.parent / "assets" / "demo" +DEMO_MODE = _env_enabled("COGMAP_DEMO") def _resolve_app(): + if DEMO_MODE: + return _env("COGMAP_DEMO_APP") or pathlib.Path.cwd() / "cogmap-demo", True env = _env("COGMAP_APP", "OSLER_APP") if env: return env, False @@ -53,19 +64,26 @@ def _resolve_app(): APP, _project_local = _resolve_app() +APP = APP.expanduser().resolve() # Pin the resolved app into the environment so child processes (refresh.py spawns # the stage scripts with a different cwd) resolve to the SAME workspace instead of # re-deriving a project-local path from their own cwd. os.environ["COGMAP_APP"] = str(APP) +if DEMO_MODE: + os.environ["COGMAP_DEMO_APP"] = str(APP) -SOURCES = _env("COGMAP_SOURCES", "OSLER_SOURCES") or APP / "sources" -WORK = _env("COGMAP_WORK", "OSLER_WORK") or APP / "work" -OUTPUT = _env("COGMAP_OUTPUT", "OSLER_OUTPUT") or APP / "output" +if DEMO_MODE: + # Demo mode must never inherit path overrides that point at a real corpus. + SOURCES, WORK, OUTPUT = APP / "sources", APP / "work", APP / "output" +else: + SOURCES = _env("COGMAP_SOURCES", "OSLER_SOURCES") or APP / "sources" + WORK = _env("COGMAP_WORK", "OSLER_WORK") or APP / "work" + OUTPUT = _env("COGMAP_OUTPUT", "OSLER_OUTPUT") or APP / "output" def _seed_from_demo(): - """Copy the bundled demo into a brand-new project-local workspace once.""" + """Copy the bundled demo into an explicitly selected demo workspace once.""" marker = APP / ".cogmap_initialized" if marker.exists() or not _DEMO.exists(): return @@ -77,13 +95,22 @@ def _seed_from_demo(): marker.write_text("seeded from bundled demo\n", encoding="utf-8") -if _project_local: +if DEMO_MODE: _seed_from_demo() for _d in (SOURCES, WORK, OUTPUT): _d.mkdir(parents=True, exist_ok=True) +def contains_bundled_sample(): + """Return whether the real source folder contains the exact synthetic corpus.""" + bundled = _DEMO / "sources" / "sample-notes.txt" + candidate = SOURCES / "sample-notes.txt" + if not bundled.is_file() or not candidate.is_file(): + return False + return bundled.read_bytes() == candidate.read_bytes() + + def load_resolved(path): """Load v3_resolved.json tolerantly and return {'concepts': [...]}. @@ -110,4 +137,3 @@ def load_resolved(path): return {"concepts": v} return {"concepts": []} return {"concepts": []} - diff --git a/.agents/skills/build-cogmap/scripts/refresh.py b/.agents/skills/build-cogmap/scripts/refresh.py index 5eb3efe..fb576bd 100644 --- a/.agents/skills/build-cogmap/scripts/refresh.py +++ b/.agents/skills/build-cogmap/scripts/refresh.py @@ -12,6 +12,7 @@ continues. When nothing is left to do it rebuilds the HTML and exits 0. Flags: + --demo render the synthetic corpus in a separate demo workspace --from-onex re-extract the .onex -> clean text first (best-effort ~66%) --with-resolve re-run entity resolution when the concept set changed --with-synth re-run insight synthesis when the graph changed @@ -21,8 +22,18 @@ publish the finished HTML to a GitHub Pages branch """ import json, re, hashlib, pathlib, subprocess, sys, os, time, shutil +ARGS = set(sys.argv[1:]) +if '--demo' in ARGS: + os.environ['COGMAP_DEMO'] = '1' sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from cogmap_paths import WORK, OUTPUT, PIPELINE, load_resolved +from cogmap_paths import ( + DEMO_MODE, + WORK, + OUTPUT, + PIPELINE, + contains_bundled_sample, + load_resolved, +) from extraction_batches import ( MAX_EXTRACTION_CONCURRENCY, batch_items, @@ -39,7 +50,6 @@ STATEF = CACHE / 'state.json' DATA = ROOT / 'knowledge-base-viz-data.json' PY = sys.executable -ARGS = set(sys.argv[1:]) def arg_value(name, default): @@ -358,8 +368,16 @@ def run_soft(script): def main(): from cogmap_paths import APP, SOURCES print('workspace: {}'.format(APP)) + print(' mode: {}'.format('synthetic demo' if DEMO_MODE else 'real corpus')) print(' sources: {}'.format(SOURCES)) print(' output: {}'.format(OUTPUT)) + if not DEMO_MODE and contains_bundled_sample(): + print('\nERROR: the real workspace contains the bundled synthetic ' + '`sample-notes.txt` corpus.\n' + 'Remove that file before building a real CogMap. To view the sample ' + 'without mixing it with real notes, run `refresh.py --demo --no-open`; ' + 'it uses a separate `cogmap-demo/` workspace.') + sys.exit(2) if '--from-onex' in ARGS: print('extracting .onex -> clean text (best-effort)...') r = run_soft('extract_onex.py') diff --git a/README.md b/README.md index c899e1e..07aadfa 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,16 @@ or dashboard. ## See it in action -A synthetic demo corpus ships inside the skill. On first run, CogMap seeds a -project-local `cogmap/` workspace from that demo and renders: +A synthetic demo corpus ships inside the skill, but it is isolated from real +CogMaps. Run the demo explicitly with `refresh.py --demo`; it renders in: ```text -cogmap/output/knowledge-base-viz.html +cogmap-demo/output/knowledge-base-viz.html ``` The generated page is fully local: no server, database, or API key is required. -Replace the demo notes with your own Markdown or text files when you are ready. +Real runs use a separate, initially empty `cogmap/` workspace, so synthetic +concepts cannot leak into a user's map. ## How CogMap uses agents diff --git a/cogmap-app/README.md b/cogmap-app/README.md index ac7a084..79043a5 100644 --- a/cogmap-app/README.md +++ b/cogmap-app/README.md @@ -56,8 +56,7 @@ cogmap-app/ Claude Code plugin root The **engine** (the `scripts/` folder) is immutable and lives in the skill. Your **data** lives in a separate, project-local **workspace** so an installed skill never writes notes or output into a shared skill directory. The workspace defaults -to `./cogmap` under the directory you run from, and is **auto-created and seeded -from the bundled demo** on first run: +to an initially empty `./cogmap` under the directory you run from: ``` cogmap/ the workspace (created in your working dir) @@ -68,6 +67,8 @@ cogmap/ the workspace (created in your working dir) Relocate the workspace with `COGMAP_APP` (or the finer `COGMAP_SOURCES` / `COGMAP_WORK` / `COGMAP_OUTPUT`; legacy `OSLER_*` names still accepted). +The synthetic corpus is available only through `refresh.py --demo`, which seeds a +separate `./cogmap-demo` workspace. ## Prerequisites @@ -82,9 +83,10 @@ CogMap is a **coding agent skill**, not a standalone app you run by hand: 1. Install the skill in your agent (see below), or open this checkout in the agent. 2. Tell the agent: *"build the knowledge base"* (or *"refresh the knowledge base"*). - On first run it seeds `./cogmap` from the demo and renders it. -3. To use your own notes, tell the agent *"add my notes from ``"* — - it writes them into `cogmap/sources/`, removes the demo corpus, and refreshes. + It creates an empty real workspace at `./cogmap`. +3. Tell the agent *"add my notes from ``"* — it writes them into + `cogmap/sources/` and refreshes. To evaluate the synthetic example instead, run + `refresh.py --demo`; demo state and output stay under `cogmap-demo/`. 4. Open `cogmap/output/knowledge-base-viz.html` when the agent reports completion (the agent prints the exact workspace path on every run). 5. To share the result from a GitHub-backed repo, ask the agent to publish it to @@ -168,8 +170,8 @@ python cogmap-app/tools/sync_skill.py --check # CI/pre-commit drift check results are retained while only missing or invalid batches are re-run. - **Idempotent ingestion** — deterministic batch IDs make retries replace the same extraction archive rather than duplicating evidence or edge weights. -- **Foreign-corpus reset** — the first time you swap in your own notes, the demo's - resolved/insight artifacts are cleared so your corpus doesn't inherit demo - categories. +- **Demo isolation** — synthetic sources, state, and output live only in the + explicit `cogmap-demo/` workspace; real refreshes reject the bundled sample + corpus instead of producing a mixed graph. - **Name-keyed graph** — concepts/insights are keyed by canonical name, so the incremental resolve/synth steps merge cleanly into the existing graph. diff --git a/cogmap-app/skills/build-cogmap/SKILL.md b/cogmap-app/skills/build-cogmap/SKILL.md index 93e4440..4420845 100644 --- a/cogmap-app/skills/build-cogmap/SKILL.md +++ b/cogmap-app/skills/build-cogmap/SKILL.md @@ -35,15 +35,15 @@ Everything else resolves automatically via `scripts/cogmap_paths.py` (using build-cogmap/ <- this skill SKILL.md scripts/ engine: refresh.py + stage scripts - assets/demo/ bundled demo corpus + prebuilt artifacts (seed data) + assets/demo/ bundled demo corpus + prebuilt artifacts (demo-only) requirements.txt ``` ## The workspace (where the user's data lives) Runtime data is written to a **project-local workspace**, never into the skill -folder. It defaults to `./cogmap` under the current working directory and is -auto-created + seeded from `assets/demo/` on first run: +folder. A real run defaults to an empty `./cogmap` under the current working +directory: ``` cogmap/ <- the workspace (in the user's working dir) @@ -52,6 +52,11 @@ cogmap/ <- the workspace (in the user's working dir) output/ knowledge-base-viz.html + knowledge-base-viz-data.json <- deliverable ``` +The synthetic corpus is never copied into a real workspace. To render it +explicitly, run `refresh.py --demo --no-open`; this creates and uses a separate +`./cogmap-demo` workspace. Never add user notes to `cogmap-demo/`, and never copy +`sample-notes.txt` into `cogmap/sources/`. + Run `refresh.py` from a stable directory (e.g. the user's project root) so the `./cogmap` workspace is predictable. On startup `refresh.py` prints the resolved workspace paths — use those when reporting the deliverable location. To relocate @@ -87,9 +92,9 @@ When the user wants to add notes, put them in the workspace `sources/` and refre - If they give a **path**, copy the file(s) into `/sources/`. - If they **paste** text, write it to a new `/sources/.md` (or `.txt`). - Encourage a date near the top of each note for a chronological timeline. -- Remove the seeded `sources/sample-notes.txt` once real notes are added (it's a - synthetic demo corpus, not user content). Adding real notes also triggers the - pipeline's foreign-corpus reset, which clears the demo's resolved/insight files. +- Use only the user's content in this real workspace. If the exact bundled + `sample-notes.txt` is present, the refresh stops instead of generating a mixed + graph. Then run the loop below. See `sources/README.md` in the workspace for the user-facing guide. @@ -110,6 +115,8 @@ PYTHONIOENCODING=utf-8 python /scripts/refresh.py --with-resolve --with-s ``` Flags: +- `--demo` — render the bundled synthetic corpus in a separate `./cogmap-demo` + workspace. This is for product evaluation only, never real note ingestion. - `--with-resolve` — on concept-set change, cluster new names into existing/new canonical concepts (best quality). Without it, assemble uses a lossless singleton fallback (new concepts appear un-merged). @@ -244,14 +251,12 @@ warning and the branch it pushed so the user can enable Pages manually. ## Notes -- First run **seeds the workspace** from the bundled demo (`assets/demo/`) and - seeds the extraction baseline from whatever extractions already cover the - current notes: a fresh install left unchanged is a no-op (renders the demo), but - if the user has swapped in their own notes those chunks aren't covered, so - extraction is triggered for them. -- The seeded `sources/sample-notes.txt` + its extractions/resolution/insights are - a **synthetic demo** so a fresh install renders immediately. They are replaced - the moment the user adds real notes and refreshes. +- A first real run creates an empty `cogmap/` workspace and exits with instructions + until user notes are present. +- `refresh.py --demo` creates a distinct `cogmap-demo/` workspace seeded from + `assets/demo/`. The demo and real corpus never share sources, state, or output. +- A real refresh rejects the exact bundled `sample-notes.txt` so sample concepts + cannot silently leak into a user's graph. - Correctness mechanism: `v3_aggregate.py` filters extractions by the current chunk IDs, so stale extractions for edited/removed chunks are auto-dropped. Chunk IDs are content-only (position-independent), so mid-file edits don't cascade. diff --git a/cogmap-app/skills/build-cogmap/assets/demo/sources/README.md b/cogmap-app/skills/build-cogmap/assets/demo/sources/README.md index b96548b..99208e7 100644 --- a/cogmap-app/skills/build-cogmap/assets/demo/sources/README.md +++ b/cogmap-app/skills/build-cogmap/assets/demo/sources/README.md @@ -1,7 +1,10 @@ -# Your notes go here +# Synthetic demo sources -Drop your notes into this `sources/` folder, then ask your agent to -**"refresh the knowledge base"** (or run `python ../pipeline/refresh.py`). +This folder belongs only to the explicit CogMap demo. Do not add user notes here. +Run `refresh.py --demo` to render it in a separate `cogmap-demo/` workspace. + +For a real CogMap, put user notes in `cogmap/sources/` and run `refresh.py` +without `--demo`. ## Supported formats @@ -29,15 +32,7 @@ current year (override with the `COGMAP_DEFAULT_YEAR` env var; legacy `OSLER_DEFAULT_YEAR` is also accepted). If nothing in the corpus is dated, the timeline collapses — so add at least a few date anchors. -## Adding notes with your coding agent - -You don't have to touch files manually. Just tell the agent, e.g.: - -- *"Add these notes to the knowledge base: <paste text>"* — it saves them here and refreshes. -- *"Import my journal.md into the knowledge base."* -- *"I have a OneNote export (.onex) — ingest it."* — it runs the `.onex` converter (best-effort ~66%; a clean `.txt`/`.md` export is always better). - -## Sample content +## Demo content `sample-notes.txt` is a **synthetic demo** (a research journal on urban mobility). -Delete it and add your own notes when you're ready — then refresh. +It is not a template or user content and must remain isolated from real CogMaps. diff --git a/cogmap-app/skills/build-cogmap/scripts/cogmap_paths.py b/cogmap-app/skills/build-cogmap/scripts/cogmap_paths.py index df651cf..46c709b 100644 --- a/cogmap-app/skills/build-cogmap/scripts/cogmap_paths.py +++ b/cogmap-app/skills/build-cogmap/scripts/cogmap_paths.py @@ -5,19 +5,19 @@ user's notes or generated output into a shared/global skill directory. Workspace root (APP) resolution order: - 1. `COGMAP_APP` env var, if set (advanced / explicit relocation). - 2. Legacy repo layout: if these scripts sit in a folder literally named + 1. In demo mode, `COGMAP_DEMO_APP` or a dedicated `./cogmap-demo`. + 2. `COGMAP_APP` env var, if set (advanced / explicit relocation). + 3. Legacy repo layout: if these scripts sit in a folder literally named `pipeline`, the app is its parent (backward compatible with the original `cogmap-app/` checkout that kept sources/work/output beside `pipeline/`). - 3. Otherwise a project-local `./cogmap` folder under the current working dir. + 4. Otherwise a project-local `./cogmap` folder under the current working dir. Individual folders can still be overridden with `COGMAP_SOURCES`, `COGMAP_WORK`, `COGMAP_OUTPUT` (legacy `OSLER_*` names are accepted as fallbacks). -On first use of a project-local workspace, the pipeline seeds it from the bundled -demo (`/assets/demo/`) so a fresh install renders immediately. Once the -user drops their own notes into `sources/` and refreshes, the pipeline's -foreign-corpus reset takes over and clears the demo artifacts. +Real workspaces are always created empty. The bundled synthetic corpus is copied +only when `COGMAP_DEMO=1` (normally set by `refresh.py --demo`), and the CLI puts +that explicit demo in a separate `./cogmap-demo` workspace by default. """ import os import pathlib @@ -32,6 +32,14 @@ def _env(*names): return None +def _env_enabled(*names): + for n in names: + value = os.environ.get(n, "").strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + return False + + # Where these scripts live (the skill's scripts/ dir). refresh.py runs its # sibling stage scripts from here; PIPELINE is kept as a backward-compatible alias. SCRIPTS = pathlib.Path(__file__).resolve().parent @@ -39,9 +47,12 @@ def _env(*names): # Bundled demo corpus + prebuilt artifacts (present only in the packaged skill). _DEMO = SCRIPTS.parent / "assets" / "demo" +DEMO_MODE = _env_enabled("COGMAP_DEMO") def _resolve_app(): + if DEMO_MODE: + return _env("COGMAP_DEMO_APP") or pathlib.Path.cwd() / "cogmap-demo", True env = _env("COGMAP_APP", "OSLER_APP") if env: return env, False @@ -53,19 +64,26 @@ def _resolve_app(): APP, _project_local = _resolve_app() +APP = APP.expanduser().resolve() # Pin the resolved app into the environment so child processes (refresh.py spawns # the stage scripts with a different cwd) resolve to the SAME workspace instead of # re-deriving a project-local path from their own cwd. os.environ["COGMAP_APP"] = str(APP) +if DEMO_MODE: + os.environ["COGMAP_DEMO_APP"] = str(APP) -SOURCES = _env("COGMAP_SOURCES", "OSLER_SOURCES") or APP / "sources" -WORK = _env("COGMAP_WORK", "OSLER_WORK") or APP / "work" -OUTPUT = _env("COGMAP_OUTPUT", "OSLER_OUTPUT") or APP / "output" +if DEMO_MODE: + # Demo mode must never inherit path overrides that point at a real corpus. + SOURCES, WORK, OUTPUT = APP / "sources", APP / "work", APP / "output" +else: + SOURCES = _env("COGMAP_SOURCES", "OSLER_SOURCES") or APP / "sources" + WORK = _env("COGMAP_WORK", "OSLER_WORK") or APP / "work" + OUTPUT = _env("COGMAP_OUTPUT", "OSLER_OUTPUT") or APP / "output" def _seed_from_demo(): - """Copy the bundled demo into a brand-new project-local workspace once.""" + """Copy the bundled demo into an explicitly selected demo workspace once.""" marker = APP / ".cogmap_initialized" if marker.exists() or not _DEMO.exists(): return @@ -77,13 +95,22 @@ def _seed_from_demo(): marker.write_text("seeded from bundled demo\n", encoding="utf-8") -if _project_local: +if DEMO_MODE: _seed_from_demo() for _d in (SOURCES, WORK, OUTPUT): _d.mkdir(parents=True, exist_ok=True) +def contains_bundled_sample(): + """Return whether the real source folder contains the exact synthetic corpus.""" + bundled = _DEMO / "sources" / "sample-notes.txt" + candidate = SOURCES / "sample-notes.txt" + if not bundled.is_file() or not candidate.is_file(): + return False + return bundled.read_bytes() == candidate.read_bytes() + + def load_resolved(path): """Load v3_resolved.json tolerantly and return {'concepts': [...]}. @@ -110,4 +137,3 @@ def load_resolved(path): return {"concepts": v} return {"concepts": []} return {"concepts": []} - diff --git a/cogmap-app/skills/build-cogmap/scripts/refresh.py b/cogmap-app/skills/build-cogmap/scripts/refresh.py index 5eb3efe..fb576bd 100644 --- a/cogmap-app/skills/build-cogmap/scripts/refresh.py +++ b/cogmap-app/skills/build-cogmap/scripts/refresh.py @@ -12,6 +12,7 @@ continues. When nothing is left to do it rebuilds the HTML and exits 0. Flags: + --demo render the synthetic corpus in a separate demo workspace --from-onex re-extract the .onex -> clean text first (best-effort ~66%) --with-resolve re-run entity resolution when the concept set changed --with-synth re-run insight synthesis when the graph changed @@ -21,8 +22,18 @@ publish the finished HTML to a GitHub Pages branch """ import json, re, hashlib, pathlib, subprocess, sys, os, time, shutil +ARGS = set(sys.argv[1:]) +if '--demo' in ARGS: + os.environ['COGMAP_DEMO'] = '1' sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from cogmap_paths import WORK, OUTPUT, PIPELINE, load_resolved +from cogmap_paths import ( + DEMO_MODE, + WORK, + OUTPUT, + PIPELINE, + contains_bundled_sample, + load_resolved, +) from extraction_batches import ( MAX_EXTRACTION_CONCURRENCY, batch_items, @@ -39,7 +50,6 @@ STATEF = CACHE / 'state.json' DATA = ROOT / 'knowledge-base-viz-data.json' PY = sys.executable -ARGS = set(sys.argv[1:]) def arg_value(name, default): @@ -358,8 +368,16 @@ def run_soft(script): def main(): from cogmap_paths import APP, SOURCES print('workspace: {}'.format(APP)) + print(' mode: {}'.format('synthetic demo' if DEMO_MODE else 'real corpus')) print(' sources: {}'.format(SOURCES)) print(' output: {}'.format(OUTPUT)) + if not DEMO_MODE and contains_bundled_sample(): + print('\nERROR: the real workspace contains the bundled synthetic ' + '`sample-notes.txt` corpus.\n' + 'Remove that file before building a real CogMap. To view the sample ' + 'without mixing it with real notes, run `refresh.py --demo --no-open`; ' + 'it uses a separate `cogmap-demo/` workspace.') + sys.exit(2) if '--from-onex' in ARGS: print('extracting .onex -> clean text (best-effort)...') r = run_soft('extract_onex.py') diff --git a/tests/test_cogmap_pipeline.py b/tests/test_cogmap_pipeline.py index a8f2324..4615241 100644 --- a/tests/test_cogmap_pipeline.py +++ b/tests/test_cogmap_pipeline.py @@ -377,14 +377,44 @@ def test_date_parsing_preserves_valid_month_days(self): self.assertNotIn("2025-04-30", dates) self.assertNotIn("2025-04-31", dates) - def test_demo_refresh_renders_html(self): + def test_real_refresh_starts_empty_without_sample_data(self): with tempfile.TemporaryDirectory() as td: workspace = Path(td) / "cogmap" result = run_script("refresh.py", None, Path(td), "--no-open") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("mode: real corpus", result.stdout) + self.assertIn("No notes found", result.stdout) + self.assertFalse((workspace / "sources" / "sample-notes.txt").exists()) + self.assertFalse((workspace / "output" / "knowledge-base-viz.html").exists()) + + def test_demo_refresh_uses_separate_workspace_and_renders_html(self): + with tempfile.TemporaryDirectory() as td: + workspace = Path(td) / "cogmap-demo" + configured_real_workspace = Path(td) / "configured-real-cogmap" + real_sources = configured_real_workspace / "sources" + real_sources.mkdir(parents=True) + real_note = real_sources / "private-notes.txt" + real_note.write_text("This real note must remain untouched.", encoding="utf-8") + + result = run_script( + "refresh.py", + configured_real_workspace, + Path(td), + "--demo", + "--no-open", + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("mode: synthetic demo", result.stdout) + self.assertIn(f"workspace: {workspace}", result.stdout) self.assertIn("DONE.", result.stdout) + self.assertEqual( + real_note.read_text(encoding="utf-8"), + "This real note must remain untouched.", + ) + self.assertFalse((configured_real_workspace / "work").exists()) html = workspace / "output" / "knowledge-base-viz.html" data_path = workspace / "output" / "knowledge-base-viz-data.json" self.assertTrue(html.exists(), result.stdout) @@ -400,6 +430,20 @@ def test_demo_refresh_renders_html(self): self.assertIn("window.location.pathname", html_text) self.assertIn("Review evidence", html_text) + def test_real_refresh_rejects_bundled_sample_corpus(self): + with tempfile.TemporaryDirectory() as td: + workspace = Path(td) / "cogmap" + sources = workspace / "sources" + sources.mkdir(parents=True) + bundled = SKILL_DIR / "assets" / "demo" / "sources" / "sample-notes.txt" + (sources / "sample-notes.txt").write_bytes(bundled.read_bytes()) + + result = run_script("refresh.py", workspace, None, "--no-open") + + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("real workspace contains the bundled synthetic", result.stdout) + self.assertFalse((workspace / "output" / "knowledge-base-viz.html").exists()) + def test_publish_remote_parsing_and_url(self): cases = [ ("https://github.com/octo/demo.git", ("octo", "demo")),