Persist Brain corpus for nightly synthesis

.changeset/bright-brains-dream.md

@@ -0,0 +1,5 @@
+---
+"@roomote/web": minor
+---
+
+Store the Brain corpus as Markdown on its persistent volume so gbrain's built-in nightly synthesis and pattern detection can operate on collected tasks and integrations.

.docker/gbrain/Dockerfile

@@ -11,7 +11,7 @@ FROM oven/bun:1.3.14
 ARG GBRAIN_REF=ac402f5
 
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends postgresql-client \
+  && apt-get install -y --no-install-recommends git postgresql-client \
   && rm -rf /var/lib/apt/lists/*
 
 RUN bun install -g "github:garrytan/gbrain#${GBRAIN_REF}"
@@ -30,8 +30,8 @@ RUN set -e; \
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-# Generated tokens and local service config live on the volume; corpus data
-# lives in Postgres.
+# Generated tokens, local service config, and the Markdown corpus live on the
+# volume. Postgres remains gbrain's query index and durable job backend.
 VOLUME /data
 ENV GBRAIN_DATA_DIR=/data
 

.docker/gbrain/entrypoint.sh

@@ -14,6 +14,9 @@ DATA_DIR="${GBRAIN_DATA_DIR:-/data}"
 BRAIN_DIR="$DATA_DIR/brain"
 TOKEN_FILE="$DATA_DIR/admin-bootstrap-token"
 CONFIG_FILE="$DATA_DIR/.gbrain/config.json"
+STORAGE_LAYOUT_FILE="$DATA_DIR/roomote-brain-storage-layout"
+STORAGE_LAYOUT_VERSION="filesystem-v1"
+STORAGE_LAYOUT_RESETTING="${STORAGE_LAYOUT_VERSION}-resetting"
 PORT="${GBRAIN_PORT:-8931}"
 # Width of the vector column, fixed when the brain is created. Defaults to
 # text-embedding-3-small, which is what both providers serve by default.
@@ -24,6 +27,13 @@ EMBEDDING_DIMENSIONS="${GBRAIN_EMBEDDING_DIMENSIONS:-1536}"
 
 mkdir -p "$DATA_DIR"
 
+write_storage_layout() {
+  layout="$1"
+  temporary_layout_file="${STORAGE_LAYOUT_FILE}.tmp"
+  printf '%s\n' "$layout" > "$temporary_layout_file"
+  mv -f "$temporary_layout_file" "$STORAGE_LAYOUT_FILE"
+}
+
 # gbrain keeps its registration in $HOME/.gbrain, not in the data dir.
 # Anchor HOME on the volume so a rebuilt container still knows its brain.
 export HOME="$DATA_DIR"
@@ -67,16 +77,44 @@ GBRAIN_DATABASE_URL="$(DATABASE_SEED_URL="$DATABASE_SEED_URL" \
   ')"
 export GBRAIN_DATABASE_URL
 
-echo "[gbrain-entrypoint] ensuring isolated Postgres database $GBRAIN_DATABASE_NAME"
-psql --dbname="$DATABASE_BOOTSTRAP_URL" --no-psqlrc --quiet -v ON_ERROR_STOP=1 \
-  -v brain_database="$GBRAIN_DATABASE_NAME" <<'SQL'
+CURRENT_STORAGE_LAYOUT="$(cat "$STORAGE_LAYOUT_FILE" 2>/dev/null || true)"
+
+if [ "$CURRENT_STORAGE_LAYOUT" != "$STORAGE_LAYOUT_VERSION" ]; then
+  echo "[gbrain-entrypoint] initializing filesystem-backed Brain (existing Brain content will be rebuilt)"
+  # Only the reset itself may repeat after an interruption. Once it completes,
+  # persist the final layout before gbrain init and the remaining config steps,
+  # so a later bootstrap failure resumes instead of dropping the fresh Brain.
+  if [ "$CURRENT_STORAGE_LAYOUT" != "$STORAGE_LAYOUT_RESETTING" ]; then
+    write_storage_layout "$STORAGE_LAYOUT_RESETTING"
+  fi
+  psql --dbname="$DATABASE_BOOTSTRAP_URL" --no-psqlrc --quiet -v ON_ERROR_STOP=1 \
+    -v brain_database="$GBRAIN_DATABASE_NAME" <<'SQL'
+SELECT pg_advisory_lock(hashtext('roomote-gbrain-database-bootstrap')) AS locked \gset
+SELECT pg_terminate_backend(pid)
+FROM pg_stat_activity
+WHERE datname = :'brain_database' AND pid <> pg_backend_pid();
+SELECT format('DROP DATABASE IF EXISTS %I', :'brain_database') \gexec
+SELECT format('CREATE DATABASE %I', :'brain_database') \gexec
+SELECT pg_advisory_unlock(hashtext('roomote-gbrain-database-bootstrap')) AS unlocked \gset
+SQL
+
+  # The target is fixed under DATA_DIR. Keep deployment credentials alongside
+  # it, but remove the old corpus/config so gbrain cannot mix storage layouts.
+  rm -rf "$BRAIN_DIR"
+  rm -f "$CONFIG_FILE"
+  write_storage_layout "$STORAGE_LAYOUT_VERSION"
+else
+  echo "[gbrain-entrypoint] ensuring isolated Postgres database $GBRAIN_DATABASE_NAME"
+  psql --dbname="$DATABASE_BOOTSTRAP_URL" --no-psqlrc --quiet -v ON_ERROR_STOP=1 \
+    -v brain_database="$GBRAIN_DATABASE_NAME" <<'SQL'
 SELECT pg_advisory_lock(hashtext('roomote-gbrain-database-bootstrap')) AS locked \gset
 SELECT format('CREATE DATABASE %I', :'brain_database')
 WHERE NOT EXISTS (
   SELECT FROM pg_database WHERE datname = :'brain_database'
 ) \gexec
 SELECT pg_advisory_unlock(hashtext('roomote-gbrain-database-bootstrap')) AS unlocked \gset
 SQL
+fi
 
 if [ -z "${GBRAIN_ADMIN_BOOTSTRAP_TOKEN:-}" ]; then
   if [ ! -s "$TOKEN_FILE" ]; then
@@ -218,6 +256,17 @@ if [ ! -s "$CONFIG_FILE" ]; then
   fi
 fi
 
+# Roomote's collectors write through gbrain's MCP API. Pointing the default
+# source at a real directory makes every successful put_page also render a
+# Markdown artifact there. The same checkout is the corpus for gbrain's
+# built-in synthesize and pattern phases, which otherwise skip on a Postgres
+# brain because they have no filesystem input.
+mkdir -p "$BRAIN_DIR"
+gbrain config set sync.repo_path "$BRAIN_DIR" >/dev/null
+gbrain config set dream.synthesize.session_corpus_dir "$BRAIN_DIR" >/dev/null
+gbrain config set dream.synthesize.enabled true >/dev/null
+echo "[gbrain-entrypoint] corpus checkout: $BRAIN_DIR (filesystem + Postgres index)"
+
 # Route gbrain's OpenRouter reranker through the same Roomote credential
 # gateway as embeddings and chat. Do this after initialization so exposing an
 # OpenRouter-compatible endpoint does not change which provider gbrain chooses

apps/docs/brain.mdx

@@ -62,9 +62,13 @@ precedence over the deployment's general provider keys.
 With no provider configured anywhere, the Brain stays inert. Agents are not
 told it exists, and nothing is ingested.
 
-Roomote schedules one maintenance pass each night. The gbrain service runs
-gbrain's own durable worker, so synthesis, embedding catch-up, and the rest of
-the maintenance algorithm stay upstream behavior rather than a Roomote fork.
+Roomote schedules one maintenance pass each night. Every page written through
+the Brain is also rendered into the persistent Markdown corpus, so gbrain's
+built-in synthesis can judge the newly collected material, write linked
+reflections and original ideas, and detect recurring patterns across those
+reflections. The gbrain service runs its own durable worker, so that synthesis,
+embedding catch-up, and the rest of the maintenance algorithm stay upstream
+behavior rather than a Roomote fork.
 
 <Note>
   Self-hosted Compose deployments start the Brain from the `brain`
@@ -142,13 +146,22 @@ matching on keywords alone.
 
 ## Operating it
 
-- **Back up Postgres.** The Brain uses an isolated `gbrain` database in the
-  same Postgres instance as Roomote, so the normal database backup covers the
-  corpus. Its small container volume holds only service configuration and
-  generated bootstrap credentials.
-- **Losing the Brain database is recoverable but not free.** Task history and
-  integration sources re-ingest from scratch, but the deployment starts cold
-  until that finishes.
+- **Back up the Brain volume.** The Railway template schedules daily and
+  weekly backups for it. On other platforms, include `/data/brain` in the
+  deployment's normal volume backups. That directory is the Markdown corpus,
+  including pages produced by nightly synthesis. Postgres holds the searchable
+  index, extracted facts, and durable maintenance jobs; keeping it in the
+  normal database backup makes restores faster, but it is not the only copy of
+  the source pages.
+- **Losing the Brain is recoverable but not free.** If the volume or isolated
+  `gbrain` database is recreated, Roomote re-registers its clients, resets its
+  ingestion checkpoints, and backfills task history and integration sources.
+  The deployment starts cold until that finishes, and regenerated synthesis
+  may not be byte-for-byte identical.
+- **The filesystem cutover rebuilds older Brains once.** The first start of a
+  filesystem-backed image replaces a Postgres-only Brain instead of trying to
+  merge the old index into an empty checkout. Roomote then repopulates it from
+  its connected sources.
 - **The Brain has no public route.** It is never exposed to the internet, and
   task sandboxes reach it only through Roomote's API with their run token,
   which grants read access only.

deploy/ci/validate-deployment-artifacts.mjs

@@ -209,6 +209,27 @@ assert(
   'R_DISCORD_GATEWAY_SECRET' in railway.services.bullmq.env,
   'railway: bullmq must receive R_DISCORD_GATEWAY_SECRET',
 );
+assert(
+  railway.services.gbrain?.volume === '/data' &&
+    JSON.stringify(railway.services.gbrain?.backup_schedules) ===
+      JSON.stringify(['DAILY', 'WEEKLY']),
+  'railway: gbrain must retain daily and weekly volume backups',
+);
+
+const gbrainEntrypoint = read('.docker/gbrain/entrypoint.sh');
+const gbrainResetIndex = gbrainEntrypoint.indexOf(
+  'write_storage_layout "$STORAGE_LAYOUT_RESETTING"',
+);
+const gbrainCutoverCompleteIndex = gbrainEntrypoint.indexOf(
+  'write_storage_layout "$STORAGE_LAYOUT_VERSION"',
+);
+const gbrainInitIndex = gbrainEntrypoint.indexOf('\n    gbrain init');
+assert(
+  gbrainResetIndex >= 0 &&
+    gbrainCutoverCompleteIndex > gbrainResetIndex &&
+    gbrainCutoverCompleteIndex < gbrainInitIndex,
+  'gbrain: filesystem cutover must be recorded before fallible initialization',
+);
 
 const render = YAML.parse(read('render.yaml'));
 const renderServices = new Map(
@@ -496,6 +517,7 @@ for (const script of [
   'deploy/ci/upgrade-compatibility.sh',
   'deploy/host/tests/backup-restore.integration.sh',
   'deploy/host/tests/upgrade-failed-pull.sh',
+  '.docker/gbrain/entrypoint.sh',
 ]) {
   execFileSync('bash', ['-n', join(root, script)], { stdio: 'pipe' });
 }

deploy/coolify/README.md

@@ -269,14 +269,14 @@ the key is set, because Coolify's compose parser does not honour `profiles`.
 That is a supported state and costs about 370 MB of idle memory. With the
 key empty, agents are told nothing about the Brain and no ingestion runs.
 Deployments that want no memory at all can delete the `gbrain` service and
-the `gbrain_data` configuration volume from the compose file before deploying.
+the `gbrain_data` corpus volume from the compose file before deploying.
 
 Two operational notes:
 
-- **Back up Postgres.** The corpus lives in an isolated `gbrain` database in
-  the same Postgres instance as Roomote, so the normal `pg_data` backup covers
-  it. `gbrain_data` only holds service configuration and generated bootstrap
-  credentials.
+- **Back up the Brain volume.** `gbrain_data` holds the Markdown corpus,
+  including pages produced by nightly synthesis. The isolated `gbrain`
+  database in Postgres holds its searchable index, extracted facts, and
+  durable jobs, so keep it in the normal `pg_data` backup too.
 - **Model choice is a variable, not a rebuild.** `R_BRAIN_MODEL` selects the
   synthesis model, `R_BRAIN_EMBEDDING_MODEL` the embedding model, and
   `R_BRAIN_RERANKER_MODEL` the reranker. Set them on the app services. Leave

deploy/coolify/docker-compose.yaml

@@ -146,7 +146,7 @@ services:
   # it through the api service's proxy with their run token. It runs
   # unconditionally because Coolify's compose parser does not honour
   # `profiles`, and idling costs about 370 MB. Delete this service and its
-  # small configuration volume if the deployment should have no memory.
+  # corpus volume if the deployment should have no memory.
   gbrain:
     image: ghcr.io/roocodeinc/roomote-gbrain:develop
     restart: unless-stopped

deploy/railway/README.md

@@ -111,6 +111,7 @@ follow the image.
 | `Postgres`      | Railway managed PostgreSQL     | —                                                  | no                         | managed            |
 | `Redis`         | Railway managed Redis          | —                                                  | no                         | managed            |
 | `minio`         | pinned `minio/minio` + `/data` | `minio server /data --console-address :9001`       | yes (HTTP proxy port 9000) | —                  |
+| `gbrain`        | `roomote-gbrain:<channel>` + `/data` | image entrypoint                               | no                         | `/health`          |
 | `web`           | `roomote-app:<channel>`        | `/roomote/.docker/app/entrypoint.sh web`           | yes (HTTP proxy port 8080) | `/health`          |
 | `api`           | `roomote-app:<channel>`        | `/roomote/.docker/app/entrypoint.sh api`           | yes (HTTP proxy port 8080) | `/health/liveness` |
 | `controller`    | `roomote-app:<channel>`        | `/roomote/.docker/app/entrypoint.sh controller`    | no                         | —                  |
@@ -424,10 +425,12 @@ want memory can delete the `gbrain` service entirely.
 
 Two operational notes:
 
-- **Back up the volume.** The `/data` volume holds the corpus. Losing it is
-  recoverable — task history and integration sources re-ingest, and Roomote
-  re-registers its clients automatically when the Brain no longer recognizes
-  them — but the deployment starts cold until that finishes.
+- **The template backs up the volume.** The `/data` volume holds the corpus,
+  and new template deployments schedule Railway's daily and weekly backups
+  for it. Losing it is recoverable — task history and integration sources
+  re-ingest, and Roomote re-registers its clients automatically when the Brain
+  no longer recognizes them — but the deployment starts cold until that
+  finishes.
 - **Model choice is a variable, not a rebuild.** `R_BRAIN_MODEL` selects the
   synthesis model, `R_BRAIN_EMBEDDING_MODEL` the embedding model, and
   `R_BRAIN_RERANKER_MODEL` the reranker, all set on **api**. Leave them empty
@@ -460,9 +463,10 @@ Two operational notes:
   **bullmq** (and any other service that already copies `api.*` secrets)
   before enabling or continuing Discord. Template edits do not rewrite
   variables on existing projects.
-- **Back up** the Railway Postgres database (Railway backups or `pg_dump`)
-  and the MinIO volume or external bucket. Everything else is reproducible
-  from config.
+- **Back up** the Railway Postgres database, the MinIO volume or external
+  bucket, and the Brain volume. The template schedules volume backups for all
+  three Railway-managed copies; keep equivalent coverage if you replace any
+  of them with external infrastructure.
 - **Costs** split three ways: Railway hosts the control plane (web, api, preview-proxy,
   controller, bullmq, Postgres, Redis, MinIO), while task execution bills
   through your hosted sandbox provider and model usage bills
@@ -513,6 +517,10 @@ references, like `R_APP_URL` — also open each app service's Variables
 tab on the scratch project and confirm the resolved values are real URLs,
 not literal `${{...}}` strings.
 
+For Brain storage changes, also open the `gbrain` service's **Backups** tab
+and confirm its `/data` volume has both Daily and Weekly schedules before
+publishing the template update.
+
 The `R_APP_URL` deploy-time prompt needs its own check on the scratch
 deploy: on the deploy screen, open the api service's **Configure** step and
 expand its pre-configured environment variables — `R_APP_URL` must

deploy/railway/template.yaml

@@ -130,6 +130,7 @@ services:
     image: gbrain
     public_domain: no # private network only; the api service proxies agent access
     volume: /data
+    backup_schedules: [DAILY, WEEKLY] # configure both on the /data volume in Template Composer
     env:
       GBRAIN_DATABASE_URL: ${{Postgres.DATABASE_URL}}
       # Railway's private network is IPv6-only, so the Brain must bind :: to
@@ -155,10 +156,10 @@ services:
       through the api service — the Brain itself has no public domain and is
       never reachable from the internet. It stays idle until
       a Brain provider key is set on api, so deploying with both keys
-      empty costs only the container. The volume holds the corpus, so back it
-      up like Postgres; deleting it is recoverable (task history and
-      integration sources re-ingest) but not free. Delete this service
-      entirely if the deployment should have no memory.
+      empty costs only the container. The volume holds the corpus and has
+      daily plus weekly Railway backups; deleting it is recoverable (task
+      history and integration sources re-ingest) but not free. Delete this
+      service entirely if the deployment should have no memory.
 
   api:
     image: app