Fix deploy handling of source-only catalogue contexts

.github/workflows/nightly.yml

@@ -6,13 +6,18 @@ on:
   workflow_dispatch:
   pull_request:
     # Only the deploy-build job below runs from this trigger (each other job's `if:` stays schedule/dispatch-only):
-    # a full `pnpm build` is too slow to run on every PR, but a PR touching the code this specific path resolves
+    # a full `pnpm build:deploy` is too slow to run on every PR, but a PR touching the code this specific path resolves
     # through gets it anyway, instead of waiting for the nightly schedule to notice a break.
     paths:
+      - '.github/workflows/nightly.yml'
+      - 'package.json'
+      - 'pnpm-lock.yaml'
       - 'site/**'
       - 'astro.config.mts'
       - 'src/renderers/css/rendering/**'
       - '.github/workflows/deploy.yml'
+      - 'tools/prepare-*.mts'
+      - 'tools/setup*.mts'
 
 permissions:
   contents: read
@@ -86,7 +91,7 @@ jobs:
 
   asset-origin-build:
     # Exercises the one path that has broken two real deploys and that no other CI job builds at all: a full
-    # `pnpm build` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static
+    # `pnpm build:deploy` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static
     # scan catches emitted same-origin addresses; the production browser test also exercises the client loader,
     # whose descriptor path can otherwise discard an origin that is present in the emitted page.
     name: Build with ASSET_ORIGIN and exercise the client loader
@@ -123,7 +128,7 @@ jobs:
           NODE_OPTIONS: --max-old-space-size=6144
           CSSEARTH_ALLOW_MISSING_ASSETS: "1"
           ASSET_ORIGIN: https://asset-origin-ci-check.example
-        run: pnpm build
+        run: pnpm build:deploy
       - name: Check no page references a same-origin /scenes/ address
         run: node tools/check-asset-origin-scenes.mts dist
       - name: Check production navigation loads content-addressed assets

package.json

@@ -110,8 +110,9 @@
     "publish:earth-global": "node tools/objects/geographic-pages/operations/publish-global-wmts.mts --object=earth",
     "publish:runtime-assets": "node tools/publish-runtime-assets.mts",
     "refresh:earth-enso": "node tools/objects/paged-ellipsoid/refresh-earth-enso.mts && node tools/objects/dist/prepare-authored.js earth --write && node tools/prepare-text.mts earth",
-    "setup:assets": "pnpm prepare:shell && node tools/setup.mts && node tools/setup-prepared.mts",
+    "setup:assets": "pnpm prepare:shell && node tools/setup.mts && node tools/setup-prepared.mts && node tools/setup-volume-metadata.mts",
     "setup:prepared": "node tools/setup-prepared.mts",
+    "setup:volume-metadata": "node tools/setup-volume-metadata.mts",
     "status:earth-city-global": "node tools/objects/geographic-pages/operations/status-global-city.mts --object=earth",
     "telescope:publish-map": "node tools/run-typed-module.mjs tools/objects/body-map-publication.mts",
     "telescope:query": "node tools/run-typed-module.mjs tools/objects/telescopes/query.mts",

site/test/navigation-production-browser.mts

@@ -7,6 +7,7 @@ import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { inventoriedAssets, inventoriedObjectIds } from '../../tools/runtime-assets.mts';
 import { contentType } from '../../tools/publish-runtime-assets.mts';
+import { sha256 } from '../../src/platform/sha256.mts';
 
 const origin = process.env.CSSEARTH_TEST_ORIGIN ?? 'http://127.0.0.1:4212';
 const channel = process.env.PLAYWRIGHT_CHANNEL ?? 'chrome';
@@ -36,7 +37,19 @@ try {
       const url = new URL(route.request().url()), asset = assets.get(url.pathname);
       if (!asset) return route.fulfill({ status: 404, body: `Uninventoried asset: ${url.pathname}` });
       assetRequests.push(url.href);
-      return route.fulfill({ body: await readFile(asset.file), contentType: contentType(asset.key),
+      let body = await readFile(asset.file).catch((error: unknown) => {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return undefined;
+        throw error;
+      });
+      // A deploy intentionally keeps large context datasets on R2. The fake ASSET_ORIGIN route still verifies
+      // their real content-addressed bytes instead of requiring the CI checkout to download every inventory.
+      if (!body) {
+        const response = await fetch(asset.url, { signal: AbortSignal.timeout(120000) });
+        if (!response.ok) throw new Error(`Published asset unavailable: ${asset.url} (HTTP ${response.status}).`);
+        body = Buffer.from(await response.arrayBuffer());
+      }
+      if (body.length !== asset.bytes || sha256(body) !== asset.sha256) throw new Error(`Published asset identity changed: ${asset.key}.`);
+      return route.fulfill({ body, contentType: contentType(asset.key),
         headers: { 'access-control-allow-origin': '*' } });
     });
   }

tools/check-ci.test.mts

@@ -37,8 +37,15 @@ test('the deploy consumes installed assets, rebuilds only catalogues and rejects
  assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:galaxy-field:data/);
  assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:deploy-catalogues/);
  assert.equal(packageFile.scripts['prepare:deploy-catalogues'],'node tools/prepare-facilities.mts --catalog-only');
+ assert.match(packageFile.scripts['setup:assets']??'',/node tools\/setup-volume-metadata\.mts/);
  assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:(?:facilities|provenance|nebulae)(?:\s|$)/);
 });
+test('the PR asset-origin check exercises the exact deploy build path',async()=>{
+ const workflow=await readFile(new URL('../.github/workflows/nightly.yml',import.meta.url),'utf8');
+ const steps=readCiSteps(workflow,'asset-origin-build');
+ const build=steps.find(step=>step.name==='Build the site with ASSET_ORIGIN set to a test origin');
+ assert.equal(build?.run.trim(),'pnpm build:deploy');
+});
 test('--quick skips only the network and documentation steps, and refuses a job without them',async()=>{
  const lint=readCiSteps(await readFile(new URL('../.github/workflows/universe.yml',import.meta.url),'utf8'),'lint');
  const quick=quickSteps(lint);

tools/prepare-context-availability.test.mts

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { resolve } from 'node:path';
 import { inspectContextAvailability, prepareContextAvailability } from './prepare-context-availability.mts';
@@ -65,7 +65,9 @@ test('an asset-origin build verifies a missing local preview against its publish
   const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
   manifest.assets[0].sha256 = '0'.repeat(64);
   await writeFile(manifestPath, JSON.stringify(manifest));
-  assert.match((await inspectContextAvailability(root, { publicAssets: 'manifest' })).helix.reason ?? '', /Unpublished dataset preview/);
+  const unavailable = (await inspectContextAvailability(root, { publicAssets: 'manifest' })).helix;
+  if (unavailable.available) assert.fail('Changed preview identity must make the package unavailable.');
+  assert.match(unavailable.reason, /Unpublished dataset preview/);
 });
 
 test('deploy catalogue input reads the installed prepared volume instead of regenerating it', async t => {
@@ -79,3 +81,11 @@ test('deploy catalogue input reads the installed prepared volume instead of rege
   assert.deepEqual(volumes[0]?.provenance, fixture.provenance);
   assert.deepEqual(volumes[0]?.outputs, []);
 });
+
+test('deploy volume input ignores source-only catalogue contexts with no prepared lens metadata', async t => {
+  const root = await mkdtemp(resolve(tmpdir(), 'cssearth-source-context-')); t.after(() => rm(root, { recursive: true, force: true }));
+  const path = resolve(root, 'src/objects/galaxy-clusters/source/presentation.json');
+  await mkdir(resolve(path, '..'), { recursive: true });
+  await writeFile(path, JSON.stringify({ provenance: { products: [] } }));
+  assert.deepEqual(await readPreparedVolumeProvenance({ root }), []);
+});

tools/prepare-facilities.mts

@@ -142,7 +142,8 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '.
   // Deploys consume the exact prepared package restored from R2. Authoring preparation still rebuilds provenance
   // and previews from their sources, but catalog-only publication must never invent a second package identity.
   const volumes = publish === 'catalogues'
-    ? await readPreparedVolumeProvenance({ root, input })
+    ? [...await readPreparedVolumeProvenance({ root, input }),
+      ...((await prepareContextProvenance({ root, input })).map(context => ({ ...context, outputs: [] })))]
     : [...await prepareVolumeProvenance({ root, input, mirrorOrigin }), ...await prepareContextProvenance({ root, input })];
   for (const volume of volumes) {
     const document = validateObjectProvenance(volume.provenance, volume.id);

tools/prepare-volume-provenance.mts

@@ -153,28 +153,24 @@ export async function readPreparedVolumeProvenance({ root = process.cwd(), input
     });
     if (!exists) continue;
     const sourcePresentation = sourceObject(json(await input(sourcePresentationPath)));
-    if (sourcePresentation.schema === 'cssearth-volume-presentation-source@1') {
-      if (sourcePresentation.objectId !== id) throw new TypeError(`Mismatched volume presentation object: ${id}.`);
-      const descriptor = parseObjectDescriptor(json(await input(`${base}/object.json`)));
-      if (descriptor.id !== id || !descriptor.prepared || !['volume-lens-bank', 'image-layer-bank'].includes(descriptor.type))
-        throw new TypeError(`Invalid prepared volume descriptor: ${id}.`);
-      const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id);
-      const defaultLens = sourceId(sourcePresentation.defaultLens);
-      const lensIds = sourceArray(sourcePresentation.lenses, raw => sourceId(sourceObject(raw).id));
-      const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)),
-        { id, defaultLens, lenses: lensIds.map(lensId => ({ id: lensId })) }, provenance);
-      const bankUrl = `${base}/${descriptor.prepared!.url}`;
-      const bankPin = provenance.products.flatMap(product => product.outputs).find(output => output.url === bankUrl);
-      if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`);
-      const hostedBy = await hostedDatasets(root, base, id, prepared.controls.map(control => control.id), input);
-      results.push({ id, name: sourceText(sourcePresentation.name), route: hostedBy?.route ?? `/sun/?focus=${id}`, base,
-        controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], ...(hostedBy ? { hostedBy } : {}) });
-      continue;
-    }
-    if (sourcePresentation.provenance === undefined) continue;
-    const preparedPresentation = sourceObject(json(await input(`${base}/prepared/presentation.json`)));
+    // The three source-only catalogue contexts use source/presentation.json too, but are not prepared lens packages.
+    // prepareContextProvenance owns their in-memory catalogue records below; there are no R2 metadata files to read.
+    if (sourcePresentation.schema !== 'cssearth-volume-presentation-source@1') continue;
+    if (sourcePresentation.objectId !== id) throw new TypeError(`Mismatched volume presentation object: ${id}.`);
+    const descriptor = parseObjectDescriptor(json(await input(`${base}/object.json`)));
+    if (descriptor.id !== id || !descriptor.prepared || !['volume-lens-bank', 'image-layer-bank'].includes(descriptor.type))
+      throw new TypeError(`Invalid prepared volume descriptor: ${id}.`);
     const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id);
-    results.push({ id, name: sourceText(preparedPresentation.name), route: '/sun/', base, controls: [], defaultLens: '', provenance, outputs: [] });
+    const defaultLens = sourceId(sourcePresentation.defaultLens);
+    const lensIds = sourceArray(sourcePresentation.lenses, raw => sourceId(sourceObject(raw).id));
+    const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)),
+      { id, defaultLens, lenses: lensIds.map(lensId => ({ id: lensId })) }, provenance);
+    const bankUrl = `${base}/${descriptor.prepared!.url}`;
+    const bankPin = provenance.products.flatMap(product => product.outputs).find(output => output.url === bankUrl);
+    if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`);
+    const hostedBy = await hostedDatasets(root, base, id, prepared.controls.map(control => control.id), input);
+    results.push({ id, name: sourceText(sourcePresentation.name), route: hostedBy?.route ?? `/sun/?focus=${id}`, base,
+      controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], ...(hostedBy ? { hostedBy } : {}) });
   }
   return results;
 }

tools/runtime-assets.mts

@@ -66,14 +66,16 @@ async function locatedAssets<T extends RuntimeAsset>(root: string, id: string, a
   return located;
 }
 
-export async function runtimeAssets(root: string, objectIds: readonly string[]): Promise<RuntimeAssetLocation[]> {
+export async function runtimeAssets(root: string, objectIds: readonly string[],
+  { filenames }: { filenames?: readonly string[] } = {}): Promise<RuntimeAssetLocation[]> {
   const assets: RuntimeAssetLocation[] = [];
+  const selectedFilenames = filenames ? new Set(filenames) : null;
   for (const id of objectIds) {
     if (!/^[a-z][a-z0-9-]*$/u.test(id)) throw new TypeError(`Unsafe runtime object identity: ${id}`);
     const base = resolve(root, `src/objects/${id}`);
     const bytes = await readFile(resolve(base, "runtime-assets.json"));
     const manifest = requireRuntimeAssetManifest(id, JSON.parse(bytes.toString("utf8")));
-    for (const asset of manifest.assets) {
+    for (const asset of manifest.assets.filter(asset => !selectedFilenames || selectedFilenames.has(asset.filename))) {
       const assetRoot = manifest.resourceRoot === "prepared" && asset.location !== "public" ? resolve(base, "prepared") : resolve(root, `public/scenes/${id}`);
       assets.push(...await locatedAssets(root, id, assetRoot, [asset]));
     }
@@ -82,14 +84,17 @@ export async function runtimeAssets(root: string, objectIds: readonly string[]):
 }
 
 /** Counterpart of `runtimeAssets` for `prepared-assets.json`: always resourceRoot `prepared`, never public. */
-export async function preparedAssets(root: string, objectIds: readonly string[]): Promise<RuntimeAssetLocation[]> {
+export async function preparedAssets(root: string, objectIds: readonly string[],
+  { filenames }: { filenames?: readonly string[] } = {}): Promise<RuntimeAssetLocation[]> {
   const assets: RuntimeAssetLocation[] = [];
+  const selectedFilenames = filenames ? new Set(filenames) : null;
   for (const id of objectIds) {
     if (!/^[a-z][a-z0-9-]*$/u.test(id)) throw new TypeError(`Unsafe prepared object identity: ${id}`);
     const base = resolve(root, `src/objects/${id}`);
     const bytes = await readFile(resolve(base, "prepared-assets.json"));
     const manifest = requirePreparedAssetManifest(id, JSON.parse(bytes.toString("utf8")));
-    assets.push(...await locatedAssets(root, id, resolve(base, "prepared"), manifest.assets));
+    assets.push(...await locatedAssets(root, id, resolve(base, "prepared"),
+      manifest.assets.filter(asset => !selectedFilenames || selectedFilenames.has(asset.filename))));
   }
   return assets;
 }

tools/setup-volume-metadata.mts

@@ -0,0 +1,53 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { sourceObject, sourceText } from '../src/platform/source-catalog.mts';
+import { preparedAssets, runtimeAssets } from './runtime-assets.mts';
+import { installRuntimeAssets } from './setup.mts';
+import { hasErrorCode } from './source-values.mts';
+
+export const VOLUME_METADATA_FILENAMES = ['presentation.json', 'provenance.json'] as const;
+
+/** Select only the small R2-backed package metadata needed to compile deploy catalogues. Volume textures and
+ * slices stay remote; restoring the full runtime-assets closure here can be tens of gigabytes. */
+export async function preparedVolumeMetadataAssets(root = resolve(import.meta.dirname, '..')) {
+  const ids: string[] = [];
+  const folders = await readdir(resolve(root, 'src/objects'), { withFileTypes: true });
+  for (const folder of folders.filter(folder => folder.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) {
+    const path = resolve(root, 'src/objects', folder.name, 'source/presentation.json');
+    const bytes = await readFile(path).catch((error: unknown) => {
+      if (hasErrorCode(error, 'ENOENT')) return undefined;
+      throw error;
+    });
+    if (!bytes) continue;
+    const presentation = sourceObject(JSON.parse(bytes.toString('utf8')));
+    if (presentation.schema !== 'cssearth-volume-presentation-source@1') continue;
+    if (sourceText(presentation.objectId) !== folder.name) throw new TypeError(`Mismatched volume presentation object: ${folder.name}.`);
+    ids.push(folder.name);
+  }
+
+  const preparedIds = ids.filter(id => existsSync(resolve(root, 'src/objects', id, 'prepared-assets.json')));
+  const assets = [
+    ...await runtimeAssets(root, ids, { filenames: VOLUME_METADATA_FILENAMES }),
+    ...await preparedAssets(root, preparedIds, { filenames: VOLUME_METADATA_FILENAMES }),
+  ];
+  for (const id of ids) for (const filename of VOLUME_METADATA_FILENAMES) {
+    const matches = assets.filter(asset => asset.id === id && asset.filename === filename);
+    if (matches.length !== 1 || matches[0]!.location === 'public' ||
+        matches[0]!.file !== resolve(root, 'src/objects', id, 'prepared', filename)) {
+      throw new TypeError(`Volume ${id} must inventory exactly one prepared/${filename} runtime asset.`);
+    }
+  }
+  return { ids, assets };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
+  const root = resolve(import.meta.dirname, '..');
+  const { ids, assets } = await preparedVolumeMetadataAssets(root);
+  console.log(`Setting up catalogue metadata for ${ids.join(', ')}: ${assets.length} file(s); volume data stays on R2.`);
+  const result = await installRuntimeAssets(assets, { onProgress: ({ completed, total }) => {
+    if (completed % 10 === 0 || completed === total) console.log(`Volume metadata: ${completed}/${total}`);
+  } });
+  console.log(`Volume metadata setup complete: ${result.installed} downloaded, ${result.reused} reused.`);
+}