From db0e98989a4c7fcc889e8745155b6eac57adb071 Mon Sep 17 00:00:00 2001 From: Juan Cruz Fortunatti Date: Sun, 20 Sep 2026 10:59:30 +0200 Subject: [PATCH 1/8] fix(ci): restore deployment catalogues without rebaking assets --- .github/workflows/deploy.yml | 6 +-- .github/workflows/nightly.yml | 20 +++++++-- package.json | 2 +- tools/check-ci.test.mts | 8 +++- tools/check-deploy-assets.mts | 2 +- tools/check-deploy-assets.test.mts | 7 +++ tools/prepare-context-provenance.mts | 23 +++++++++- tools/prepare-deploy-catalogues.test.mts | 56 ++++++++++++++++++++++++ tools/prepare-facilities.mts | 12 +++-- tools/prepare-volume-provenance.mts | 21 ++++++++- tools/runtime-assets.mts | 9 +++- tools/runtime-assets.test.mts | 2 + 12 files changed, 151 insertions(+), 17 deletions(-) create mode 100644 tools/prepare-deploy-catalogues.test.mts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7e1259b092..fbc9243d7e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,9 +73,9 @@ jobs: CSSEARTH_ALLOW_MISSING_ASSETS: "1" run: pnpm setup:assets - name: Build the site - # The deploy build consumes the committed prepared metadata and the restored R2 closure. It does not run - # prepare:facilities: that authoring command can encode new preview bytes and rewrite their inventories, - # which must be published explicitly before a deploy is allowed to reference them. + # The deploy build consumes committed object metadata and the restored R2 closure. Its prepared-only + # catalogue pass creates the two ignored shared graphs without encoding previews or rewriting inventories; + # new object assets must still be prepared and published explicitly before deployment. # ASSET_ORIGIN points every texture, scene JSON and startup preload at the published R2 # bucket instead of bundling public/scenes (1.44 GB) into the deploy; astro.config.mts # removes dist/scenes once the build finishes and assemble:planets tolerates its absence. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4410969737..991fc579ff 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,13 +6,22 @@ on: workflow_dispatch: pull_request: # Only the deploy-build job below runs from this trigger (each other job's `if:` stays schedule/dispatch-only): - # a full `pnpm build` is too slow to run on every PR, but a PR touching the code this specific path resolves + # a full `pnpm build:deploy` is too slow to run on every PR, but a PR touching the code this specific path resolves # through gets it anyway, instead of waiting for the nightly schedule to notice a break. paths: - 'site/**' - 'astro.config.mts' - 'src/renderers/css/rendering/**' - '.github/workflows/deploy.yml' + - '.github/workflows/nightly.yml' + - 'package.json' + - 'tools/prepare-facilities.mts' + - 'tools/prepare-volume-provenance.mts' + - 'tools/prepare-context-provenance.mts' + - 'tools/runtime-assets.mts' + - 'tools/setup*.mts' + - 'tools/prepare-deploy-catalogues.test.mts' + - 'tools/check-deploy-assets*.mts' permissions: contents: read @@ -86,7 +95,7 @@ jobs: asset-origin-build: # Exercises the one path that has broken two real deploys and that no other CI job builds at all: a full - # `pnpm build` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static + # `pnpm build:deploy` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static # scan catches emitted same-origin addresses; the production browser test also exercises the client loader, # whose descriptor path can otherwise discard an origin that is present in the emitted page. name: Build with ASSET_ORIGIN and exercise the client loader @@ -123,7 +132,12 @@ jobs: NODE_OPTIONS: --max-old-space-size=6144 CSSEARTH_ALLOW_MISSING_ASSETS: "1" ASSET_ORIGIN: https://asset-origin-ci-check.example - run: pnpm build + run: | + test ! -e site/prepared-sources.json + test ! -e site/prepared-facilities.json + pnpm build:deploy + git diff --exit-code -- src/objects + node --test tools/prepare-deploy-catalogues.test.mts tools/check-deploy-assets.test.mts - name: Check no page references a same-origin /scenes/ address run: node tools/check-asset-origin-scenes.mts dist - name: Check production navigation loads content-addressed assets diff --git a/package.json b/package.json index a7a6866c94..324e7c7b45 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "predev": "pnpm build:tools && pnpm prepare:object-json && pnpm prepare:environment-images && pnpm prepare:minimap && pnpm prepare:galaxy-field", "prepare:catalog": "node tools/prepare-catalog.mts", "prepare:checkout": "node tools/restore-source-inputs.mts && pnpm prepare:planets", - "prepare:deploy": "pnpm build:tools && pnpm setup:assets && pnpm prepare:object-json && pnpm prepare:environment-images && node tools/nebula/prepare.mts --if-missing && pnpm prepare:minimap && pnpm prepare:galaxy-field:data", + "prepare:deploy": "pnpm build:tools && node tools/setup.mts --all-inventoried && pnpm setup:prepared && pnpm prepare:object-json && pnpm prepare:environment-images && node tools/nebula/prepare.mts --if-missing && pnpm prepare:minimap && node tools/prepare-facilities.mts --prepared-only", "prepare:earth-global": "node tools/objects/geographic-pages/operations/prepare-global-wmts.mts --object=earth && node tools/objects/geographic-pages/operations/integrate-global-wmts.mts --object=earth --latest", "prepare:environment-images": "node tools/objects/dist/restore-environment-images.js", "prepare:factsheets": "node tools/prepare-factsheets.mts", diff --git a/tools/check-ci.test.mts b/tools/check-ci.test.mts index 9795d87dd8..a7a1f7268c 100644 --- a/tools/check-ci.test.mts +++ b/tools/check-ci.test.mts @@ -34,8 +34,14 @@ test('the deploy consumes installed assets and rejects generated metadata or uni assert.match(workflow,/git diff --quiet -- src\/objects site\/prepared-facilities\.json site\/prepared-sources\.json/); assert.doesNotMatch(workflow,/ASSET_ORIGIN=https:\/\/earth-assets\.lowpoly\.cc pnpm build(?:\s|$)/); assert.match(packageFile.scripts['prepare:deploy']??'',/node tools\/nebula\/prepare\.mts --if-missing/); - assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:galaxy-field:data/); + assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:galaxy-field/); + assert.match(packageFile.scripts['prepare:deploy']??'',/node tools\/setup\.mts --all-inventoried/); + assert.match(packageFile.scripts['prepare:deploy']??'',/node tools\/prepare-facilities\.mts --prepared-only/); assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:(?:facilities|provenance|nebulae)(?:\s|$)/); + const buildSmoke=await readFile(new URL('../.github/workflows/nightly.yml',import.meta.url),'utf8'); + assert.match(buildSmoke,/test ! -e site\/prepared-sources\.json/); + assert.match(buildSmoke,/pnpm build:deploy/); + assert.doesNotMatch(buildSmoke,/run: pnpm build\s*\n/); }); test('--quick skips only the network and documentation steps, and refuses a job without them',async()=>{ const lint=readCiSteps(await readFile(new URL('../.github/workflows/universe.yml',import.meta.url),'utf8'),'lint'); diff --git a/tools/check-deploy-assets.mts b/tools/check-deploy-assets.mts index 97b94f02e8..7162b82797 100644 --- a/tools/check-deploy-assets.mts +++ b/tools/check-deploy-assets.mts @@ -8,7 +8,7 @@ import { inventoriedAssets, inventoriedObjectIds, RUNTIME_ASSET_ORIGIN } from '. const execFileAsync = promisify(execFile); const textExtensions = new Set(['.css', '.html', '.js', '.json', '.map', '.svg', '.txt', '.xml']); const escapedOrigin = RUNTIME_ASSET_ORIGIN.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&'); -const runtimeAssetPattern = new RegExp(`${escapedOrigin}/runtime-assets/[a-f0-9]{64}/[a-zA-Z0-9._/-]+`, 'gu'); +const runtimeAssetPattern = new RegExp(`${escapedOrigin}/runtime-assets/[a-f0-9]{64}/[a-zA-Z0-9._/@-]+`, 'gu'); export function runtimeAssetUrls(text: string): string[] { return [...new Set(text.match(runtimeAssetPattern) ?? [])].sort(); diff --git a/tools/check-deploy-assets.test.mts b/tools/check-deploy-assets.test.mts index 40b949ad3a..9c91eec864 100644 --- a/tools/check-deploy-assets.test.mts +++ b/tools/check-deploy-assets.test.mts @@ -15,6 +15,13 @@ test('deploy asset closure ignores unrelated URLs and truncated identities', () assert.deepEqual(runtimeAssetUrls(`https://example.test/runtime-assets/${hash}/x.webp ${RUNTIME_ASSET_ORIGIN_FIXTURE()}`), []); }); +test('deploy asset closure preserves the complete @2x filenames emitted by surface textures', () => { + const retina = `https://earth-assets.lowpoly.cc/runtime-assets/${hash}/mercury-poles@2x.webp`; + const urls = runtimeAssetUrls(``); + assert.deepEqual(urls, [retina]); + assert.deepEqual(unknownRuntimeAssetUrls(urls, new Set([retina])), []); +}); + function RUNTIME_ASSET_ORIGIN_FIXTURE(): string { return `https://earth-assets.lowpoly.cc/runtime-assets/${'c'.repeat(63)}/x.webp`; } diff --git a/tools/prepare-context-provenance.mts b/tools/prepare-context-provenance.mts index 04a55b171f..16980b2c7a 100644 --- a/tools/prepare-context-provenance.mts +++ b/tools/prepare-context-provenance.mts @@ -1,11 +1,12 @@ import { sha256 } from '../src/platform/sha256.mts'; import { readFile, readdir } from 'node:fs/promises'; import { resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; import { sourceArray, sourceObject, sourcePath, sourceText, sourceDigest } from '../src/platform/source-catalog.mts'; import { validateObjectProvenance } from '../src/platform/object-provenance.mts'; import { manifestSources } from './context-source-records.mts'; export const contextProvenanceCompilerClosure = ['tools/prepare-context-provenance.mts', 'tools/context-source-records.mts']; -export async function prepareContextProvenance({ root = process.cwd(), input = (path: string) => readFile(resolve(root, path)) } = {}) { +export async function prepareContextProvenance({ root = process.cwd(), input = (path: string) => readFile(resolve(root, path)), preparedOnly = false } = {}) { const results = []; const generator = await input(contextProvenanceCompilerClosure[0]!); for (const path of contextProvenanceCompilerClosure.slice(1)) await input(path); @@ -20,6 +21,26 @@ export async function prepareContextProvenance({ root = process.cwd(), input = ( await input(presentationPath); const manifestBytes = await input(`${base}/source/manifest.json`), manifest = sourceObject(JSON.parse(manifestBytes.toString())); if (manifest.schema !== 'cssearth-volume-source-manifest@1' || manifest.pathBase !== 'repository') throw new TypeError(`Invalid context manifest: ${id}`); + if (preparedOnly) { + const provenance = validateObjectProvenance(JSON.parse((await input(`${base}/prepared/provenance.json`)).toString()), id); + const prepared = sourceObject(JSON.parse((await input(`${base}/prepared/presentation.json`)).toString())); + if (prepared.name !== sourceText(presentation.name)) throw new Error(`Stale prepared context presentation: ${id}.`); + const declared = sourceArray(presentation.products, sourceObject).map(product => ({ ...product, + selector: product.selector === '/' ? '' : product.selector })); + const retained = provenance.products.map(product => { + const recipe = provenance.recipes.find(recipe => recipe.id === product.recipe); + return { id: product.id, label: product.label, process: product.process, + recipe: recipe?.path.slice(base.length + 1), selector: product.selector, + inputs: [...product.inputs], outputs: product.outputs.map(output => output.url.slice(base.length + 1)), + interpretation: product.interpretation, limitations: [...product.limitations] }; + }); + if (!isDeepStrictEqual(declared, retained)) throw new Error(`Stale prepared context products: ${id}; prepare and publish them before deploying.`); + for (const recipe of provenance.recipes) if (sha256(await input(recipe.path)) !== recipe.sha256) + throw new Error(`Stale prepared context recipe: ${id}; prepare and publish it before deploying.`); + await input(`${base}/runtime-assets.json`); + results.push({ id, name: sourceText(prepared.name), route: '/sun/', base, controls: [], provenance, outputs: [] }); + continue; + } const sources = await manifestSources(manifest, root, input); const receipt = sourceObject(JSON.parse((await input(`${base}/prepared/manifest.json`)).toString())); const pins = sourceArray(receipt.outputs, sourceObject); diff --git a/tools/prepare-deploy-catalogues.test.mts b/tools/prepare-deploy-catalogues.test.mts new file mode 100644 index 0000000000..d20523775a --- /dev/null +++ b/tools/prepare-deploy-catalogues.test.mts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { prepareVolumeProvenance, volumeProvenanceCompilerClosure } from './prepare-volume-provenance.mts'; +import { prepareContextProvenance, contextProvenanceCompilerClosure } from './prepare-context-provenance.mts'; +import { prepareFacilities } from './prepare-facilities.mts'; + +test('deploy catalogues reuse published volume and context receipts without native images or a baked field', async t => { + const root = await mkdtemp(join(tmpdir(), 'deploy-catalogues-')); + t.after(() => rm(root, { recursive: true, force: true })); + const files = [...volumeProvenanceCompilerClosure, ...contextProvenanceCompilerClosure, + ...['m45', 'betelgeuse-shell'].flatMap(id => ['source/presentation.json', 'source/delivery.json', + 'prepared/provenance.json', 'prepared/presentation.json', 'runtime-assets.json', 'object.json'].map(path => `src/objects/${id}/${path}`)), + 'src/objects/betelgeuse/source/content/object.json', + 'src/objects/nearby-universe/source/preparation/field.json', + ...['source/presentation.json', 'source/manifest.json', 'prepared/provenance.json', 'prepared/presentation.json', 'runtime-assets.json'] + .map(path => `src/objects/nearby-universe/${path}`)]; + for (const path of new Set(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await copyFile(path, join(root, path)); + } + const volumes = await prepareVolumeProvenance({ root, preparedOnly: true }); + assert.deepEqual(volumes.map(volume => volume.id), ['betelgeuse-shell', 'm45']); + assert.ok(volumes.every(volume => volume.outputs.length === 0)); + const m45 = volumes.find(volume => volume.id === 'm45')!; + assert.equal(m45.route, '/sun/?focus=m45'); + assert.equal(m45.controls.length, 5); + const attached = volumes.find(volume => volume.id === 'betelgeuse-shell')!; + assert.equal(attached.hostedBy?.objectId, 'betelgeuse'); + assert.equal(attached.route, '/betelgeuse/'); + const contexts = await prepareContextProvenance({ root, preparedOnly: true }); + assert.deepEqual(contexts.map(context => context.id), ['nearby-universe']); + assert.equal(contexts[0]!.outputs.length, 0); + assert.ok(contexts[0]!.provenance.products.length > 0); + const contextPath = join(root, 'src/objects/nearby-universe/source/presentation.json'); + const contextSource = await readFile(contextPath, 'utf8'); + await writeFile(contextPath, contextSource.replace('"label": "Nearby galaxy distribution"', '"label": "Changed product"')); + assert.notEqual(await readFile(contextPath, 'utf8'), contextSource); + await assert.rejects(prepareContextProvenance({ root, preparedOnly: true }), /Stale prepared context products/); + const presentationPath = join(root, 'src/objects/m45/source/presentation.json'); + const original = await readFile(presentationPath, 'utf8'); + await writeFile(presentationPath, original.replace('"name": "Pleiades"', '"name": "Changed presentation"')); + assert.notEqual(await readFile(presentationPath, 'utf8'), original); + await assert.rejects(prepareVolumeProvenance({ root, preparedOnly: true }), /Stale prepared volume presentation/); +}); + +test('deploy source compilation emits only the two ignored shared catalogues', async () => { + const result = await prepareFacilities({ preparedOnly: true, publish: false }); + assert.deepEqual(result.outputs.map(output => output.path).sort(), + ['site/prepared-facilities.json', 'site/prepared-sources.json'].map(path => resolve(path))); + assert.ok(result.preparedSources.usage.datasets.some(dataset => dataset.objectId === 'm45')); + assert.ok(result.preparedSources.usage.datasets.some(dataset => dataset.objectId === 'betelgeuse')); + assert.equal(result.prepared.sourceCatalogSha256, result.preparedSources.catalogSha256); +}); diff --git a/tools/prepare-facilities.mts b/tools/prepare-facilities.mts index b7d29d05db..b4a53921fd 100644 --- a/tools/prepare-facilities.mts +++ b/tools/prepare-facilities.mts @@ -45,11 +45,13 @@ export const explorationCompilerClosure = [ ] as const; interface Options { root?: string; publish?: boolean; provenance?: ReadonlyMap; sourceTransport?: FactsheetSourceTransport; + /** Rebuild only ignored shared graphs from existing object metadata; never author preview bytes or inventories. */ + preparedOnly?: boolean; /** Opt-in (default null/off) content-addressed mirror for volume previews; a production caller names * RUNTIME_ASSET_ORIGIN explicitly. Left off by default so a test never makes a surprise real request. */ mirrorOrigin?: string | null; } /** Compile evidenced links and reuse approved artwork, restoring only missing cited evidence. */ -export async function prepareFacilities({ root = resolve(import.meta.dirname, '..'), publish = true, provenance = new Map(), sourceTransport, mirrorOrigin = null }: Options = {}) { +export async function prepareFacilities({ root = resolve(import.meta.dirname, '..'), publish = true, provenance = new Map(), sourceTransport, mirrorOrigin = null, preparedOnly = false }: Options = {}) { const closure: Record = {}; const input = async (path: string) => { const bytes = await readFile(resolve(root, path)); closure[path] = sha256(bytes); return bytes; @@ -116,6 +118,8 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. if (contentPin.length !== 1 || contentPin[0]!.expectedBytes !== contentBytes.length || contentPin[0]!.expectedSha256 !== sha256(contentBytes)) throw new Error(`Changed content source for ${object.id}.`); const content = explorationRecord(JSON.parse(contentBytes.toString('utf8'))); const objectDirectory = resolve(root, base); + // Missing citation documents may be restored byte-for-byte from their pinned download plans. + // This verifies source evidence; it never prepares or replaces metadata, imagery or inventories. const panel = await verifyFactsheetSources(content.panel, { objectDirectory, manifest, sources, read: path => input(`${base}/${path}`), restoreMissing: path => restoreFactsheetEvidence({ objectDirectory, path, manifest, transport: sourceTransport }), @@ -139,7 +143,7 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. inventory.push(...sourceInventory(manifest, `${base}/source/manifest.json`, sources, new Set(document.sources.map(source => source.path)))); objects.push({ id: object.id, name: object.name, route: object.route, base, controls: lenses, provenance: document }); } - const volumes = [...await prepareVolumeProvenance({ root, input, mirrorOrigin }), ...await prepareContextProvenance({ root, input })]; + const volumes = [...await prepareVolumeProvenance({ root, input, mirrorOrigin, preparedOnly }), ...await prepareContextProvenance({ root, input, preparedOnly })]; for (const volume of volumes) { const document = validateObjectProvenance(volume.provenance, volume.id); const manifestPath = `${sourcePath(volume.base)}/${sourcePath(document.manifest.path)}`; @@ -173,7 +177,9 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. } if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { // The real CLI entry point: opts into the mirror explicitly (library code above defaults it off). - const { prepared, factsheets } = await prepareFacilities({ mirrorOrigin: RUNTIME_ASSET_ORIGIN }); + const args = process.argv.slice(2); + if (args.length > 1 || args.some(arg => arg !== '--prepared-only')) throw new TypeError('Usage: prepare-facilities.mts [--prepared-only]'); + const { prepared, factsheets } = await prepareFacilities({ mirrorOrigin: RUNTIME_ASSET_ORIGIN, preparedOnly: args.includes('--prepared-only') }); console.log(`Prepared ${prepared.catalog.missions.length} missions, ${prepared.catalog.facilities.length} facilities and ${prepared.graph.datasets.length} dataset destinations.`); console.log(`Factsheets: ${factsheets.facts} facts, each with its own citation.`); } diff --git a/tools/prepare-volume-provenance.mts b/tools/prepare-volume-provenance.mts index bccba9c0b2..54e051eefb 100644 --- a/tools/prepare-volume-provenance.mts +++ b/tools/prepare-volume-provenance.mts @@ -16,8 +16,9 @@ import { writePreparedSet } from './write-prepared-set.mts'; import { manifestSources } from './context-source-records.mts'; import { composeSkyBandPng, skyBandCompositeFile, verifySkyBandRecipe } from './objects/observation/sky-band-composite.mts'; import { RUNTIME_ASSET_ORIGIN, fetchWithRetry, sourceCacheUrl } from './source-mirror.mts'; +import { parsePreparedVolumePresentation } from '../site/volume-presentation.mts'; -export const volumeProvenanceCompilerClosure = ['tools/prepare-volume-provenance.mts', 'site/dataset-content.mts', 'tools/context-source-records.mts', +export const volumeProvenanceCompilerClosure = ['tools/prepare-volume-provenance.mts', 'site/dataset-content.mts', 'site/volume-presentation.mts', 'site/prepared-panel-content.mts', 'tools/context-source-records.mts', 'tools/objects/observation/sky-band-composite.mts', 'tools/objects/observation/wise-atlas-mosaic.mts', 'tools/objects/color-transfer.mts', 'tools/fits.mts'] as const; const integer = (value: unknown): number => { @@ -137,6 +138,8 @@ async function hostedDatasets(root: string, base: string, objectId: string, lens } interface Options { root?: string; + /** Build shared catalogues from committed receipts without encoding or replacing object assets. */ + preparedOnly?: boolean; /** Repository-relative, tracked compiler inputs only. Downloads never enter source closure. */ input?: (path: string) => Promise; /** Opt-in (default null/off): the real content-addressed mirror origin, named explicitly by a production caller. @@ -197,7 +200,7 @@ export async function preparePreview(root: string, pin: Preview, input: (path: s } /** Recover portable lineage from source-owned byte pins without replaying the cloud compiler. */ -export async function prepareVolumeProvenance({ root = process.cwd(), input = path => readFile(resolve(root, path)), mirrorOrigin = null }: Options = {}): Promise { +export async function prepareVolumeProvenance({ root = process.cwd(), input = path => readFile(resolve(root, path)), mirrorOrigin = null, preparedOnly = false }: Options = {}): Promise { const results: PreparedVolumeProvenance[] = []; const generatorBytes = await input(volumeProvenanceCompilerClosure[0]); for (const path of volumeProvenanceCompilerClosure.slice(1)) await input(path); @@ -209,6 +212,20 @@ export async function prepareVolumeProvenance({ root = process.cwd(), input = pa const ownedPresentationBytes = await input(presentationPath); const record = presentation(json(ownedPresentationBytes)); if (record.objectId !== folder.name) throw new TypeError('Mismatched volume presentation object.'); + if (preparedOnly) { + const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), record.objectId); + if (!provenance.recipes.some(recipe => recipe.path === presentationPath && recipe.sha256 === sha256(ownedPresentationBytes))) + throw new Error(`Stale prepared volume presentation: ${record.objectId}; prepare and publish it before deploying.`); + const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)), + { id: record.objectId, defaultLens: record.defaultLens, lenses: record.lenses }, provenance); + await input(`${base}/object.json`); + await input(`${base}/runtime-assets.json`); + const hostedBy = await hostedDatasets(root, base, record.objectId, prepared.controls.map(lens => lens.id), input); + results.push({ id: record.objectId, name: record.name, route: hostedBy?.route ?? `/sun/?focus=${record.objectId}`, + base, controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], + ...(hostedBy === undefined ? {} : { hostedBy }) }); + continue; + } const manifestPath = `${base}/source/manifest.json`, manifestBytes = await input(manifestPath); const manifest = sourceObject(json(manifestBytes), ['schema', 'pathBase', 'inputs', 'documents', 'generatedIntermediates']); if (manifest.schema !== 'cssearth-volume-source-manifest@1' || manifest.pathBase !== 'repository') throw new TypeError('Invalid volume source manifest.'); diff --git a/tools/runtime-assets.mts b/tools/runtime-assets.mts index 108375980d..7ebbc13ed4 100644 --- a/tools/runtime-assets.mts +++ b/tools/runtime-assets.mts @@ -17,8 +17,13 @@ function parseObjectArgs(args: readonly string[]): string[] { } export function setupObjectIds(args: readonly string[], root = resolve(import.meta.dirname, "..")) { - const ids = parseObjectArgs(args); - const selected = ids.length ? ids : SCENE_OBJECTS.map(({ id }) => id); + const allInventoried = args.includes('--all-inventoried'); + const ids = parseObjectArgs(args.filter(arg => arg !== '--all-inventoried')); + if (allInventoried && ids.length) throw new Error('Choose --all-inventoried or --object=, not both.'); + const selected = allInventoried ? readdirSync(resolve(root, 'src/objects'), { withFileTypes: true }) + .filter(entry => entry.isDirectory() && /^[a-z][a-z0-9-]*$/u.test(entry.name) && existsSync(resolve(root, 'src/objects', entry.name, 'runtime-assets.json'))) + .map(entry => entry.name).sort((left, right) => left.localeCompare(right)) + : ids.length ? ids : SCENE_OBJECTS.map(({ id }) => id); if (new Set(selected).size !== selected.length || selected.some(id => !/^[a-z][a-z0-9-]*$/u.test(id) || (!SCENE_OBJECTS.some(object => object.id === id) && !existsSync(resolve(root, `src/objects/${id}/runtime-assets.json`))))) { diff --git a/tools/runtime-assets.test.mts b/tools/runtime-assets.test.mts index 7ea9f13bae..5995c8f0bc 100644 --- a/tools/runtime-assets.test.mts +++ b/tools/runtime-assets.test.mts @@ -33,6 +33,8 @@ test('explicit context setup uses prepared paths and hash URLs without changing await writeFile(resolve(base, 'prepared/levels/catalogue.json'), bytes); assert.deepEqual(setupObjectIds(['--object=context-fixture'], root), ['context-fixture']); assert(!setupObjectIds([], root).includes('context-fixture')); + assert.deepEqual(setupObjectIds(['--all-inventoried'], root), ['context-fixture']); + assert.throws(() => setupObjectIds(['--all-inventoried', '--object=context-fixture'], root), /not both/); assert.throws(() => setupObjectIds(['--object=../context-fixture'], root)); assert.throws(() => setupObjectIds(['--object=missing-fixture'], root)); const [asset] = await runtimeAssets(root, ['context-fixture']); From 42afa3a0204477e3ab7d7714f3f587d33bf13327 Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 07:22:36 -0300 Subject: [PATCH 2/8] fix(deploy): rebuild generated source catalogues --- .github/workflows/deploy.yml | 10 +++++----- package.json | 3 ++- tools/check-ci.test.mts | 6 ++++-- tools/check-deploy-assets.mts | 2 +- tools/prepare-facilities.mts | 14 +++++++++----- tools/source-catalogue.test.mts | 1 + 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7e1259b092..a751e8155b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -73,9 +73,9 @@ jobs: CSSEARTH_ALLOW_MISSING_ASSETS: "1" run: pnpm setup:assets - name: Build the site - # The deploy build consumes the committed prepared metadata and the restored R2 closure. It does not run - # prepare:facilities: that authoring command can encode new preview bytes and rewrite their inventories, - # which must be published explicitly before a deploy is allowed to reference them. + # The deploy build regenerates its ignored source/facility catalogues from committed inputs and the restored + # R2 closure. Its catalog-only preparation does not publish preview bytes or rewrite asset inventories; + # those authoring outputs must be published explicitly before a deploy is allowed to reference them. # ASSET_ORIGIN points every texture, scene JSON and startup preload at the published R2 # bucket instead of bundling public/scenes (1.44 GB) into the deploy; astro.config.mts # removes dist/scenes once the build finishes and assemble:planets tolerates its absence. @@ -113,9 +113,9 @@ jobs: echo "::error::Only $page_count HTML pages were built (expected roughly 974, refusing below $min_pages). Refusing to deploy a truncated site." exit 1 fi - if ! git diff --quiet -- src/objects site/prepared-facilities.json site/prepared-sources.json; then + if ! git diff --quiet -- src/objects; then echo "::error::Deploy preparation changed committed object metadata. Prepare and publish it explicitly before deploying." - git diff --name-only -- src/objects site/prepared-facilities.json site/prepared-sources.json + git diff --name-only -- src/objects exit 1 fi - name: Publish the prebuilt site diff --git a/package.json b/package.json index a7a6866c94..7ca8bd6b9a 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "predev": "pnpm build:tools && pnpm prepare:object-json && pnpm prepare:environment-images && pnpm prepare:minimap && pnpm prepare:galaxy-field", "prepare:catalog": "node tools/prepare-catalog.mts", "prepare:checkout": "node tools/restore-source-inputs.mts && pnpm prepare:planets", - "prepare:deploy": "pnpm build:tools && pnpm setup:assets && pnpm prepare:object-json && pnpm prepare:environment-images && node tools/nebula/prepare.mts --if-missing && pnpm prepare:minimap && pnpm prepare:galaxy-field:data", + "prepare:deploy": "pnpm build:tools && pnpm setup:assets && pnpm prepare:object-json && pnpm prepare:environment-images && node tools/nebula/prepare.mts --if-missing && pnpm prepare:minimap && pnpm prepare:galaxy-field:data && pnpm prepare:deploy-catalogues", + "prepare:deploy-catalogues": "node tools/prepare-facilities.mts --catalog-only", "prepare:earth-global": "node tools/objects/geographic-pages/operations/prepare-global-wmts.mts --object=earth && node tools/objects/geographic-pages/operations/integrate-global-wmts.mts --object=earth --latest", "prepare:environment-images": "node tools/objects/dist/restore-environment-images.js", "prepare:factsheets": "node tools/prepare-factsheets.mts", diff --git a/tools/check-ci.test.mts b/tools/check-ci.test.mts index 9795d87dd8..d9d1a189ce 100644 --- a/tools/check-ci.test.mts +++ b/tools/check-ci.test.mts @@ -26,15 +26,17 @@ test('local CI reads the actual workflow jobs in order, including strict TypeScr const ownership=universe.find(step=>step.name.includes('runtime ownership')); assert.equal(ownership?.env.RUNTIME_OWNERSHIP_ARGS,'--all'); }); -test('the deploy consumes installed assets and rejects generated metadata or uninventoried output',async()=>{ +test('the deploy consumes installed assets, rebuilds only catalogues and rejects uninventoried output',async()=>{ const workflow=await readFile(new URL('../.github/workflows/deploy.yml',import.meta.url),'utf8'); const packageFile=JSON.parse(await readFile(new URL('../package.json',import.meta.url),'utf8')) as {scripts:Record}; assert.match(workflow,/pnpm build:deploy/); assert.match(workflow,/pnpm check:deploy-assets/); - assert.match(workflow,/git diff --quiet -- src\/objects site\/prepared-facilities\.json site\/prepared-sources\.json/); + assert.match(workflow,/git diff --quiet -- src\/objects/); assert.doesNotMatch(workflow,/ASSET_ORIGIN=https:\/\/earth-assets\.lowpoly\.cc pnpm build(?:\s|$)/); assert.match(packageFile.scripts['prepare:deploy']??'',/node tools\/nebula\/prepare\.mts --if-missing/); assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:galaxy-field:data/); + assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:deploy-catalogues/); + assert.equal(packageFile.scripts['prepare:deploy-catalogues'],'node tools/prepare-facilities.mts --catalog-only'); assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:(?:facilities|provenance|nebulae)(?:\s|$)/); }); test('--quick skips only the network and documentation steps, and refuses a job without them',async()=>{ diff --git a/tools/check-deploy-assets.mts b/tools/check-deploy-assets.mts index 97b94f02e8..66f5b041c5 100644 --- a/tools/check-deploy-assets.mts +++ b/tools/check-deploy-assets.mts @@ -29,7 +29,7 @@ export function unknownRuntimeAssetUrls(referenced: readonly string[], inventori } export async function checkDeployAssets(root = resolve(import.meta.dirname, '..')): Promise<{ files: number; urls: number }> { - const { stdout } = await execFileAsync('git', ['diff', '--name-only', '--', 'src/objects', 'site/prepared-facilities.json', 'site/prepared-sources.json'], { cwd: root }); + const { stdout } = await execFileAsync('git', ['diff', '--name-only', '--', 'src/objects'], { cwd: root }); const drift = stdout.split('\n').map(path => path.trim()).filter(Boolean); if (drift.length) throw new Error(`The deploy preparation changed committed object metadata:\n${drift.join('\n')}\nPrepare and publish those assets explicitly before deploying.`); const files = await textFiles(resolve(root, 'dist')); diff --git a/tools/prepare-facilities.mts b/tools/prepare-facilities.mts index b7d29d05db..04a5f1f9d6 100644 --- a/tools/prepare-facilities.mts +++ b/tools/prepare-facilities.mts @@ -44,7 +44,7 @@ export const explorationCompilerClosure = [ 'tools/objects/provenance.mts', 'tools/objects/provenance-records.mts', 'tools/objects/provenance-recipes.mts', 'tools/prepare-provenance.mts', ] as const; -interface Options { root?: string; publish?: boolean; provenance?: ReadonlyMap; sourceTransport?: FactsheetSourceTransport; +interface Options { root?: string; publish?: boolean | 'catalogues'; provenance?: ReadonlyMap; sourceTransport?: FactsheetSourceTransport; /** Opt-in (default null/off) content-addressed mirror for volume previews; a production caller names * RUNTIME_ASSET_ORIGIN explicitly. Left off by default so a test never makes a surprise real request. */ mirrorOrigin?: string | null; } @@ -166,14 +166,18 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. const prepared = parsePreparedExploration(payload,sources); const output = { path: resolve(root, 'site/prepared-facilities.json'), text: JSON.stringify(payload, null, 2) + '\n' }; const sourcesOutput = {path:resolve(root,'site/prepared-sources.json'),text:JSON.stringify(sourcePayload,null,2)+'\n'}; - const outputs = [...volumes.flatMap(volume => volume.outputs),sourcesOutput,output]; - if (publish) await writePreparedSet(outputs); - return { prepared, preparedSources, output, outputs, factsheets }; + const catalogueOutputs = [sourcesOutput,output]; + const outputs = [...volumes.flatMap(volume => volume.outputs),...catalogueOutputs]; + if (publish) await writePreparedSet(publish === 'catalogues' ? catalogueOutputs : outputs); + return { prepared, preparedSources, output, outputs, catalogueOutputs, factsheets }; } if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const args = process.argv.slice(2); + if (args.some(arg => arg !== '--catalog-only')) throw new TypeError('Usage: node tools/prepare-facilities.mts [--catalog-only]'); // The real CLI entry point: opts into the mirror explicitly (library code above defaults it off). - const { prepared, factsheets } = await prepareFacilities({ mirrorOrigin: RUNTIME_ASSET_ORIGIN }); + const { prepared, factsheets } = await prepareFacilities({ mirrorOrigin: RUNTIME_ASSET_ORIGIN, + publish: args.includes('--catalog-only') ? 'catalogues' : true }); console.log(`Prepared ${prepared.catalog.missions.length} missions, ${prepared.catalog.facilities.length} facilities and ${prepared.graph.datasets.length} dataset destinations.`); console.log(`Factsheets: ${factsheets.facts} facts, each with its own citation.`); } diff --git a/tools/source-catalogue.test.mts b/tools/source-catalogue.test.mts index 98e46da71c..155b689fbc 100644 --- a/tools/source-catalogue.test.mts +++ b/tools/source-catalogue.test.mts @@ -178,6 +178,7 @@ test('both catalogues prepare deterministically from the same input closure befo assert.ok(facts.some(edge => edge.objectId === 'abundantia' && edge.citationUrl?.includes('/4625'))); assert.ok(Object.hasOwn(result.preparedSources.closure, 'src/objects/earth/source/editorial/factsheet-review.json')); assert.ok(Object.hasOwn(result.preparedSources.closure, 'src/objects/abundantia/source/reference/damit-model.json')); + assert.deepEqual(result.catalogueOutputs, result.outputs.slice(-2), 'catalog-only publication excludes prepared volume and R2 outputs'); assert.deepEqual(sourceDatasetViews(prepared.usage, 'damit-models'), [], 'factsheet metadata is not a shape or imagery contribution'); for (const output of result.outputs) assert.deepEqual(typeof output.text === 'string' ? Buffer.from(output.text) : output.text,await readFile(output.path),output.path); assert.equal(result.prepared.sourceCatalogSha256,result.preparedSources.catalogSha256); From 838a607ea100d85a1685a65cc43f3b02847e9fff Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 07:54:54 -0300 Subject: [PATCH 3/8] fix(deploy): bind catalogues to installed R2 metadata --- astro.config.mts | 5 ++- tests/fixtures/context-package.mts | 4 ++ tools/prepare-context-availability.mts | 23 +++++++++-- tools/prepare-context-availability.test.mts | 26 +++++++++++++ tools/prepare-facilities.mts | 8 +++- tools/prepare-volume-provenance.mts | 43 +++++++++++++++++++++ 6 files changed, 102 insertions(+), 7 deletions(-) diff --git a/astro.config.mts b/astro.config.mts index 4cf4aba104..d3a48dfc0d 100644 --- a/astro.config.mts +++ b/astro.config.mts @@ -35,7 +35,10 @@ export default defineConfig({ // context package that setup:assets deliberately left missing after a 404 from R2, instead of failing the // whole build over one object. CI and local builds never set this flag and stay strict. const allowMissing = process.env.CSSEARTH_ALLOW_MISSING_ASSETS === '1'; - const { availability, failures } = await prepareContextAvailability({ strict: command === 'build' && !allowMissing }); + // An asset-origin build deliberately leaves public/scenes absent. Its tracked manifest is the local, + // content-addressed contract for previews already published to R2; all prepared package bytes stay strict. + const { availability, failures } = await prepareContextAvailability({ strict: command === 'build' && !allowMissing, + publicAssets: assetOrigin() ? 'manifest' : 'local' }); updateConfig({ vite: { define: { __CSSEARTH_CONTEXT_AVAILABILITY__: JSON.stringify(availability) } } }); if (failures.length) logger.warn(`Some 3D views are unavailable in this installation:\n${failures.join('\n')}\nPrepare their packages and restart the server to enable them.`); }, diff --git a/tests/fixtures/context-package.mts b/tests/fixtures/context-package.mts index c98535f217..3319451138 100644 --- a/tests/fixtures/context-package.mts +++ b/tests/fixtures/context-package.mts @@ -38,6 +38,10 @@ export async function writeContextPackage(root: string, id: string) { const files: [string, string | Uint8Array][] = [ [`${directory}/object.json`, JSON.stringify(descriptor)], [`${directory}/prepared/lenses.json`, bankBytes], [`${directory}/prepared/provenance.json`, JSON.stringify(provenance)], [`${directory}/prepared/presentation.json`, JSON.stringify(presentation)], + [`${directory}/runtime-assets.json`, JSON.stringify({ schema: `css${id}-runtime-assets@1`, resourceRoot: 'prepared', + assets: [{ filename: 'preview.webp', location: 'public', bytes: image.length, sha256: digest }] })], + [`${directory}/source/presentation.json`, JSON.stringify({ schema: 'cssearth-volume-presentation-source@1', objectId: id, + name: `${id} fixture`, defaultLens: 'optical', lenses: [{ id: 'optical' }] })], [`${directory}/prepared/slice.webp`, image], [`public${preview}`, image], ]; for (const [path, bytes] of files) { const file = resolve(root, path); await mkdir(dirname(file), { recursive: true }); await writeFile(file, bytes); } diff --git a/tools/prepare-context-availability.mts b/tools/prepare-context-availability.mts index e532d6e2bb..3c7ef33fa3 100644 --- a/tools/prepare-context-availability.mts +++ b/tools/prepare-context-availability.mts @@ -9,11 +9,15 @@ import type { ContextAvailability } from '../src/platform/context-availability.m import { parsePreparedVolumePresentation } from '../site/volume-presentation.mts'; import { readContextObjects } from './prepare-catalog.mts'; import { hasErrorCode } from './source-values.mts'; +import { requireRuntimeAssetManifest } from '../src/platform/runtime-asset-closure.mts'; const root = resolve(import.meta.dirname, '..'); +type PublicAssetAvailability = 'local' | 'manifest'; /** Verify complete volume packages once before serving; no source processing or downloads. */ -export async function inspectContextAvailability(projectRoot = root): Promise { +export async function inspectContextAvailability(projectRoot = root, { publicAssets = 'local' }: { + publicAssets?: PublicAssetAvailability; +} = {}): Promise { const contexts = await readContextObjects(resolve(projectRoot, 'src/objects')); const entries = await Promise.all(contexts.filter(object => object.type === 'volume-lens-bank').map(async ({ id }) => { const directory = resolve(projectRoot, 'src/objects', id); @@ -50,11 +54,20 @@ export async function inspectContextAvailability(projectRoot = root): Promise output.url === bankUrl); if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`); await verify(projectRoot, bankUrl, bankPin); + const published = publicAssets === 'manifest' + ? requireRuntimeAssetManifest(id, JSON.parse((await read(directory, 'runtime-assets.json')).toString())) + : null; for (const lens of presentation.controls) for (const url of new Set([lens.thumbnailUrl, lens.texture?.url])) { if (!url?.startsWith(`/scenes/${id}/`)) throw new TypeError(`Invalid dataset preview URL: ${url}.`); const pin = outputs.find(output => output.url === url); if (!pin) throw new TypeError(`Unpinned dataset preview: ${url}.`); - await verify(resolve(projectRoot, 'public'), url.slice(1), pin); + if (published) { + const filename = url.slice(`/scenes/${id}/`.length); + const asset = published.assets.find(candidate => candidate.filename === filename && + (published.resourceRoot !== 'prepared' || candidate.location === 'public')); + if (!asset || asset.sha256 !== pin.sha256 || asset.bytes !== pin.bytes) + throw new TypeError(`Unpublished dataset preview: ${url}.`); + } else await verify(resolve(projectRoot, 'public'), url.slice(1), pin); } return [id, { available: true }] as const; } catch (error) { @@ -64,8 +77,10 @@ export async function inspectContextAvailability(projectRoot = root): Promise state.available ? [] : [`${id}: ${state.reason}`]); if (strict && failures.length) throw new Error(`Prepared context packages unavailable:\n${failures.join('\n')}`); return { availability, failures }; diff --git a/tools/prepare-context-availability.test.mts b/tools/prepare-context-availability.test.mts index 0e17dd6136..41814bf59d 100644 --- a/tools/prepare-context-availability.test.mts +++ b/tools/prepare-context-availability.test.mts @@ -6,6 +6,7 @@ import { resolve } from 'node:path'; import { inspectContextAvailability, prepareContextAvailability } from './prepare-context-availability.mts'; import { parseContextAvailability } from '../src/platform/context-availability.mts'; import { writeContextPackage } from '../tests/fixtures/context-package.mts'; +import { readPreparedVolumeProvenance } from './prepare-volume-provenance.mts'; test('a missing bank isolates one object; restoring it admits the complete package on the next startup', async t => { const root = await mkdtemp(resolve(tmpdir(), 'cssearth-availability-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -53,3 +54,28 @@ test('invalid presentation and provenance cannot become available merely because assert.throws(() => parseContextAvailability({ helix: { available: 'true' } })); assert.throws(() => parseContextAvailability({ helix: { available: false } })); }); + +test('an asset-origin build verifies a missing local preview against its published manifest', async t => { + const root = await mkdtemp(resolve(tmpdir(), 'cssearth-availability-')); t.after(() => rm(root, { recursive: true, force: true })); + const f = await writeContextPackage(root, 'helix'); + await rm(resolve(root, 'public/scenes/helix/preview.webp')); + assert.equal((await inspectContextAvailability(root)).helix.available, false); + assert.deepEqual(await inspectContextAvailability(root, { publicAssets: 'manifest' }), { helix: { available: true } }); + const manifestPath = resolve(f.directory, 'runtime-assets.json'); + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + manifest.assets[0].sha256 = '0'.repeat(64); + await writeFile(manifestPath, JSON.stringify(manifest)); + assert.match((await inspectContextAvailability(root, { publicAssets: 'manifest' })).helix.reason ?? '', /Unpublished dataset preview/); +}); + +test('deploy catalogue input reads the installed prepared volume instead of regenerating it', async t => { + const root = await mkdtemp(resolve(tmpdir(), 'cssearth-prepared-volume-')); t.after(() => rm(root, { recursive: true, force: true })); + const fixture = await writeContextPackage(root, 'helix'); + const volumes = await readPreparedVolumeProvenance({ root }); + assert.equal(volumes.length, 1); + assert.equal(volumes[0]?.id, 'helix'); + assert.equal(volumes[0]?.name, 'helix fixture'); + assert.deepEqual(volumes[0]?.controls.map(control => control.id), ['optical']); + assert.deepEqual(volumes[0]?.provenance, fixture.provenance); + assert.deepEqual(volumes[0]?.outputs, []); +}); diff --git a/tools/prepare-facilities.mts b/tools/prepare-facilities.mts index 04a5f1f9d6..49cf5a66ce 100644 --- a/tools/prepare-facilities.mts +++ b/tools/prepare-facilities.mts @@ -24,7 +24,7 @@ import type { ProvenanceDocument } from '../src/platform/object-provenance.mts'; import { writePreparedSet } from './write-prepared-set.mts'; import { restoreFactsheetEvidence } from './restore-factsheet-evidence.mts'; import type { FactsheetSourceTransport } from './restore-factsheet-evidence.mts'; -import { prepareVolumeProvenance, volumeProvenanceCompilerClosure } from './prepare-volume-provenance.mts'; +import { prepareVolumeProvenance, readPreparedVolumeProvenance, volumeProvenanceCompilerClosure } from './prepare-volume-provenance.mts'; import { RUNTIME_ASSET_ORIGIN } from './source-mirror.mts'; export const explorationCompilerClosure = [ 'tools/prepare-facilities.mts', 'tools/spatial-source-citations.mts', 'packages/catalog/src/spatial.ts', 'packages/catalog/src/spatial-relations.ts', 'packages/catalog/src/clusters.ts', 'src/platform/exploration-catalog.mts', 'src/platform/exploration-contributions.mts', @@ -139,7 +139,11 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. inventory.push(...sourceInventory(manifest, `${base}/source/manifest.json`, sources, new Set(document.sources.map(source => source.path)))); objects.push({ id: object.id, name: object.name, route: object.route, base, controls: lenses, provenance: document }); } - const volumes = [...await prepareVolumeProvenance({ root, input, mirrorOrigin }), ...await prepareContextProvenance({ root, input })]; + // Deploys consume the exact prepared package restored from R2. Authoring preparation still rebuilds provenance + // and previews from their sources, but catalog-only publication must never invent a second package identity. + const volumes = publish === 'catalogues' + ? await readPreparedVolumeProvenance({ root, input }) + : [...await prepareVolumeProvenance({ root, input, mirrorOrigin }), ...await prepareContextProvenance({ root, input })]; for (const volume of volumes) { const document = validateObjectProvenance(volume.provenance, volume.id); const manifestPath = `${sourcePath(volume.base)}/${sourcePath(document.manifest.path)}`; diff --git a/tools/prepare-volume-provenance.mts b/tools/prepare-volume-provenance.mts index bccba9c0b2..6af97b3b2e 100644 --- a/tools/prepare-volume-provenance.mts +++ b/tools/prepare-volume-provenance.mts @@ -5,8 +5,10 @@ import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { basename, dirname, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import sharp from 'sharp'; +import { parseObjectDescriptor } from '@cssearth/objects'; import type { Lens } from '../site/planet-shell-types.ts'; import { validateDatasetText } from '../site/dataset-content.mts'; +import { parsePreparedVolumePresentation } from '../site/volume-presentation.mts'; import { parseCapture } from '../src/platform/exploration-catalog.mts'; import { validateObjectProvenance } from '../src/platform/object-provenance.mts'; import type { ProvenanceDocument, ProvenanceSource, ProvenanceJson } from '../src/platform/object-provenance.mts'; @@ -135,6 +137,47 @@ async function hostedDatasets(root: string, base: string, objectId: string, lens for (const lensId of lensIds) if (!datasets[lensId]) throw new TypeError(`No dataset of ${hostId} shows ${objectId}/${lensId}.`); return { objectId: hostId, name: sourceText(content.displayName), route: `/${hostId}/`, datasets }; } + +/** Read the prepared package that setup:assets installed. Deploy catalogue compilation must bind to these + * R2-backed bytes; rebuilding provenance from authoring inputs can describe a different package. */ +export async function readPreparedVolumeProvenance({ root = process.cwd(), input = path => readFile(resolve(root, path)) }: { + root?: string; input?: (path: string) => Promise; +} = {}): Promise { + const results: PreparedVolumeProvenance[] = []; + const folders = await readdir(resolve(root, 'src/objects'), { withFileTypes: true }); + for (const folder of folders.filter(folder => folder.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { + const id = folder.name, base = `src/objects/${id}`, sourcePresentationPath = `${base}/source/presentation.json`; + const exists = await readFile(resolve(root, sourcePresentationPath)).then(() => true, (error: unknown) => { + if (hasErrorCode(error, 'ENOENT')) return false; + throw error; + }); + if (!exists) continue; + const sourcePresentation = sourceObject(json(await input(sourcePresentationPath))); + if (sourcePresentation.schema === 'cssearth-volume-presentation-source@1') { + if (sourcePresentation.objectId !== id) throw new TypeError(`Mismatched volume presentation object: ${id}.`); + const descriptor = parseObjectDescriptor(json(await input(`${base}/object.json`))); + if (descriptor.id !== id || !descriptor.prepared || !['volume-lens-bank', 'image-layer-bank'].includes(descriptor.type)) + throw new TypeError(`Invalid prepared volume descriptor: ${id}.`); + const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id); + const defaultLens = sourceId(sourcePresentation.defaultLens); + const lensIds = sourceArray(sourcePresentation.lenses, raw => sourceId(sourceObject(raw).id)); + const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)), + { id, defaultLens, lenses: lensIds.map(lensId => ({ id: lensId })) }, provenance); + const bankUrl = `${base}/${descriptor.prepared!.url}`; + const bankPin = provenance.products.flatMap(product => product.outputs).find(output => output.url === bankUrl); + if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`); + const hostedBy = await hostedDatasets(root, base, id, prepared.controls.map(control => control.id), input); + results.push({ id, name: sourceText(sourcePresentation.name), route: hostedBy?.route ?? `/sun/?focus=${id}`, base, + controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], ...(hostedBy ? { hostedBy } : {}) }); + continue; + } + if (sourcePresentation.provenance === undefined) continue; + const preparedPresentation = sourceObject(json(await input(`${base}/prepared/presentation.json`))); + const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id); + results.push({ id, name: sourceText(preparedPresentation.name), route: '/sun/', base, controls: [], defaultLens: '', provenance, outputs: [] }); + } + return results; +} interface Options { root?: string; /** Repository-relative, tracked compiler inputs only. Downloads never enter source closure. */ From f378799feb0001291dd3148cdfab4fa82d5114fc Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 08:06:35 -0300 Subject: [PATCH 4/8] fix(deploy): separate source-only catalogue contexts --- tools/prepare-context-availability.test.mts | 10 +++++- tools/prepare-facilities.mts | 3 +- tools/prepare-volume-provenance.mts | 38 +++++++++------------ 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tools/prepare-context-availability.test.mts b/tools/prepare-context-availability.test.mts index 41814bf59d..3cde45c295 100644 --- a/tools/prepare-context-availability.test.mts +++ b/tools/prepare-context-availability.test.mts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; import { inspectContextAvailability, prepareContextAvailability } from './prepare-context-availability.mts'; @@ -79,3 +79,11 @@ test('deploy catalogue input reads the installed prepared volume instead of rege assert.deepEqual(volumes[0]?.provenance, fixture.provenance); assert.deepEqual(volumes[0]?.outputs, []); }); + +test('deploy volume input ignores source-only catalogue contexts with no prepared lens metadata', async t => { + const root = await mkdtemp(resolve(tmpdir(), 'cssearth-source-context-')); t.after(() => rm(root, { recursive: true, force: true })); + const path = resolve(root, 'src/objects/galaxy-clusters/source/presentation.json'); + await mkdir(resolve(path, '..'), { recursive: true }); + await writeFile(path, JSON.stringify({ provenance: { products: [] } })); + assert.deepEqual(await readPreparedVolumeProvenance({ root }), []); +}); diff --git a/tools/prepare-facilities.mts b/tools/prepare-facilities.mts index 49cf5a66ce..377ee2095a 100644 --- a/tools/prepare-facilities.mts +++ b/tools/prepare-facilities.mts @@ -142,7 +142,8 @@ export async function prepareFacilities({ root = resolve(import.meta.dirname, '. // Deploys consume the exact prepared package restored from R2. Authoring preparation still rebuilds provenance // and previews from their sources, but catalog-only publication must never invent a second package identity. const volumes = publish === 'catalogues' - ? await readPreparedVolumeProvenance({ root, input }) + ? [...await readPreparedVolumeProvenance({ root, input }), + ...((await prepareContextProvenance({ root, input })).map(context => ({ ...context, outputs: [] })))] : [...await prepareVolumeProvenance({ root, input, mirrorOrigin }), ...await prepareContextProvenance({ root, input })]; for (const volume of volumes) { const document = validateObjectProvenance(volume.provenance, volume.id); diff --git a/tools/prepare-volume-provenance.mts b/tools/prepare-volume-provenance.mts index 6af97b3b2e..e02ad9c474 100644 --- a/tools/prepare-volume-provenance.mts +++ b/tools/prepare-volume-provenance.mts @@ -153,28 +153,24 @@ export async function readPreparedVolumeProvenance({ root = process.cwd(), input }); if (!exists) continue; const sourcePresentation = sourceObject(json(await input(sourcePresentationPath))); - if (sourcePresentation.schema === 'cssearth-volume-presentation-source@1') { - if (sourcePresentation.objectId !== id) throw new TypeError(`Mismatched volume presentation object: ${id}.`); - const descriptor = parseObjectDescriptor(json(await input(`${base}/object.json`))); - if (descriptor.id !== id || !descriptor.prepared || !['volume-lens-bank', 'image-layer-bank'].includes(descriptor.type)) - throw new TypeError(`Invalid prepared volume descriptor: ${id}.`); - const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id); - const defaultLens = sourceId(sourcePresentation.defaultLens); - const lensIds = sourceArray(sourcePresentation.lenses, raw => sourceId(sourceObject(raw).id)); - const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)), - { id, defaultLens, lenses: lensIds.map(lensId => ({ id: lensId })) }, provenance); - const bankUrl = `${base}/${descriptor.prepared!.url}`; - const bankPin = provenance.products.flatMap(product => product.outputs).find(output => output.url === bankUrl); - if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`); - const hostedBy = await hostedDatasets(root, base, id, prepared.controls.map(control => control.id), input); - results.push({ id, name: sourceText(sourcePresentation.name), route: hostedBy?.route ?? `/sun/?focus=${id}`, base, - controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], ...(hostedBy ? { hostedBy } : {}) }); - continue; - } - if (sourcePresentation.provenance === undefined) continue; - const preparedPresentation = sourceObject(json(await input(`${base}/prepared/presentation.json`))); + // The three source-only catalogue contexts use source/presentation.json too, but are not prepared lens packages. + // prepareContextProvenance owns their in-memory catalogue records below; there are no R2 metadata files to read. + if (sourcePresentation.schema !== 'cssearth-volume-presentation-source@1') continue; + if (sourcePresentation.objectId !== id) throw new TypeError(`Mismatched volume presentation object: ${id}.`); + const descriptor = parseObjectDescriptor(json(await input(`${base}/object.json`))); + if (descriptor.id !== id || !descriptor.prepared || !['volume-lens-bank', 'image-layer-bank'].includes(descriptor.type)) + throw new TypeError(`Invalid prepared volume descriptor: ${id}.`); const provenance = validateObjectProvenance(json(await input(`${base}/prepared/provenance.json`)), id); - results.push({ id, name: sourceText(preparedPresentation.name), route: '/sun/', base, controls: [], defaultLens: '', provenance, outputs: [] }); + const defaultLens = sourceId(sourcePresentation.defaultLens); + const lensIds = sourceArray(sourcePresentation.lenses, raw => sourceId(sourceObject(raw).id)); + const prepared = parsePreparedVolumePresentation(json(await input(`${base}/prepared/presentation.json`)), + { id, defaultLens, lenses: lensIds.map(lensId => ({ id: lensId })) }, provenance); + const bankUrl = `${base}/${descriptor.prepared!.url}`; + const bankPin = provenance.products.flatMap(product => product.outputs).find(output => output.url === bankUrl); + if (!bankPin || bankPin.sha256 !== descriptor.prepared!.sha256) throw new TypeError(`Unbound prepared bank: ${bankUrl}.`); + const hostedBy = await hostedDatasets(root, base, id, prepared.controls.map(control => control.id), input); + results.push({ id, name: sourceText(sourcePresentation.name), route: hostedBy?.route ?? `/sun/?focus=${id}`, base, + controls: prepared.controls, defaultLens: prepared.defaultLens, provenance, outputs: [], ...(hostedBy ? { hostedBy } : {}) }); } return results; } From 6072e39ea3d6c2bf47251d557429f406354bf50a Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 08:10:49 -0300 Subject: [PATCH 5/8] test(deploy): exercise deploy preparation in PR checks --- .github/workflows/nightly.yml | 6 +++--- tools/check-ci.test.mts | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4410969737..03e5bbd996 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: pull_request: # Only the deploy-build job below runs from this trigger (each other job's `if:` stays schedule/dispatch-only): - # a full `pnpm build` is too slow to run on every PR, but a PR touching the code this specific path resolves + # a full `pnpm build:deploy` is too slow to run on every PR, but a PR touching the code this specific path resolves # through gets it anyway, instead of waiting for the nightly schedule to notice a break. paths: - 'site/**' @@ -86,7 +86,7 @@ jobs: asset-origin-build: # Exercises the one path that has broken two real deploys and that no other CI job builds at all: a full - # `pnpm build` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static + # `pnpm build:deploy` with `ASSET_ORIGIN` set, the same as .github/workflows/deploy.yml's production path. The static # scan catches emitted same-origin addresses; the production browser test also exercises the client loader, # whose descriptor path can otherwise discard an origin that is present in the emitted page. name: Build with ASSET_ORIGIN and exercise the client loader @@ -123,7 +123,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=6144 CSSEARTH_ALLOW_MISSING_ASSETS: "1" ASSET_ORIGIN: https://asset-origin-ci-check.example - run: pnpm build + run: pnpm build:deploy - name: Check no page references a same-origin /scenes/ address run: node tools/check-asset-origin-scenes.mts dist - name: Check production navigation loads content-addressed assets diff --git a/tools/check-ci.test.mts b/tools/check-ci.test.mts index d9d1a189ce..83fd1c8a54 100644 --- a/tools/check-ci.test.mts +++ b/tools/check-ci.test.mts @@ -39,6 +39,12 @@ test('the deploy consumes installed assets, rebuilds only catalogues and rejects assert.equal(packageFile.scripts['prepare:deploy-catalogues'],'node tools/prepare-facilities.mts --catalog-only'); assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:(?:facilities|provenance|nebulae)(?:\s|$)/); }); +test('the PR asset-origin check exercises the exact deploy build path',async()=>{ + const workflow=await readFile(new URL('../.github/workflows/nightly.yml',import.meta.url),'utf8'); + const steps=readCiSteps(workflow,'asset-origin-build'); + const build=steps.find(step=>step.name==='Build the site with ASSET_ORIGIN set to a test origin'); + assert.equal(build?.run.trim(),'pnpm build:deploy'); +}); test('--quick skips only the network and documentation steps, and refuses a job without them',async()=>{ const lint=readCiSteps(await readFile(new URL('../.github/workflows/universe.yml',import.meta.url),'utf8'),'lint'); const quick=quickSteps(lint); From d09b726765891eb62366785d8e49a79591fbf646 Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 08:12:08 -0300 Subject: [PATCH 6/8] ci(deploy): run deploy build for preparation changes --- .github/workflows/nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 03e5bbd996..1d29da6a4b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,10 +9,15 @@ on: # a full `pnpm build:deploy` is too slow to run on every PR, but a PR touching the code this specific path resolves # through gets it anyway, instead of waiting for the nightly schedule to notice a break. paths: + - '.github/workflows/nightly.yml' + - 'package.json' + - 'pnpm-lock.yaml' - 'site/**' - 'astro.config.mts' - 'src/renderers/css/rendering/**' - '.github/workflows/deploy.yml' + - 'tools/prepare-*.mts' + - 'tools/setup-assets.mts' permissions: contents: read From e9e023a189ad9c874be4bd595489649776e7db67 Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 08:45:50 -0300 Subject: [PATCH 7/8] fix(deploy): restore volume catalogue metadata from R2 --- .github/workflows/nightly.yml | 2 +- package.json | 3 +- tools/check-ci.test.mts | 1 + tools/prepare-context-availability.test.mts | 4 +- tools/runtime-assets.mts | 13 +++-- tools/setup-volume-metadata.mts | 53 +++++++++++++++++ tools/setup-volume-metadata.test.mts | 63 +++++++++++++++++++++ 7 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 tools/setup-volume-metadata.mts create mode 100644 tools/setup-volume-metadata.test.mts diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1d29da6a4b..a5029fb8ec 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -17,7 +17,7 @@ on: - 'src/renderers/css/rendering/**' - '.github/workflows/deploy.yml' - 'tools/prepare-*.mts' - - 'tools/setup-assets.mts' + - 'tools/setup*.mts' permissions: contents: read diff --git a/package.json b/package.json index 7ca8bd6b9a..cb0a157f21 100644 --- a/package.json +++ b/package.json @@ -110,8 +110,9 @@ "publish:earth-global": "node tools/objects/geographic-pages/operations/publish-global-wmts.mts --object=earth", "publish:runtime-assets": "node tools/publish-runtime-assets.mts", "refresh:earth-enso": "node tools/objects/paged-ellipsoid/refresh-earth-enso.mts && node tools/objects/dist/prepare-authored.js earth --write && node tools/prepare-text.mts earth", - "setup:assets": "pnpm prepare:shell && node tools/setup.mts && node tools/setup-prepared.mts", + "setup:assets": "pnpm prepare:shell && node tools/setup.mts && node tools/setup-prepared.mts && node tools/setup-volume-metadata.mts", "setup:prepared": "node tools/setup-prepared.mts", + "setup:volume-metadata": "node tools/setup-volume-metadata.mts", "status:earth-city-global": "node tools/objects/geographic-pages/operations/status-global-city.mts --object=earth", "telescope:publish-map": "node tools/run-typed-module.mjs tools/objects/body-map-publication.mts", "telescope:query": "node tools/run-typed-module.mjs tools/objects/telescopes/query.mts", diff --git a/tools/check-ci.test.mts b/tools/check-ci.test.mts index 83fd1c8a54..0a41038c7f 100644 --- a/tools/check-ci.test.mts +++ b/tools/check-ci.test.mts @@ -37,6 +37,7 @@ test('the deploy consumes installed assets, rebuilds only catalogues and rejects assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:galaxy-field:data/); assert.match(packageFile.scripts['prepare:deploy']??'',/pnpm prepare:deploy-catalogues/); assert.equal(packageFile.scripts['prepare:deploy-catalogues'],'node tools/prepare-facilities.mts --catalog-only'); + assert.match(packageFile.scripts['setup:assets']??'',/node tools\/setup-volume-metadata\.mts/); assert.doesNotMatch(packageFile.scripts['prepare:deploy']??'',/prepare:(?:facilities|provenance|nebulae)(?:\s|$)/); }); test('the PR asset-origin check exercises the exact deploy build path',async()=>{ diff --git a/tools/prepare-context-availability.test.mts b/tools/prepare-context-availability.test.mts index 3cde45c295..151bcf145d 100644 --- a/tools/prepare-context-availability.test.mts +++ b/tools/prepare-context-availability.test.mts @@ -65,7 +65,9 @@ test('an asset-origin build verifies a missing local preview against its publish const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); manifest.assets[0].sha256 = '0'.repeat(64); await writeFile(manifestPath, JSON.stringify(manifest)); - assert.match((await inspectContextAvailability(root, { publicAssets: 'manifest' })).helix.reason ?? '', /Unpublished dataset preview/); + const unavailable = (await inspectContextAvailability(root, { publicAssets: 'manifest' })).helix; + if (unavailable.available) assert.fail('Changed preview identity must make the package unavailable.'); + assert.match(unavailable.reason, /Unpublished dataset preview/); }); test('deploy catalogue input reads the installed prepared volume instead of regenerating it', async t => { diff --git a/tools/runtime-assets.mts b/tools/runtime-assets.mts index 108375980d..3bb3817155 100644 --- a/tools/runtime-assets.mts +++ b/tools/runtime-assets.mts @@ -66,14 +66,16 @@ async function locatedAssets(root: string, id: string, a return located; } -export async function runtimeAssets(root: string, objectIds: readonly string[]): Promise { +export async function runtimeAssets(root: string, objectIds: readonly string[], + { filenames }: { filenames?: readonly string[] } = {}): Promise { const assets: RuntimeAssetLocation[] = []; + const selectedFilenames = filenames ? new Set(filenames) : null; for (const id of objectIds) { if (!/^[a-z][a-z0-9-]*$/u.test(id)) throw new TypeError(`Unsafe runtime object identity: ${id}`); const base = resolve(root, `src/objects/${id}`); const bytes = await readFile(resolve(base, "runtime-assets.json")); const manifest = requireRuntimeAssetManifest(id, JSON.parse(bytes.toString("utf8"))); - for (const asset of manifest.assets) { + for (const asset of manifest.assets.filter(asset => !selectedFilenames || selectedFilenames.has(asset.filename))) { const assetRoot = manifest.resourceRoot === "prepared" && asset.location !== "public" ? resolve(base, "prepared") : resolve(root, `public/scenes/${id}`); assets.push(...await locatedAssets(root, id, assetRoot, [asset])); } @@ -82,14 +84,17 @@ export async function runtimeAssets(root: string, objectIds: readonly string[]): } /** Counterpart of `runtimeAssets` for `prepared-assets.json`: always resourceRoot `prepared`, never public. */ -export async function preparedAssets(root: string, objectIds: readonly string[]): Promise { +export async function preparedAssets(root: string, objectIds: readonly string[], + { filenames }: { filenames?: readonly string[] } = {}): Promise { const assets: RuntimeAssetLocation[] = []; + const selectedFilenames = filenames ? new Set(filenames) : null; for (const id of objectIds) { if (!/^[a-z][a-z0-9-]*$/u.test(id)) throw new TypeError(`Unsafe prepared object identity: ${id}`); const base = resolve(root, `src/objects/${id}`); const bytes = await readFile(resolve(base, "prepared-assets.json")); const manifest = requirePreparedAssetManifest(id, JSON.parse(bytes.toString("utf8"))); - assets.push(...await locatedAssets(root, id, resolve(base, "prepared"), manifest.assets)); + assets.push(...await locatedAssets(root, id, resolve(base, "prepared"), + manifest.assets.filter(asset => !selectedFilenames || selectedFilenames.has(asset.filename)))); } return assets; } diff --git a/tools/setup-volume-metadata.mts b/tools/setup-volume-metadata.mts new file mode 100644 index 0000000000..cbd3d670c7 --- /dev/null +++ b/tools/setup-volume-metadata.mts @@ -0,0 +1,53 @@ +import { readFile, readdir } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { sourceObject, sourceText } from '../src/platform/source-catalog.mts'; +import { preparedAssets, runtimeAssets } from './runtime-assets.mts'; +import { installRuntimeAssets } from './setup.mts'; +import { hasErrorCode } from './source-values.mts'; + +export const VOLUME_METADATA_FILENAMES = ['presentation.json', 'provenance.json'] as const; + +/** Select only the small R2-backed package metadata needed to compile deploy catalogues. Volume textures and + * slices stay remote; restoring the full runtime-assets closure here can be tens of gigabytes. */ +export async function preparedVolumeMetadataAssets(root = resolve(import.meta.dirname, '..')) { + const ids: string[] = []; + const folders = await readdir(resolve(root, 'src/objects'), { withFileTypes: true }); + for (const folder of folders.filter(folder => folder.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { + const path = resolve(root, 'src/objects', folder.name, 'source/presentation.json'); + const bytes = await readFile(path).catch((error: unknown) => { + if (hasErrorCode(error, 'ENOENT')) return undefined; + throw error; + }); + if (!bytes) continue; + const presentation = sourceObject(JSON.parse(bytes.toString('utf8'))); + if (presentation.schema !== 'cssearth-volume-presentation-source@1') continue; + if (sourceText(presentation.objectId) !== folder.name) throw new TypeError(`Mismatched volume presentation object: ${folder.name}.`); + ids.push(folder.name); + } + + const preparedIds = ids.filter(id => existsSync(resolve(root, 'src/objects', id, 'prepared-assets.json'))); + const assets = [ + ...await runtimeAssets(root, ids, { filenames: VOLUME_METADATA_FILENAMES }), + ...await preparedAssets(root, preparedIds, { filenames: VOLUME_METADATA_FILENAMES }), + ]; + for (const id of ids) for (const filename of VOLUME_METADATA_FILENAMES) { + const matches = assets.filter(asset => asset.id === id && asset.filename === filename); + if (matches.length !== 1 || matches[0]!.location === 'public' || + matches[0]!.file !== resolve(root, 'src/objects', id, 'prepared', filename)) { + throw new TypeError(`Volume ${id} must inventory exactly one prepared/${filename} runtime asset.`); + } + } + return { ids, assets }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const root = resolve(import.meta.dirname, '..'); + const { ids, assets } = await preparedVolumeMetadataAssets(root); + console.log(`Setting up catalogue metadata for ${ids.join(', ')}: ${assets.length} file(s); volume data stays on R2.`); + const result = await installRuntimeAssets(assets, { onProgress: ({ completed, total }) => { + if (completed % 10 === 0 || completed === total) console.log(`Volume metadata: ${completed}/${total}`); + } }); + console.log(`Volume metadata setup complete: ${result.installed} downloaded, ${result.reused} reused.`); +} diff --git a/tools/setup-volume-metadata.test.mts b/tools/setup-volume-metadata.test.mts new file mode 100644 index 0000000000..db6c55cdb9 --- /dev/null +++ b/tools/setup-volume-metadata.test.mts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import test from 'node:test'; +import { installRuntimeAssets } from './setup.mts'; +import { preparedVolumeMetadataAssets } from './setup-volume-metadata.mts'; + +const sha256 = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex'); + +test('catalogue bootstrap restores only prepared volume metadata and leaves dataset bytes on R2', async t => { + const root = await mkdtemp(resolve(tmpdir(), 'cssearth-volume-metadata-')); + t.after(() => rm(root, { recursive: true, force: true })); + const base = resolve(root, 'src/objects/m31'); + const files = new Map([ + ['presentation.json', Buffer.from('{"schema":"prepared-presentation"}')], + ['provenance.json', Buffer.from('{"schema":"prepared-provenance"}')], + ['datasets/large.webp', Buffer.from('dataset fixture that must remain remote')], + ]); + const sourcePresentation = resolve(base, 'source/presentation.json'); + await mkdir(dirname(sourcePresentation), { recursive: true }); + await writeFile(sourcePresentation, JSON.stringify({ schema: 'cssearth-volume-presentation-source@1', objectId: 'm31' })); + await writeFile(resolve(base, 'runtime-assets.json'), JSON.stringify({ + schema: 'cssm31-runtime-assets@1', resourceRoot: 'prepared', + assets: [...files].map(([filename, bytes]) => ({ filename, bytes: bytes.length, sha256: sha256(bytes) })), + })); + const preparedBase = resolve(root, 'src/objects/helix'); + const preparedSource = resolve(preparedBase, 'source/presentation.json'); + await mkdir(dirname(preparedSource), { recursive: true }); + await writeFile(preparedSource, JSON.stringify({ schema: 'cssearth-volume-presentation-source@1', objectId: 'helix' })); + await writeFile(resolve(preparedBase, 'runtime-assets.json'), JSON.stringify({ + schema: 'csshelix-runtime-assets@1', resourceRoot: 'prepared', + assets: [{ filename: 'datasets/large.webp', location: 'public', bytes: files.get('datasets/large.webp')!.length, + sha256: sha256(files.get('datasets/large.webp')!) }], + })); + await writeFile(resolve(preparedBase, 'prepared-assets.json'), JSON.stringify({ + schema: 'csshelix-prepared-assets@1', resourceRoot: 'prepared', + assets: ['presentation.json', 'provenance.json'].map(filename => ({ filename, bytes: files.get(filename)!.length, + sha256: sha256(files.get(filename)!) })), + })); + const sourceOnly = resolve(root, 'src/objects/local-group/source/presentation.json'); + await mkdir(dirname(sourceOnly), { recursive: true }); + await writeFile(sourceOnly, JSON.stringify({ provenance: { products: [] } })); + + const selected = await preparedVolumeMetadataAssets(root); + assert.deepEqual(selected.ids, ['helix', 'm31']); + assert.deepEqual(selected.assets.map(asset => `${asset.id}/${asset.filename}`).sort(), [ + 'helix/presentation.json', 'helix/provenance.json', 'm31/presentation.json', 'm31/provenance.json', + ]); + const requested: string[] = []; + assert.deepEqual(await installRuntimeAssets(selected.assets, { fetcher: async url => { + const asset = selected.assets.find(asset => asset.url === String(url)); + assert.ok(asset); + requested.push(asset.filename); + return new Response(files.get(asset.filename)); + } }), { installed: 4, reused: 0, skipped: 0 }); + assert.deepEqual(requested.sort(), ['presentation.json', 'presentation.json', 'provenance.json', 'provenance.json']); + assert.equal(await readFile(resolve(base, 'prepared/presentation.json'), 'utf8'), files.get('presentation.json')!.toString()); + assert.equal(await readFile(resolve(preparedBase, 'prepared/provenance.json'), 'utf8'), files.get('provenance.json')!.toString()); + await assert.rejects(readFile(resolve(base, 'prepared/datasets/large.webp')), { code: 'ENOENT' }); + await assert.rejects(readFile(resolve(root, 'public/scenes/helix/datasets/large.webp')), { code: 'ENOENT' }); +}); From d7fc3a9b5907309d5d06cda8ec19e825fa518b13 Mon Sep 17 00:00:00 2001 From: alowpoly Date: Sun, 20 Sep 2026 08:56:33 -0300 Subject: [PATCH 8/8] test(deploy): exercise remote R2 assets without bulk restore --- site/test/navigation-production-browser.mts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/site/test/navigation-production-browser.mts b/site/test/navigation-production-browser.mts index 80e5a0bc47..85df413194 100644 --- a/site/test/navigation-production-browser.mts +++ b/site/test/navigation-production-browser.mts @@ -7,6 +7,7 @@ import { readFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { inventoriedAssets, inventoriedObjectIds } from '../../tools/runtime-assets.mts'; import { contentType } from '../../tools/publish-runtime-assets.mts'; +import { sha256 } from '../../src/platform/sha256.mts'; const origin = process.env.CSSEARTH_TEST_ORIGIN ?? 'http://127.0.0.1:4212'; const channel = process.env.PLAYWRIGHT_CHANNEL ?? 'chrome'; @@ -36,7 +37,19 @@ try { const url = new URL(route.request().url()), asset = assets.get(url.pathname); if (!asset) return route.fulfill({ status: 404, body: `Uninventoried asset: ${url.pathname}` }); assetRequests.push(url.href); - return route.fulfill({ body: await readFile(asset.file), contentType: contentType(asset.key), + let body = await readFile(asset.file).catch((error: unknown) => { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return undefined; + throw error; + }); + // A deploy intentionally keeps large context datasets on R2. The fake ASSET_ORIGIN route still verifies + // their real content-addressed bytes instead of requiring the CI checkout to download every inventory. + if (!body) { + const response = await fetch(asset.url, { signal: AbortSignal.timeout(120000) }); + if (!response.ok) throw new Error(`Published asset unavailable: ${asset.url} (HTTP ${response.status}).`); + body = Buffer.from(await response.arrayBuffer()); + } + if (body.length !== asset.bytes || sha256(body) !== asset.sha256) throw new Error(`Published asset identity changed: ${asset.key}.`); + return route.fulfill({ body, contentType: contentType(asset.key), headers: { 'access-control-allow-origin': '*' } }); }); }