From 2e19936d1a12b6878078bc53d3cf0ad7054bf3fd Mon Sep 17 00:00:00 2001 From: Niels Bik Date: Sun, 23 Aug 2026 21:09:57 +0200 Subject: [PATCH] fix: always regenerate missing images --- README.md | 7 ++ src/server/dictionaries.test.ts | 26 ++++- src/server/dictionaries.ts | 36 +++++- src/server/generate.test.ts | 79 ++++++++++++- src/server/generate.ts | 189 +++++++++++++++++++++++++++----- src/server/reconcile.test.ts | 77 +++++++++++++ src/server/reconcile.ts | 69 ++++++++++++ src/server/refresh.ts | 41 +++++-- 8 files changed, 480 insertions(+), 44 deletions(-) create mode 100644 src/server/reconcile.test.ts create mode 100644 src/server/reconcile.ts diff --git a/README.md b/README.md index d529011..50941de 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,15 @@ How it behaves: - **Offline-safe / disable.** A failed fetch is non-fatal (silhouette until art exists). Set `ILLUSTRATIONS_REPO=` (empty) to turn downloading off entirely; pin `ILLUSTRATIONS_REF` to a release tag instead of `main` for a fixed art set. +- **Gaps it can't fill are remembered, not retried forever.** A cutout the repo doesn't have (or + that generation declined) is logged once and left alone for a while — a week for a repo miss, a + day for a generation miss — instead of being re-requested on every refresh. The record lives in + `_misses.json` in the illustrations volume; delete it to retry everything immediately. - **Self-healing.** Delete a cutout from the volume and the service notices, re-downloads it (or regenerates it), and rebuilds the manifest — which is also how you replace art you don't like. + This works for a single pose of a pair, and for a bird that hasn't been heard in weeks: every + refresh compares what's on disk against what should be there and repairs the difference, at + startup as well as while running. - **Licensing.** The illustrations (and the generation pipeline the image bundles) are **CC-BY-NC-SA-4.0** — non-commercial. See the illustrations repo and *Credits and licensing* below. diff --git a/src/server/dictionaries.test.ts b/src/server/dictionaries.test.ts index e308df3..0c9f114 100644 --- a/src/server/dictionaries.test.ts +++ b/src/server/dictionaries.test.ts @@ -27,7 +27,7 @@ describe('publishDictionaries', () => { return Promise.resolve(notFound()) }) - const index = await publishDictionaries({ + const { index } = await publishDictionaries({ baseUrl: 'http://bng:8080', htmlDir, locales: ['de', 'nl'], @@ -46,7 +46,7 @@ describe('publishDictionaries', () => { Promise.resolve(url.endsWith('/species/dictionary/de') ? ok({ x: 'y' }) : notFound()), ) - const index = await publishDictionaries({ + const { index } = await publishDictionaries({ baseUrl: 'http://bng:8080', htmlDir, locales: ['de', 'nl'], @@ -69,4 +69,26 @@ describe('publishDictionaries', () => { const [, opts] = fetchMock.mock.calls[0] expect(opts.headers.Authorization).toBe('Bearer secret') }) + + it('indexes slugs from the English dictionary so disk-only slugs can be named', async () => { + const htmlDir = await tmpHtmlDir() + fetchMock.mockImplementation((url: string) => { + if (url.endsWith('/species/dictionary/en')) + return Promise.resolve(ok({ 'Turdus merula': 'Eurasian Blackbird' })) + if (url.endsWith('/species/dictionary/nl')) + return Promise.resolve(ok({ 'Turdus merula': 'Merel' })) + return Promise.resolve(notFound()) + }) + + const { sciBySlug } = await publishDictionaries({ + baseUrl: 'http://bng:8080', + htmlDir, + locales: ['nl', 'en'], + }) + + expect(sciBySlug.get('turdus-merula')).toEqual({ + sci: 'Turdus merula', + com: 'Eurasian Blackbird', + }) + }) }) diff --git a/src/server/dictionaries.ts b/src/server/dictionaries.ts index 58de7dc..e16e14a 100644 --- a/src/server/dictionaries.ts +++ b/src/server/dictionaries.ts @@ -2,6 +2,7 @@ import { mkdir, rename, writeFile } from 'node:fs/promises' import { join } from 'node:path' import type { DictionaryIndex } from '../domain/dictionary.ts' import { type DictLocale, reduceToDictLocale, SUPPORTED_DICT_LOCALES } from '../domain/locale.ts' +import { slugify } from '../domain/slug.ts' import { birdnetFetch } from './birdnet.ts' // The refresh service downloads BirdNET-Go's species-name dictionaries and @@ -42,18 +43,46 @@ async function fetchStationDefault(baseUrl: string, token?: string): Promise species name, over every species BirdNET-Go knows — not just the ones + * this station has heard. Lets the refresh service name a cutout it finds on disk + * whose bird predates the detection window, so a deleted pose can be regenerated + * under the right name instead of a guess reversed out of the slug. */ + sciBySlug: ReadonlyMap +} + +/** English if it was published, else whatever was — the keys (scientific names) are + * the same in every locale, only the display name differs. */ +function buildSciBySlug( + published: readonly DictLocale[], + maps: ReadonlyMap>, +): ReadonlyMap { + const locale = published.includes('en') ? 'en' : published[0] + const map = locale ? maps.get(locale) : undefined + const out = new Map() + if (!map) return out + for (const [sci, com] of Object.entries(map)) { + const slug = slugify(sci) + if (slug) out.set(slug, { sci, com: com || sci }) + } + return out +} + /** Download each requested species dictionary from BirdNET-Go and publish it as a * static file under /species-dict/, plus an index.json listing what was * published and the station's default language. Locales the backend lacks (404) or * that fail are skipped, so an older build simply yields an empty index and the - * browser falls back to the station's own names. Returns the published index. */ + * browser falls back to the station's own names. Returns the published index plus + * a slug index (see buildSciBySlug). */ export async function publishDictionaries( cfg: PublishDictionariesConfig, -): Promise { +): Promise { const dir = join(cfg.htmlDir, 'species-dict') await mkdir(dir, { recursive: true }) const wanted = cfg.locales ?? SUPPORTED_DICT_LOCALES const published: DictLocale[] = [] + const maps = new Map>() for (const locale of wanted) { try { @@ -62,6 +91,7 @@ export async function publishDictionaries( const map = (await res.json()) as Record await writeJson(join(dir, `${locale}.json`), map) published.push(locale) + maps.set(locale, map) } catch (e) { log(`skip ${locale}: ${e instanceof Error ? e.message : String(e)}`) } @@ -75,5 +105,5 @@ export async function publishDictionaries( log( `published ${published.length}/${wanted.length} dictionaries (default: ${index.default ?? 'none'})`, ) - return index + return { index, sciBySlug: buildSciBySlug(published, maps) } } diff --git a/src/server/generate.test.ts b/src/server/generate.test.ts index 89eb719..c4814fc 100644 --- a/src/server/generate.test.ts +++ b/src/server/generate.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readdir, rm } from 'node:fs/promises' +import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest' @@ -34,14 +34,19 @@ const BASE = 'http://fixtures.test' let assetsDir: string let onGenerated: Mock<() => Promise> +// Pose stems requested since the last stubFetch, in order. +const fetched: string[] = [] + // fetch stub: 200 for exactly the listed pose filenames (e.g. 'turdus-merula' for the // perched pose, 'turdus-merula-2' for flight), 404 for everything else. function stubFetch(presentPoses: string[]) { + fetched.length = 0 vi.stubGlobal( 'fetch', vi.fn(async (url: string) => { const m = url.match(/\/illustrations\/(.+)\.png$/) const name = m?.[1] + if (name) fetched.push(name) if (name && presentPoses.includes(name)) { return { ok: true, arrayBuffer: async () => PNG.buffer.slice(0) } as unknown as Response } @@ -135,6 +140,78 @@ describe('Generator art acquisition', () => { expect(onGenerated).toHaveBeenCalledTimes(1) }) + it('re-downloads only the pose deleted from an existing pair', async () => { + // The reported bug's happy path: one image removed by hand, the other left alone. + stubFetch(['turdus-merula', 'turdus-merula-2']) + await writeFile(join(assetsDir, 'turdus-merula.png'), PNG) + const g = makeGen({ enabled: true, downloadBaseUrl: BASE }) + g.enqueueRepairs([{ slug: 'turdus-merula', sci: 'Turdus merula', com: 'Eurasian Blackbird' }]) + await idle(g) + + expect(fetched).toEqual(['turdus-merula-2']) + expect(await readdir(assetsDir)).toContain('turdus-merula-2.png') + expect(spawnCalls.some((a) => a.includes('--generate'))).toBe(false) + }) + + it('downloads but never generates a repair whose species we cannot name', async () => { + stubFetch([]) // repo has nothing, and we have no scientific name to prompt with + const g = makeGen({ enabled: true, downloadBaseUrl: BASE }) + g.enqueueRepairs([{ slug: 'turdus-merula' }]) + await idle(g) + + expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2']) + expect(spawnCalls.some((a) => a.includes('--generate'))).toBe(false) + }) + + it('remembers a repo miss and stops re-requesting it, across restarts', async () => { + stubFetch([]) + const g = makeGen({ enabled: false, downloadBaseUrl: BASE }) + g.enqueue('Turdus merula', 'Eurasian Blackbird') + await idle(g) + expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2']) + + const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8')) + expect(Object.keys(misses.download).sort()).toEqual(['turdus-merula', 'turdus-merula-2']) + + fetched.length = 0 + const restarted = makeGen({ enabled: false, downloadBaseUrl: BASE }) + restarted.enqueue('Turdus merula', 'Eurasian Blackbird') + await idle(restarted) + expect(fetched).toEqual([]) + }) + + it('retries a repo miss once its backoff has expired', async () => { + const stale = Date.now() - 8 * 24 * 60 * 60 * 1000 // older than the 7d download TTL + await writeFile( + join(assetsDir, '_misses.json'), + JSON.stringify({ download: { 'turdus-merula': stale, 'turdus-merula-2': stale } }), + ) + stubFetch(['turdus-merula', 'turdus-merula-2']) + const g = makeGen({ enabled: false, downloadBaseUrl: BASE }) + g.enqueue('Turdus merula', 'Eurasian Blackbird') + await idle(g) + + expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2']) + expect(await readdir(assetsDir)).toContain('turdus-merula-2.png') + const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8')) + expect(misses.download).toEqual({}) // both poses landed + }) + + it('backs off a species the model declines rather than regenerating every sweep', async () => { + stubFetch([]) // repo has nothing; the mocked worker exits 0 without writing anything + const g = makeGen({ enabled: true, downloadBaseUrl: BASE }) + g.enqueue('Turdus merula', 'Eurasian Blackbird') + await idle(g) + expect(spawnCalls.filter((a) => a.includes('--generate'))).toHaveLength(1) + + g.enqueue('Turdus merula', 'Eurasian Blackbird') + await idle(g) + expect(spawnCalls.filter((a) => a.includes('--generate'))).toHaveLength(1) + + const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8')) + expect(Object.keys(misses.generate).sort()).toEqual(['turdus-merula', 'turdus-merula-2']) + }) + it('no-ops enqueue when both sources are disabled', async () => { stubFetch(['turdus-merula']) const g = makeGen({ enabled: false, downloadBaseUrl: '' }) diff --git a/src/server/generate.ts b/src/server/generate.ts index 73a0207..91e8353 100644 --- a/src/server/generate.ts +++ b/src/server/generate.ts @@ -1,9 +1,10 @@ import { spawn } from 'node:child_process' import { existsSync } from 'node:fs' -import { mkdir, rename, writeFile } from 'node:fs/promises' +import { mkdir, readFile, rename, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { FLIGHT_SUFFIX } from '../domain/asset.ts' import { slugify } from '../domain/slug.ts' +import type { ArtRepair } from './reconcile.ts' // Art queue: turns "this species was heard but has no cutout" into art, via two // sources that compose. For each heard species we FIRST try to download a ready-made @@ -13,14 +14,29 @@ import { slugify } from '../domain/slug.ts' // the manifest is rebuilt (so downloaded art is described even with no key) and // onGenerated republishes. A no-op when neither source is available. -interface ArtRequest { - sci: string - com: string -} +type ArtRequest = ArtRepair // '' = base (perched) pose, FLIGHT_SUFFIX = flight pose. const POSE_SUFFIXES = ['', FLIGHT_SUFFIX] +const MISS_SOURCES = ['download', 'generate'] as const +type MissSource = (typeof MISS_SOURCES)[number] + +// How long a source is left alone after it failed to supply a pose. Without this, +// every publish re-requests a 404 for every never-illustrated species — which is most +// of them — and the sweep now spans every species we've ever heard. +// download: the repo does gain art, just rarely; same TTL as the call archives. +// generate: shorter, because a pose the model declined is worth retrying sooner than +// one the repo simply lacks — but not every publish, which spends real quota. +const MISS_TTL_MS: Record = { + download: 7 * 24 * 60 * 60 * 1000, + generate: 24 * 60 * 60 * 1000, +} + +/** Leading underscore keeps it out of the manifest scan, matching the + * `_fallback.png` convention in the illustrations dir. */ +const MISSES_FILE = '_misses.json' + export interface GeneratorOptions { pythonBin: string workerScript: string @@ -38,6 +54,7 @@ export interface GeneratorOptions { } const TAG = 'saezuri-generate' +const log = (msg: string) => console.log(`${TAG}: ${msg}`) const logErr = (e: unknown) => console.error(`${TAG}: ${e instanceof Error ? e.message : String(e)}`) @@ -45,16 +62,37 @@ export class Generator { private queued = new Map() private inFlight = new Set() private busy = false + /** Pose file stem -> when that source last failed to supply it. Keyed per pose, not + * per species, so a half-supplied pair backs off only on the pose that's absent; and + * kept per source, because "the repo doesn't have it" says nothing about whether the + * model can draw it. */ + private misses: Record> = { + download: new Map(), + generate: new Map(), + } + private missesLoaded = false + private missesDirty = false constructor(private opts: GeneratorOptions) {} /** Enqueue a species for art acquisition. No-ops when neither art source is * available (no download URL and no Gemini key), already queued, or in flight. */ enqueue(sci: string, com: string): void { - if (!this.opts.enabled && !this.opts.downloadBaseUrl) return const slug = slugify(sci) - if (!slug || this.inFlight.has(slug) || this.queued.has(slug)) return - this.queued.set(slug, { sci, com }) + if (slug) this.request({ slug, sci, com }) + } + + /** Enqueue repairs planned by reconcile.ts. A repair with no `sci` is a cutout + * found on disk whose species we couldn't name: downloadable by slug, but never + * generated — we won't spend a Gemini call on a name we guessed. */ + enqueueRepairs(repairs: readonly ArtRepair[]): void { + for (const r of repairs) this.request(r) + } + + private request(req: ArtRequest): void { + if (!this.opts.enabled && !this.opts.downloadBaseUrl) return + if (this.inFlight.has(req.slug) || this.queued.has(req.slug)) return + this.queued.set(req.slug, req) void this.drain() } @@ -72,23 +110,25 @@ export class Generator { if (this.busy) return this.busy = true try { + await this.loadMisses() while (this.queued.size > 0) { const cap = this.opts.maxPerCycle > 0 ? this.opts.maxPerCycle : this.queued.size const batch = this.take(cap) - for (const s of batch) this.inFlight.add(slugify(s.sci)) + for (const s of batch) this.inFlight.add(s.slug) try { // 1) Free pre-made art first (no key needed). A species is only "done" via // download once BOTH poses are present; a partial pair still needs generation. const needGen: ArtRequest[] = [] let downloadedAny = false for (const s of batch) { - let complete = false - if (this.opts.downloadBaseUrl) { - const { got, complete: c } = await this.downloadArt(slugify(s.sci)) + let complete = this.isComplete(s.slug) + if (!complete && this.opts.downloadBaseUrl) { + const { got, complete: c } = await this.downloadArt(s.slug) if (got > 0) downloadedAny = true complete = c } - if (!complete) needGen.push(s) + // No `sci` means the slug came off disk unnamed — downloadable, never generated. + if (!complete && s.sci && this.worthGenerating(s.slug)) needGen.push(s) } // 2) Generate whatever the repo didn't fully supply, only if a Gemini key is // set. spawnGenerate ends with build_masks, so it rebuilds the manifest; @@ -97,6 +137,7 @@ export class Generator { if (this.opts.enabled && needGen.length > 0) { await this.spawnGenerate(needGen) generated = true + this.recordGenerationGaps(needGen) } // 3) If we only downloaded, rebuild the manifest so the new art is described // (no Gemini key needed). Then republish either way, BEFORE clearing @@ -106,9 +147,12 @@ export class Generator { await this.opts.onGenerated() } } catch (e) { + // A pipeline failure is transient by assumption (rate limit, restart), so + // no miss is recorded and the next sweep retries — mirrors CallProvider. logErr(e) } finally { - for (const s of batch) this.inFlight.delete(slugify(s.sci)) + for (const s of batch) this.inFlight.delete(s.slug) + await this.saveMisses() } } } finally { @@ -116,40 +160,94 @@ export class Generator { } } + private posePath(stem: string): string { + return join(this.opts.assetsDir, `${stem}.png`) + } + + private isComplete(slug: string): boolean { + return POSE_SUFFIXES.every((suffix) => existsSync(this.posePath(`${slug}${suffix}`))) + } + + /** True when this source recently failed to supply this pose. */ + private backedOff(source: MissSource, stem: string): boolean { + const at = this.misses[source].get(stem) + return at !== undefined && Date.now() - at < MISS_TTL_MS[source] + } + + private recordMiss(source: MissSource, stem: string, reason: string): void { + // Log only the transition into backoff, so a permanent gap says its piece once + // rather than on every sweep. + if (!this.misses[source].has(stem)) log(`${stem}.png: ${reason}`) + this.misses[source].set(stem, Date.now()) + this.missesDirty = true + } + + /** The pose landed, so neither source's verdict stands any more. */ + private forgetMisses(stem: string): void { + for (const source of MISS_SOURCES) { + if (this.misses[source].delete(stem)) this.missesDirty = true + } + } + + /** True while at least one absent pose is still worth asking the model for. Without + * this, a species the model keeps declining would be regenerated every publish. */ + private worthGenerating(slug: string): boolean { + return POSE_SUFFIXES.some((suffix) => { + const stem = `${slug}${suffix}` + return !existsSync(this.posePath(stem)) && !this.backedOff('generate', stem) + }) + } + + /** After a generation run, any pose still absent is one the model didn't produce + * (worker.py exits 0 on a partial run so the batch isn't aborted). */ + private recordGenerationGaps(batch: readonly ArtRequest[]): void { + for (const s of batch) { + for (const suffix of POSE_SUFFIXES) { + const stem = `${s.slug}${suffix}` + if (existsSync(this.posePath(stem))) this.forgetMisses(stem) + else this.recordMiss('generate', stem, 'generation produced nothing; retrying in 24h') + } + } + } + /** Download whichever poses the repo has for a slug into assetsDir, skipping poses - * already on disk. Returns how many new files landed (`got`) and whether the species - * is now `complete` (both poses present). A miss/error is non-fatal — the species - * just falls through to generation. Writes atomically (tmp + rename). */ + * already on disk or recently missed. Returns how many new files landed (`got`) and + * whether the species is now `complete` (both poses present). A miss/error is + * non-fatal — the species just falls through to generation. Writes atomically. */ private async downloadArt(slug: string): Promise<{ got: number; complete: boolean }> { await mkdir(this.opts.assetsDir, { recursive: true }) let got = 0 for (const suffix of POSE_SUFFIXES) { - const name = `${slug}${suffix}.png` - const dest = join(this.opts.assetsDir, name) + const stem = `${slug}${suffix}` + const dest = this.posePath(stem) if (existsSync(dest)) continue + if (this.backedOff('download', stem)) continue try { - const res = await fetch(`${this.opts.downloadBaseUrl}/illustrations/${name}`) - if (!res.ok) continue // 404 == not in the repo; silent, expected for many species + const res = await fetch(`${this.opts.downloadBaseUrl}/illustrations/${stem}.png`) + if (!res.ok) { + this.recordMiss('download', stem, 'not in the illustrations repo; retrying in 7d') + continue + } const buf = Buffer.from(await res.arrayBuffer()) const tmp = `${dest}.tmp` await writeFile(tmp, buf) await rename(tmp, dest) + this.forgetMisses(stem) got++ + log(`downloaded ${stem}.png`) } catch (e) { - logErr(e) // network error — non-fatal, fall through to generation + logErr(e) // network error — transient, no miss recorded; fall through to generation } } - const complete = - existsSync(join(this.opts.assetsDir, `${slug}.png`)) && - existsSync(join(this.opts.assetsDir, `${slug}${FLIGHT_SUFFIX}.png`)) - return { got, complete } + return { got, complete: this.isComplete(slug) } } private spawnGenerate(batch: ArtRequest[]): Promise { + log(`generating ${batch.map((s) => s.sci).join(', ')}`) const args = [ this.opts.workerScript, '--generate', - ...batch.map((s) => `${s.sci}|${s.com}`), + ...batch.map((s) => `${s.sci}|${s.com ?? s.sci}`), '--assets-dir', this.opts.assetsDir, '--cache-dir', @@ -171,6 +269,43 @@ export class Generator { ]) } + private async loadMisses(): Promise { + if (this.missesLoaded) return + this.missesLoaded = true + try { + const raw = await readFile(join(this.opts.assetsDir, MISSES_FILE), 'utf8') + const data = JSON.parse(raw) as Partial>> + for (const source of MISS_SOURCES) { + for (const [stem, at] of Object.entries(data[source] ?? {})) { + if (typeof at === 'number') this.misses[source].set(stem, at) + } + } + } catch { + // Absent or unreadable: start empty and rebuild it. Deleting the file is also + // the documented way to force an immediate retry of every known gap. + } + } + + private async saveMisses(): Promise { + if (!this.missesDirty) return + this.missesDirty = false + try { + await mkdir(this.opts.assetsDir, { recursive: true }) + const path = join(this.opts.assetsDir, MISSES_FILE) + const tmp = `${path}.tmp` + await writeFile( + tmp, + JSON.stringify({ + download: Object.fromEntries(this.misses.download), + generate: Object.fromEntries(this.misses.generate), + }), + ) + await rename(tmp, path) + } catch (e) { + logErr(e) // non-fatal: we just re-attempt after a restart + } + } + private spawn(args: string[]): Promise { return new Promise((resolve, reject) => { const child = spawn(this.opts.pythonBin, args, { stdio: 'inherit', env: process.env }) diff --git a/src/server/reconcile.test.ts b/src/server/reconcile.test.ts new file mode 100644 index 0000000..33882b8 --- /dev/null +++ b/src/server/reconcile.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' +import type { LayoutManifest } from '../domain/manifest.ts' +import type { Species } from '../domain/species.ts' +import { planArtRepairs, unpairedSlugs } from './reconcile.ts' + +const MASK = { w: 1, h: 1, bits: '' } + +/** A manifest describing exactly the given keys, as build_masks would from disk. */ +function manifestOf(...keys: string[]): LayoutManifest { + const masks: LayoutManifest['masks'] = { _fallback: MASK } + const dims: LayoutManifest['dims'] = { _fallback: [1, 1] } + for (const k of keys) { + masks[k] = MASK + dims[k] = [1, 1] + } + return { dims, masks, fallbackKey: '_fallback' } +} + +const species = (sci: string, com: string): Species => ({ sci, com, n: 1 }) + +const EMPTY_NAMES = new Map() + +const plan = (input: Partial[0]>) => + planArtRepairs({ + recent: [], + allSpecies: [], + manifest: manifestOf(), + sciBySlug: EMPTY_NAMES, + ...input, + }) + +describe('unpairedSlugs', () => { + it('finds a slug missing either pose and ignores complete pairs and the fallback', () => { + const manifest = manifestOf('turdus-merula', 'pica-pica', 'pica-pica-2', 'grus-grus-2') + expect(unpairedSlugs(manifest).sort()).toEqual(['grus-grus', 'turdus-merula']) + }) +}) + +describe('planArtRepairs', () => { + it('repairs a half-illustrated species heard too long ago to be in either list', () => { + // The reported bug: a pose deleted by hand for a bird outside the 7d store. + expect(plan({ manifest: manifestOf('turdus-merula') })).toEqual([{ slug: 'turdus-merula' }]) + }) + + it('names a disk-only slug from the species dictionary so it can be generated', () => { + const repairs = plan({ + manifest: manifestOf('turdus-merula'), + sciBySlug: new Map([['turdus-merula', { sci: 'Turdus merula', com: 'Eurasian Blackbird' }]]), + }) + expect(repairs).toEqual([ + { slug: 'turdus-merula', sci: 'Turdus merula', com: 'Eurasian Blackbird' }, + ]) + }) + + it('enqueues a species known only from the all-time summary', () => { + expect(plan({ allSpecies: [species('Grus grus', 'Common Crane')] })).toEqual([ + { slug: 'grus-grus', sci: 'Grus grus', com: 'Common Crane' }, + ]) + }) + + it('skips species that already have both poses', () => { + expect( + plan({ + recent: [species('Pica pica', 'Eurasian Magpie')], + manifest: manifestOf('pica-pica', 'pica-pica-2'), + }), + ).toEqual([]) + }) + + it('prefers the named source over the disk scan for the same slug', () => { + const repairs = plan({ + recent: [species('Turdus merula', 'Merel')], + manifest: manifestOf('turdus-merula'), + }) + expect(repairs).toEqual([{ slug: 'turdus-merula', sci: 'Turdus merula', com: 'Merel' }]) + }) +}) diff --git a/src/server/reconcile.ts b/src/server/reconcile.ts new file mode 100644 index 0000000..25180ec --- /dev/null +++ b/src/server/reconcile.ts @@ -0,0 +1,69 @@ +import { FLIGHT_SUFFIX, isComplete } from '../domain/asset.ts' +import type { LayoutManifest } from '../domain/manifest.ts' +import { slugify } from '../domain/slug.ts' +import type { Species } from '../domain/species.ts' + +// Works out which species still owe us art. Pure so the policy is testable +// without a store, a network, or a filesystem. +// +// Three sources, because no single one covers every gap: +// - recently heard: the rolling 7d store, the only source with live counts; +// - all-time known: the summary list, which reaches back past the store's +// 7-day horizon (a bird heard a month ago is invisible to the store); +// - art already on disk: the manifest is rebuilt from the cutout directory, so +// a slug with only one of its two poses is a gap we can see without having +// heard the bird at all. This is what repairs an image deleted by hand. + +export interface ArtRepair { + slug: string + /** Absent when we know the slug but not the species it belongs to; such a + * repair may be downloaded but must never be generated (see planArtRepairs). */ + sci?: string + com?: string +} + +export interface PlanArtRepairsInput { + /** Species heard in the store's window, with live counts. */ + recent: readonly Species[] + /** All-time species summary — reaches past the store's 7-day horizon. */ + allSpecies: readonly Species[] + /** Rebuilt from the cutout directory, so its keys are what's on disk. */ + manifest: LayoutManifest + /** slug -> species, from the published species dictionary. Names a slug found + * on disk whose bird is in neither species list. */ + sciBySlug: ReadonlyMap +} + +/** Every slug in the manifest that has one pose but not the other. */ +export function unpairedSlugs(manifest: LayoutManifest): string[] { + const out = new Set() + for (const key of Object.keys(manifest.masks)) { + if (key === manifest.fallbackKey) continue + const base = key.endsWith(FLIGHT_SUFFIX) ? key.slice(0, -FLIGHT_SUFFIX.length) : key + if (!base) continue + if (!(base in manifest.masks) || !(`${base}${FLIGHT_SUFFIX}` in manifest.masks)) out.add(base) + } + return [...out] +} + +/** The species to hand to the art queue, deduped by slug. Named repairs come + * first so a slug we can name never degrades to a download-only one. */ +export function planArtRepairs(input: PlanArtRepairsInput): ArtRepair[] { + const { recent, allSpecies, manifest, sciBySlug } = input + const out = new Map() + + for (const s of [...recent, ...allSpecies]) { + const slug = slugify(s.sci) + if (!slug || out.has(slug)) continue + if (isComplete(manifest, s.sci)) continue + out.set(slug, { slug, sci: s.sci, com: s.com }) + } + + for (const slug of unpairedSlugs(manifest)) { + if (out.has(slug)) continue + const named = sciBySlug.get(slug) + out.set(slug, named ? { slug, sci: named.sci, com: named.com } : { slug }) + } + + return [...out.values()] +} diff --git a/src/server/refresh.ts b/src/server/refresh.ts index eba34a9..78fa0ce 100644 --- a/src/server/refresh.ts +++ b/src/server/refresh.ts @@ -15,6 +15,7 @@ import { CallLibrary, publishCallManifest } from './calls.ts' import { publishDictionaries } from './dictionaries.ts' import { Generator } from './generate.ts' import { buildSnapshot, loadManifest, writeSnapshot } from './publish.ts' +import { planArtRepairs, unpairedSlugs } from './reconcile.ts' import { frameSignature, renderFrame, writeFrame } from './render.ts' import { DetectionStore } from './store.ts' import { runDetectionStream } from './stream.ts' @@ -142,6 +143,7 @@ class Refresher { private store = new DetectionStore() private manifest: LayoutManifest = DEFAULT_MANIFEST private allSpecies: Species[] = [] + private sciBySlug: ReadonlyMap = new Map() private lastSummaryMs = 0 private pendingPublish?: ReturnType private generator: Generator @@ -191,12 +193,15 @@ class Refresher { /** Download + publish BirdNET-Go's species-name dictionaries as static files so * the browser can localize display names without ever touching BirdNET-Go. */ private async publishDicts(): Promise { - await publishDictionaries({ + const { sciBySlug } = await publishDictionaries({ baseUrl: this.cfg.baseUrl, token: this.cfg.token, htmlDir: this.cfg.htmlDir, locales: this.cfg.dictLocales, }) + // Names the cutouts we find on disk for birds outside the detection window, so + // a hand-deleted pose can be regenerated under the right name (see reconcile.ts). + if (sciBySlug.size > 0) this.sciBySlug = sciBySlug } /** Serialize publishes so overlapping triggers (aging tick, debounce, @@ -246,18 +251,28 @@ class Refresher { await this.renderFrames(snapshot) } - /** Enqueue any recently-heard species missing art or a reference call, so gaps - * fill on startup / after assets are deleted — not only when a species is next - * heard live (onDetection). Cheap + idempotent: both queues dedupe in-flight - * and queued slugs, `isComplete` skips species that already have both poses, - * and the call library short-circuits on files already present or recently - * looked up, so re-running this every publish costs no network. */ + /** Enqueue any species missing art or a reference call, so gaps fill on startup / + * after assets are deleted — not only when a species is next heard live + * (onDetection). Cheap + idempotent: both queues dedupe in-flight and queued slugs, + * the art queue skips species that already have both poses and backs off on gaps it + * has already failed to fill, and the call library short-circuits on files already + * present or recently looked up, so re-running this every publish costs no network. + * + * Art looks wider than the store: a cutout deleted by hand belongs to a bird that + * may not have been heard for weeks, and the store only holds 7 days. Calls stay on + * the store — there is no on-disk gap to notice for a recording we never had. */ private enqueueMissingAssets(now: number): void { const since = (resolveWindow('7D', now) as RangeWindow).sinceMs - for (const s of this.store.aggregate(since)) { - if (!isComplete(this.manifest, s.sci)) this.generator.enqueue(s.sci, s.com) - this.callLibrary.enqueue(s.sci) - } + const recent = this.store.aggregate(since) + this.generator.enqueueRepairs( + planArtRepairs({ + recent, + allSpecies: this.allSpecies, + manifest: this.manifest, + sciBySlug: this.sciBySlug, + }), + ) + for (const s of recent) this.callLibrary.enqueue(s.sci) } /** Rebuild the manifest from disk and republish, so a vanished cutout drops out @@ -320,6 +335,10 @@ class Refresher { `starting; publishing to ${this.cfg.htmlDir} (art source: ${artSource}, call source: ${callSource})`, ) await this.safe('rebuild', () => this.generator.rebuildManifest()) + await this.safe('unpaired', async () => { + const unpaired = unpairedSlugs(await loadManifest(this.cfg.htmlDir)) + if (unpaired.length > 0) log(`${unpaired.length} half-illustrated species on disk`) + }) await this.safe('summary', () => this.refreshSummary()) // Refresh the dictionaries on the slow summary cadence so a backend upgrade is // picked up without a restart.