fix: always regenerate missing images

README.md

@@ -205,8 +205,15 @@ How it behaves:
 - **Offline-safe / disable.** A failed fetch is non-fatal (silhouette until art exists). Set
   `ILLUSTRATIONS_REPO=` (empty) to turn downloading off entirely; pin `ILLUSTRATIONS_REF` to a
   release tag instead of `main` for a fixed art set.
+- **Gaps it can't fill are remembered, not retried forever.** A cutout the repo doesn't have (or
+  that generation declined) is logged once and left alone for a while — a week for a repo miss, a
+  day for a generation miss — instead of being re-requested on every refresh. The record lives in
+  `_misses.json` in the illustrations volume; delete it to retry everything immediately.
 - **Self-healing.** Delete a cutout from the volume and the service notices, re-downloads it (or
   regenerates it), and rebuilds the manifest — which is also how you replace art you don't like.
+  This works for a single pose of a pair, and for a bird that hasn't been heard in weeks: every
+  refresh compares what's on disk against what should be there and repairs the difference, at
+  startup as well as while running.
 - **Licensing.** The illustrations (and the generation pipeline the image bundles) are
   **CC-BY-NC-SA-4.0** — non-commercial. See the illustrations repo and *Credits and licensing* below.
 

src/server/dictionaries.test.ts

@@ -27,7 +27,7 @@ describe('publishDictionaries', () => {
       return Promise.resolve(notFound())
     })
 
-    const index = await publishDictionaries({
+    const { index } = await publishDictionaries({
       baseUrl: 'http://bng:8080',
       htmlDir,
       locales: ['de', 'nl'],
@@ -46,7 +46,7 @@ describe('publishDictionaries', () => {
       Promise.resolve(url.endsWith('/species/dictionary/de') ? ok({ x: 'y' }) : notFound()),
     )
 
-    const index = await publishDictionaries({
+    const { index } = await publishDictionaries({
       baseUrl: 'http://bng:8080',
       htmlDir,
       locales: ['de', 'nl'],
@@ -69,4 +69,26 @@ describe('publishDictionaries', () => {
     const [, opts] = fetchMock.mock.calls[0]
     expect(opts.headers.Authorization).toBe('Bearer secret')
   })
+
+  it('indexes slugs from the English dictionary so disk-only slugs can be named', async () => {
+    const htmlDir = await tmpHtmlDir()
+    fetchMock.mockImplementation((url: string) => {
+      if (url.endsWith('/species/dictionary/en'))
+        return Promise.resolve(ok({ 'Turdus merula': 'Eurasian Blackbird' }))
+      if (url.endsWith('/species/dictionary/nl'))
+        return Promise.resolve(ok({ 'Turdus merula': 'Merel' }))
+      return Promise.resolve(notFound())
+    })
+
+    const { sciBySlug } = await publishDictionaries({
+      baseUrl: 'http://bng:8080',
+      htmlDir,
+      locales: ['nl', 'en'],
+    })
+
+    expect(sciBySlug.get('turdus-merula')).toEqual({
+      sci: 'Turdus merula',
+      com: 'Eurasian Blackbird',
+    })
+  })
 })

src/server/dictionaries.ts

@@ -2,6 +2,7 @@ import { mkdir, rename, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import type { DictionaryIndex } from '../domain/dictionary.ts'
 import { type DictLocale, reduceToDictLocale, SUPPORTED_DICT_LOCALES } from '../domain/locale.ts'
+import { slugify } from '../domain/slug.ts'
 import { birdnetFetch } from './birdnet.ts'
 
 // The refresh service downloads BirdNET-Go's species-name dictionaries and
@@ -42,18 +43,46 @@ async function fetchStationDefault(baseUrl: string, token?: string): Promise<Dic
   }
 }
 
+export interface PublishedDictionaries {
+  index: DictionaryIndex
+  /** slug -> species name, over every species BirdNET-Go knows — not just the ones
+   *  this station has heard. Lets the refresh service name a cutout it finds on disk
+   *  whose bird predates the detection window, so a deleted pose can be regenerated
+   *  under the right name instead of a guess reversed out of the slug. */
+  sciBySlug: ReadonlyMap<string, { sci: string; com: string }>
+}
+
+/** English if it was published, else whatever was — the keys (scientific names) are
+ *  the same in every locale, only the display name differs. */
+function buildSciBySlug(
+  published: readonly DictLocale[],
+  maps: ReadonlyMap<DictLocale, Record<string, string>>,
+): ReadonlyMap<string, { sci: string; com: string }> {
+  const locale = published.includes('en') ? 'en' : published[0]
+  const map = locale ? maps.get(locale) : undefined
+  const out = new Map<string, { sci: string; com: string }>()
+  if (!map) return out
+  for (const [sci, com] of Object.entries(map)) {
+    const slug = slugify(sci)
+    if (slug) out.set(slug, { sci, com: com || sci })
+  }
+  return out
+}
+
 /** Download each requested species dictionary from BirdNET-Go and publish it as a
  *  static file under <htmlDir>/species-dict/, plus an index.json listing what was
  *  published and the station's default language. Locales the backend lacks (404) or
  *  that fail are skipped, so an older build simply yields an empty index and the
- *  browser falls back to the station's own names. Returns the published index. */
+ *  browser falls back to the station's own names. Returns the published index plus
+ *  a slug index (see buildSciBySlug). */
 export async function publishDictionaries(
   cfg: PublishDictionariesConfig,
-): Promise<DictionaryIndex> {
+): Promise<PublishedDictionaries> {
   const dir = join(cfg.htmlDir, 'species-dict')
   await mkdir(dir, { recursive: true })
   const wanted = cfg.locales ?? SUPPORTED_DICT_LOCALES
   const published: DictLocale[] = []
+  const maps = new Map<DictLocale, Record<string, string>>()
 
   for (const locale of wanted) {
     try {
@@ -62,6 +91,7 @@ export async function publishDictionaries(
       const map = (await res.json()) as Record<string, string>
       await writeJson(join(dir, `${locale}.json`), map)
       published.push(locale)
+      maps.set(locale, map)
     } catch (e) {
       log(`skip ${locale}: ${e instanceof Error ? e.message : String(e)}`)
     }
@@ -75,5 +105,5 @@ export async function publishDictionaries(
   log(
     `published ${published.length}/${wanted.length} dictionaries (default: ${index.default ?? 'none'})`,
   )
-  return index
+  return { index, sciBySlug: buildSciBySlug(published, maps) }
 }

src/server/generate.test.ts

@@ -1,4 +1,4 @@
-import { mkdtemp, readdir, rm } from 'node:fs/promises'
+import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
@@ -34,14 +34,19 @@ const BASE = 'http://fixtures.test'
 let assetsDir: string
 let onGenerated: Mock<() => Promise<void>>
 
+// Pose stems requested since the last stubFetch, in order.
+const fetched: string[] = []
+
 // fetch stub: 200 for exactly the listed pose filenames (e.g. 'turdus-merula' for the
 // perched pose, 'turdus-merula-2' for flight), 404 for everything else.
 function stubFetch(presentPoses: string[]) {
+  fetched.length = 0
   vi.stubGlobal(
     'fetch',
     vi.fn(async (url: string) => {
       const m = url.match(/\/illustrations\/(.+)\.png$/)
       const name = m?.[1]
+      if (name) fetched.push(name)
       if (name && presentPoses.includes(name)) {
         return { ok: true, arrayBuffer: async () => PNG.buffer.slice(0) } as unknown as Response
       }
@@ -135,6 +140,78 @@ describe('Generator art acquisition', () => {
     expect(onGenerated).toHaveBeenCalledTimes(1)
   })
 
+  it('re-downloads only the pose deleted from an existing pair', async () => {
+    // The reported bug's happy path: one image removed by hand, the other left alone.
+    stubFetch(['turdus-merula', 'turdus-merula-2'])
+    await writeFile(join(assetsDir, 'turdus-merula.png'), PNG)
+    const g = makeGen({ enabled: true, downloadBaseUrl: BASE })
+    g.enqueueRepairs([{ slug: 'turdus-merula', sci: 'Turdus merula', com: 'Eurasian Blackbird' }])
+    await idle(g)
+
+    expect(fetched).toEqual(['turdus-merula-2'])
+    expect(await readdir(assetsDir)).toContain('turdus-merula-2.png')
+    expect(spawnCalls.some((a) => a.includes('--generate'))).toBe(false)
+  })
+
+  it('downloads but never generates a repair whose species we cannot name', async () => {
+    stubFetch([]) // repo has nothing, and we have no scientific name to prompt with
+    const g = makeGen({ enabled: true, downloadBaseUrl: BASE })
+    g.enqueueRepairs([{ slug: 'turdus-merula' }])
+    await idle(g)
+
+    expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2'])
+    expect(spawnCalls.some((a) => a.includes('--generate'))).toBe(false)
+  })
+
+  it('remembers a repo miss and stops re-requesting it, across restarts', async () => {
+    stubFetch([])
+    const g = makeGen({ enabled: false, downloadBaseUrl: BASE })
+    g.enqueue('Turdus merula', 'Eurasian Blackbird')
+    await idle(g)
+    expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2'])
+
+    const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8'))
+    expect(Object.keys(misses.download).sort()).toEqual(['turdus-merula', 'turdus-merula-2'])
+
+    fetched.length = 0
+    const restarted = makeGen({ enabled: false, downloadBaseUrl: BASE })
+    restarted.enqueue('Turdus merula', 'Eurasian Blackbird')
+    await idle(restarted)
+    expect(fetched).toEqual([])
+  })
+
+  it('retries a repo miss once its backoff has expired', async () => {
+    const stale = Date.now() - 8 * 24 * 60 * 60 * 1000 // older than the 7d download TTL
+    await writeFile(
+      join(assetsDir, '_misses.json'),
+      JSON.stringify({ download: { 'turdus-merula': stale, 'turdus-merula-2': stale } }),
+    )
+    stubFetch(['turdus-merula', 'turdus-merula-2'])
+    const g = makeGen({ enabled: false, downloadBaseUrl: BASE })
+    g.enqueue('Turdus merula', 'Eurasian Blackbird')
+    await idle(g)
+
+    expect(fetched).toEqual(['turdus-merula', 'turdus-merula-2'])
+    expect(await readdir(assetsDir)).toContain('turdus-merula-2.png')
+    const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8'))
+    expect(misses.download).toEqual({}) // both poses landed
+  })
+
+  it('backs off a species the model declines rather than regenerating every sweep', async () => {
+    stubFetch([]) // repo has nothing; the mocked worker exits 0 without writing anything
+    const g = makeGen({ enabled: true, downloadBaseUrl: BASE })
+    g.enqueue('Turdus merula', 'Eurasian Blackbird')
+    await idle(g)
+    expect(spawnCalls.filter((a) => a.includes('--generate'))).toHaveLength(1)
+
+    g.enqueue('Turdus merula', 'Eurasian Blackbird')
+    await idle(g)
+    expect(spawnCalls.filter((a) => a.includes('--generate'))).toHaveLength(1)
+
+    const misses = JSON.parse(await readFile(join(assetsDir, '_misses.json'), 'utf8'))
+    expect(Object.keys(misses.generate).sort()).toEqual(['turdus-merula', 'turdus-merula-2'])
+  })
+
   it('no-ops enqueue when both sources are disabled', async () => {
     stubFetch(['turdus-merula'])
     const g = makeGen({ enabled: false, downloadBaseUrl: '' })