diff --git a/scripts/cacbg/fetch.mjs b/scripts/cacbg/fetch.mjs
index f947d94a..a41a7530 100644
--- a/scripts/cacbg/fetch.mjs
+++ b/scripts/cacbg/fetch.mjs
@@ -68,6 +68,10 @@ export function assessCompleteness(perFolder, skippedFolders = []) {
sourceGaps += f.missing; // 404 — listed but unpublished at source (expected, not a shortfall)
unfetched += f.errors; // announced but not obtained for a non-404 reason — a real shortfall
}
+ // Every announced row ends in exactly one bucket, so a full pass satisfies
+ // announced == obtained + sourceGaps + unfetched. A surplus means rows were never ATTEMPTED — the
+ // --limit case, which produces no errors and would otherwise sail through the checks below.
+ const notAttempted = Math.max(0, announcedDeclarations - (obtained + sourceGaps + unfetched));
return {
reachedSets: Object.keys(perFolder).length,
skippedSets: skippedFolders.length,
@@ -75,7 +79,8 @@ export function assessCompleteness(perFolder, skippedFolders = []) {
obtained,
sourceGaps,
unfetched,
- incomplete: unfetched > 0 || skippedFolders.length > 0,
+ notAttempted,
+ incomplete: unfetched > 0 || notAttempted > 0 || skippedFolders.length > 0,
};
}
@@ -178,8 +183,12 @@ export async function run({
}
atomicWrite(path.join(dir, 'list.xml'), listRes.body); // cache list for extract.mjs
let rows = parseList(listRes.body.toString('utf8'));
+ // `announced` is what the SET declares, so it is read BEFORE --limit truncates the work. Taking it
+ // after the slice made a deliberately partial crawl report announced == obtained, i.e. the completeness
+ // gate certified a corpus it had never attempted to fetch (ydimitrof #226).
+ const announced = rows.length;
if (Number.isFinite(limit)) rows = rows.slice(0, limit);
- const fstat = { announced: rows.length, fetched: 0, cached: 0, missing: 0, errors: 0 };
+ const fstat = { announced, fetched: 0, cached: 0, missing: 0, errors: 0 };
stats.folders[folder] = fstat;
console.log(` ${folder}: ${rows.length} declarations`);
@@ -240,6 +249,7 @@ export async function run({
stats.skippedFolders.map((s) => `${s.folder} (${s.status})`).join(', ') || 'none';
const msg =
`INCOMPLETE CORPUS — ${completeness.unfetched} announced declaration(s) unfetched (non-404), ` +
+ `${completeness.notAttempted} never attempted (--limit), ` +
`${completeness.skippedSets} set(s) skipped: ${skipped}. Publishing this would omit declarations ` +
`the register lists.`;
if (allowIncomplete) {
diff --git a/scripts/cacbg/fetch.test.mjs b/scripts/cacbg/fetch.test.mjs
index 7e0150f2..8227bdf4 100644
--- a/scripts/cacbg/fetch.test.mjs
+++ b/scripts/cacbg/fetch.test.mjs
@@ -106,3 +106,25 @@ test('assessCompleteness: an empty crawl of fully-obtained sets is complete', ()
assert.equal(r.announcedDeclarations, 2);
assert.equal(r.incomplete, false);
});
+
+// --limit truncates the WORK, never the announcement. Before this, `announced` was read after the slice,
+// so a deliberately partial crawl reported announced == obtained and the gate certified a corpus it had
+// never tried to fetch. Rows that were never attempted produce no errors, so `incomplete` has to notice
+// the arithmetic hole itself: announced > obtained + sourceGaps + unfetched.
+test('assessCompleteness: rows announced but never attempted (--limit) mark the corpus INCOMPLETE', () => {
+ const r = assessCompleteness(
+ { 2024: { announced: 5000, fetched: 10, cached: 0, missing: 0, errors: 0 } },
+ [],
+ );
+ assert.equal(r.notAttempted, 4990);
+ assert.equal(r.unfetched, 0, 'no fetch was even tried, so nothing can have errored');
+ assert.equal(r.incomplete, true);
+});
+test('assessCompleteness: notAttempted is 0 when every announced row landed in a bucket', () => {
+ const r = assessCompleteness(
+ { 2024: { announced: 10, fetched: 6, cached: 3, missing: 1, errors: 0 } },
+ [],
+ );
+ assert.equal(r.notAttempted, 0);
+ assert.equal(r.incomplete, false);
+});
diff --git a/scripts/cacbg/guard.mjs b/scripts/cacbg/guard.mjs
index 8ff66e44..97a326bd 100644
--- a/scripts/cacbg/guard.mjs
+++ b/scripts/cacbg/guard.mjs
@@ -22,12 +22,26 @@ export function assertScratchIgnored() {
// Path-sanitize an xml_file / year from the untrusted list.xml before using it in a filesystem path
// or URL. Rejects traversal, absolute paths, and anything outside the expected shape.
+// The shape of a real declaration filename. Single source of truth for both the throwing guard below
+// (used on the fetch path) and the boolean twin (used by parseList to tell a real row from a phantom).
+const XML_FILE_SHAPE = /^[A-Za-z0-9._-]+\.xml$/;
+
export function safeXmlFile(name) {
const base = path.basename(String(name));
- if (!/^[A-Za-z0-9._-]+\.xml$/.test(base)) throw new Error(`unsafe xmlFile: ${name}`);
+ if (!XML_FILE_SHAPE.test(base)) throw new Error(`unsafe xmlFile: ${name}`);
return base;
}
+/**
+ * Does this value name a declaration file at all? The boolean twin of safeXmlFile — same shape, no throw.
+ * parseList needs the question answered without an exception because a non-answer there is not an error:
+ * the register's list.xml carries placeholder rows (`