diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f02c0c..864bea32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,12 @@ Historical entries retain their original delivery coordinates. ## 0.18.66 -Select the European PostHog UI only for the exact validated ingestion origin. +Accept the short legacy system JIDs and inconsistent participant fields emitted by supported wacli migrations when projecting body-free WhatsApp interaction counts. +- Map bounded short system JIDs to the existing unsupported sentinel. +- Drop participant fields that contradict the stored message direction instead of failing the complete page. +- Keep the strict schema, owner, cursor, and no-message-body guards intact. +- Select the European PostHog UI only for the exact validated ingestion origin. - Replace the ambiguous substring match with exact origin equality. - Preserve fixed trusted PostHog UI URLs. diff --git a/scripts/npm-release-workflow.test.ts b/scripts/npm-release-workflow.test.ts index d837f3f8..334a7746 100644 --- a/scripts/npm-release-workflow.test.ts +++ b/scripts/npm-release-workflow.test.ts @@ -1220,7 +1220,7 @@ describe("npm publication contract", () => { (MAX_UNPACKED_BYTES + MAX_PACKED_ENTRIES * 1_023 + 1_024) / 512, ) * 512, ); - expect(MAX_PACKAGE_TAR_BYTES).toBe(24_696_320); + expect(MAX_PACKAGE_TAR_BYTES).toBe(24_698_880); expect(MAX_PACKAGE_TAR_BYTES % 512).toBe(0); expect(artifact).toContain("maxOutputLength: MAX_PACKAGE_TAR_BYTES"); expect(artifact).not.toContain("const maximumTarBytes"); @@ -1430,8 +1430,8 @@ describe("npm publication contract", () => { expect(budget).toContain("785b8fa60c329d7ac46bc8fcf4d959b5fa9d96455bba9e7cdea63f6a3827c4f6"); expect(Object.isFrozen(repairPackageMeasurement)).toBeTrue(); expect(repairPackageMeasurement).toMatchObject({ - archiveSha256: "b462796a666a3f7acccbbf052590f6f7b3db4f956307567dca313c6b13d3393a", - packedBytes: 12_143_254, unpackedBytes: 24_055_132, entryCount: 625, + archiveSha256: "f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc", + packedBytes: 12_144_044, unpackedBytes: 24_057_855, entryCount: 625, packedPlatformProjection: 12_387, packedPortabilityAllowance: 4_096, payloadPlatformProjection: 353, payloadAllowance: 65, }); @@ -1447,8 +1447,8 @@ describe("npm publication contract", () => { expect(budget).toContain("24,024,705 + 353 + 65 = 24,025,123 unpacked"); expect(budget).toContain("12,152,562 + 12,387 + 4,096 = 12,169,045 packed"); expect(budget).toContain("24,093,786 + 353 + 65 = 24,094,204 unpacked"); - expect(MAX_PACKED_BYTES).toBe(12_159_737); - expect(MAX_PACKED_BYTES).toBe(12_143_254 + 12_387 + 4_096); + expect(MAX_PACKED_BYTES).toBe(12_160_527); + expect(MAX_PACKED_BYTES).toBe(12_144_044 + 12_387 + 4_096); expect(budget).toContain("aa127b3193c9bb3b0cb5deece5927be60ccb7111a50169320d322ffdeaa13f39"); expect(budget).toContain("0c331bab3ab3df69a108e18f5f29845b0db90c281cbd6455c0d90fa0b24081e2"); expect(budget).toContain("873cad8139fda303e2d19c6afd61cf549cf9b4d1d76b2a1d6d632a6afe6bd0d1"); @@ -1569,7 +1569,7 @@ describe("npm publication contract", () => { expect(budget).toContain("47684b3e2eb5cf3ed07fbb520aade8c7251d993f75262fbf1af627d9081a1a5f"); expect(budget).toContain("23,688,277 + 353 + 65 = 23,688,695"); expect(budget).toContain("23,759,283 + 353 + 65 = 23,759,701"); - expect(MAX_UNPACKED_BYTES).toBe(24_055_550); + expect(MAX_UNPACKED_BYTES).toBe(24_058_273); expect(budget).toContain("23,037,873 + 65 = 23,037,938"); expect(budget).toContain("f9f3ab38a682690ceaa2699a7309997512030f0fa500a9dc29dcd108123dc41f"); expect(budget).toContain("23,038,557 + 65 = 23,038,622"); @@ -1602,7 +1602,7 @@ describe("npm publication contract", () => { expect(budget).toContain("01875f12ab73a49d6c7d6bf520dc3d318db816addee2fa7981889f35c958cf7c"); expect(budget).toContain("b12909f08f7c19460ced56e30619f4860a1183f4b0106170c07837dae577a937"); expect(budget).toContain("0b212ac291218528dcf979370110a36f10850e046ca90a536057d9a44e807d1d"); - expect(MAX_UNPACKED_BYTES).toBe(24_055_132 + 353 + 65); + expect(MAX_UNPACKED_BYTES).toBe(24_057_855 + 353 + 65); expect(budget).toContain("22,794,052 + 65 = 22,794,117"); expect(budget).toContain("c482efe748f880e3717727d6d39fd92a68953e6eea766642b329ba47ae772d80"); expect(budget).toContain("22,759,423 + 65 = 22,759,488"); @@ -1638,8 +1638,8 @@ describe("npm publication contract", () => { expect(packageArtifactBudget).toEqual({ entryCount: { min: 625, max: 625 }, fileCount: { min: 625, max: 625 }, - packedBytes: { min: 1_600_000, max: 12_159_737 }, - unpackedBytes: { min: 9_000_000, max: 24_055_550 }, + packedBytes: { min: 1_600_000, max: 12_160_527 }, + unpackedBytes: { min: 9_000_000, max: 24_058_273 }, }); }); diff --git a/scripts/package-budget.ts b/scripts/package-budget.ts index 2c997286..6ad6509c 100644 --- a/scripts/package-budget.ts +++ b/scripts/package-budget.ts @@ -2390,24 +2390,22 @@ // Preserve the same projections and allowances: // 12,154,912 + 12,387 + 4,096 = 12,171,395 packed; // 24,054,352 + 353 + 65 = 24,054,770 payload. -// The 0.18.65 editorial release preserves that runtime and file inventory. -// Two byte-identical packs with the same qualified toolchain measured 625 -// entries, 12,155,065 packed and 24,054,930 payload bytes. The 578-byte payload -// increase is the new changelog entry; article and image files are website-only. -// Retain every projection and allowance: -// 12,155,065 + 12,387 + 4,096 = 12,171,548 packed; -// 24,054,930 + 353 + 65 = 24,055,348 payload. -// The 0.18.66 analytics-host release measures 625 entries under the same -// Node 24.20.0 / npm 11.19.0 / darwin-arm64 toolchain. Two packs are identical. -// Preserve the reviewed platform projections and portability allowances. +// The 0.18.66 WhatsApp projection and analytics-host repair release retains +// the 625-entry inventory after rebuilding dist. Two byte-identical packs with +// the same qualified toolchain measured 12,144,044 packed and 24,057,855 +// payload bytes; archive SHA-256 +// f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc. +// Retain the reviewed projections and allowances: +// 12,144,044 + 12,387 + 4,096 = 12,160,527 packed; +// 24,057,855 + 353 + 65 = 24,058,273 payload. export const repairPackageMeasurement = Object.freeze({ - scope: "GhostGet 0.18.66 exact analytics host release over the verified 0.18.65 runtime", + scope: "GhostGet 0.18.66 WhatsApp projection and analytics-host repair release over the verified 0.18.64 runtime", command: "npm pack --ignore-scripts", npmVersion: "11.19.0", platform: "darwin-arm64", - archiveSha256: "b462796a666a3f7acccbbf052590f6f7b3db4f956307567dca313c6b13d3393a", - packedBytes: 12_143_254, - unpackedBytes: 24_055_132, + archiveSha256: "f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc", + packedBytes: 12_144_044, + unpackedBytes: 24_057_855, entryCount: 625, packedPlatformProjection: 12_387, packedPortabilityAllowance: 4_096, diff --git a/src/providers/whatsapp-interaction-projection-helper.test.ts b/src/providers/whatsapp-interaction-projection-helper.test.ts index 557fc437..4e3a1e0e 100644 --- a/src/providers/whatsapp-interaction-projection-helper.test.ts +++ b/src/providers/whatsapp-interaction-projection-helper.test.ts @@ -446,6 +446,141 @@ describe("WhatsApp content-free interaction projection helper", () => { } }); + test("maps the short legacy system JID to the fixed unsupported sentinel", async () => { + const path = createStore(); + try { + const database = new Database(join(path, "wacli.db"), { strict: true }); + try { + database.query("INSERT INTO chats(jid, kind) VALUES (?1, ?2)") + .run("1@s.whatsapp.net", "dm"); + database.query(` + INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me) + VALUES (?1,?2,NULL,?3,?4) + `).run("1@s.whatsapp.net", "LEGACY-SYSTEM-1", 1_776_513_603, 0); + } finally { + database.close(); + } + chmodSync(join(path, "wacli.db"), 0o600); + expect(await response(path, request(path, { + cursor: "3", + cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830", + limit: 1, + }))).toMatchObject({ + status: "succeeded", + interactions: [{ rowid: "4", chatJid: "0@s.whatsapp.net", chatKind: "unknown" }], + }); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + + test("normalizes only known legacy participant fields and retains valid senders", async () => { + const path = createStore(); + try { + const database = new Database(join(path, "wacli.db"), { strict: true }); + try { + database.query(` + INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me) + VALUES + ('15557654321@s.whatsapp.net','LEGACY-DM-OUT','15557654321@s.whatsapp.net',1776513603,1), + ('120363123456789012@g.us','LEGACY-GROUP-OUT','120363123456789012@g.us',1776513604,1), + ('120363123456789012@g.us','LEGACY-GROUP-IN-OWNER',?1,1776513605,0), + ('15557654321@s.whatsapp.net','LEGACY-DM-OUT-DEVICE','15557654321:7@s.whatsapp.net',1776513606,1), + ('120363123456789012@g.us','LEGACY-GROUP-IN-LID','999999999999999:4@lid',1776513607,0), + ('120363123456789012@g.us','VALID-GROUP-IN','15559876543:5@s.whatsapp.net',1776513608,0), + ('15557654321@s.whatsapp.net','VALID-DM-OUT',?1,1776513609,1), + ('120363123456789012@g.us','VALID-GROUP-OUT-LID','999999999999999:4@lid',1776513610,1) + `).run(OWNER_JID); + } finally { + database.close(); + } + chmodSync(join(path, "wacli.db"), 0o600); + expect(await response(path, request(path, { + cursor: "3", + cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830", + limit: 8, + }))).toMatchObject({ + status: "succeeded", + interactions: [ + { rowid: "4", messageId: "LEGACY-DM-OUT", senderJid: null }, + { rowid: "5", messageId: "LEGACY-GROUP-OUT", senderJid: null }, + { rowid: "6", messageId: "LEGACY-GROUP-IN-OWNER", senderJid: null }, + { rowid: "7", messageId: "LEGACY-DM-OUT-DEVICE", senderJid: null }, + { rowid: "8", messageId: "LEGACY-GROUP-IN-LID", senderJid: null }, + { rowid: "9", messageId: "VALID-GROUP-IN", senderJid: "15559876543:5@s.whatsapp.net" }, + { rowid: "10", messageId: "VALID-DM-OUT", senderJid: OWNER_JID }, + { rowid: "11", messageId: "VALID-GROUP-OUT-LID", senderJid: "999999999999999:4@lid" }, + ], + }); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + + for (const invalid of [ + { label: "outgoing DM participant", chat: "15557654321@s.whatsapp.net", sender: "15559876543@s.whatsapp.net", fromMe: 1 }, + { label: "incoming DM participant", chat: "15557654321@s.whatsapp.net", sender: "15559876543@s.whatsapp.net", fromMe: 0 }, + { label: "outgoing group participant", chat: "120363123456789012@g.us", sender: "15559876543@s.whatsapp.net", fromMe: 1 }, + { label: "outgoing group chat", chat: "120363123456789012@g.us", sender: "120363987654321098@g.us", fromMe: 1 }, + { label: "incoming group chat", chat: "120363123456789012@g.us", sender: "120363987654321098@g.us", fromMe: 0 }, + { label: "incoming group same-chat sender", chat: "120363123456789012@g.us", sender: "120363123456789012@g.us", fromMe: 0 }, + ] as const) { + test(`rejects unrelated or contradictory legacy ${invalid.label}`, async () => { + const path = createStore(); + try { + const database = new Database(join(path, "wacli.db"), { strict: true }); + try { + database.query(` + INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me) + VALUES (?1,'INVALID-LEGACY-SENDER',?2,1776513603,?3) + `).run(invalid.chat, invalid.sender, invalid.fromMe); + } finally { + database.close(); + } + chmodSync(join(path, "wacli.db"), 0o600); + expect(await response(path, request(path, { + cursor: "3", + cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830", + limit: 1, + }))).toMatchObject({ status: "failed", errorCode: "projection-invalid" }); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }); + } + + test("rejects unrelated outgoing participant IDs across PN, LID, and device aliases", async () => { + await assertAsyncProperty(fc.asyncProperty( + fc.integer({ min: 10_000, max: 9_999_999 }), + fc.constantFrom("s.whatsapp.net", "lid"), + fc.option(fc.integer({ min: 0, max: 99_999 }), { nil: undefined }), + fc.constantFrom("15557654321@s.whatsapp.net", "120363123456789012@g.us"), + async (participant, server, device, chat) => { + const path = createStore(); + try { + const sender = `${participant}${device === undefined ? "" : `:${device}`}@${server}`; + const database = new Database(join(path, "wacli.db"), { strict: true }); + try { + database.query(` + INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me) + VALUES (?1,'INVALID-LEGACY-PROPERTY',?2,1776513603,1) + `).run(chat, sender); + } finally { + database.close(); + } + chmodSync(join(path, "wacli.db"), 0o600); + expect(await response(path, request(path, { + cursor: "3", + cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830", + limit: 1, + }))).toMatchObject({ status: "failed", errorCode: "projection-invalid" }); + } finally { + rmSync(path, { recursive: true, force: true }); + } + }, + ), { numRuns: 20 }); + }); + test("preserves rowids above JavaScript's safe-integer range", async () => { const path = createStore(); try { diff --git a/src/providers/whatsapp-interaction-projection-helper.ts b/src/providers/whatsapp-interaction-projection-helper.ts index 3b3918c6..200a6281 100644 --- a/src/providers/whatsapp-interaction-projection-helper.ts +++ b/src/providers/whatsapp-interaction-projection-helper.ts @@ -62,6 +62,11 @@ const MAX_MESSAGE_DATABASE_BYTES = 2n * 1024n * 1024n * 1024n; const SQLITE_CACHE_KIB = 4_096; const MAX_UNIX_SECONDS = 253_402_300_799n; const SYSTEM_SENTINEL_JID = "0@s.whatsapp.net"; +// Older wacli stores use short numeric JIDs for system rows. They are not +// person or conversation identifiers and must stay outside the contact +// interaction projection. Keep the public projection's single sentinel +// while accepting the bounded legacy spelling. +const LEGACY_SYSTEM_JID = /^[0-9]@s\.whatsapp\.net$/u; const MESSAGE_EXPORT_SESSION_MAX_MESSAGES = 500_000; type MessageExportSessionSelfChatsExcluded = "none-detected" | "present-excluded"; @@ -505,7 +510,7 @@ function sqliteBigInt(value: unknown): bigint { return fail("projection-invalid"); } -function projectedJid(value: unknown, nullable = false): string | null { +function projectedJid(value: unknown, nullable = false, allowLegacySystem = false): string | null { if (nullable && (value === null || value === "")) return null; if ( typeof value !== "string" @@ -513,6 +518,7 @@ function projectedJid(value: unknown, nullable = false): string | null { || value.length > 96 || !( value === SYSTEM_SENTINEL_JID + || (allowLegacySystem && typeof value === "string" && LEGACY_SYSTEM_JID.test(value)) || /^[0-9]{5,20}(?::[0-9]{1,5})?@s\.whatsapp\.net$/u.test(value) || /^[0-9]{5,32}(?::[0-9]{1,5})?@lid$/u.test(value) || /^[0-9]{5,32}(?:-[0-9]{5,20})?@g\.us$/u.test(value) @@ -532,6 +538,43 @@ function legacyInteractionParticipantJid(value: string): string { : `${match[1]}@s.whatsapp.net`; } +function legacyInteractionSender( + item: Readonly<{ + chatJid: string; + chatKind: "dm" | "group" | "broadcast" | "newsletter" | "unknown"; + senderJid: string | null; + fromMe: boolean; + }>, + owner: WhatsAppMatchedOwnerIdentity, +): string | null { + if (item.senderJid === null || (item.chatKind !== "dm" && item.chatKind !== "group")) { + return item.senderJid; + } + if (item.chatKind === "group" && item.senderJid.endsWith("@g.us")) { + // Some outgoing migrated rows carry this exact chat JID in sender_jid. + // Other group JIDs and incoming chat senders remain invalid participants. + return item.fromMe && item.senderJid === item.chatJid ? null : item.senderJid; + } + const sender = legacyInteractionParticipantJid(item.senderJid); + const self = new Set(owner.selfJids); + const chat = item.chatKind === "dm" + ? legacyInteractionParticipantJid(item.chatJid) + : item.chatJid; + if (item.chatKind === "dm" && item.fromMe && sender === chat) { + // Wacli records the recipient in sender_jid for some outgoing direct + // messages. The direction already carries the only fact PeopleBlade + // needs; dropping this inconsistent participant keeps the interaction + // body-free and prevents a false sender identity. + return null; + } + if (item.chatKind === "group" && !item.fromMe && self.has(sender)) { + // Incoming migrated rows may carry an exact session-proved owner alias. + // It identifies no counterparty; unrelated outgoing senders remain invalid. + return null; + } + return item.senderJid; +} + function assertMessageDirection( item: Pick, owner: WhatsAppMatchedOwnerIdentity, @@ -567,12 +610,12 @@ function projectItem( exactRowKeys(row, ["rowid", "chat_jid", "msg_id", "sender_jid", "ts", "from_me", "chat_kind"]); const rowid = sqliteBigInt(row.rowid); const seconds = sqliteBigInt(row.ts); - const fromMe = sqliteBigInt(row.from_me); + const fromMeValue = sqliteBigInt(row.from_me); if ( rowid < 1n || seconds < 0n || seconds > MAX_UNIX_SECONDS - || (fromMe !== 0n && fromMe !== 1n) + || (fromMeValue !== 0n && fromMeValue !== 1n) || typeof row.msg_id !== "string" || !/^[A-Za-z0-9._~:-]{1,256}$/u.test(row.msg_id) || (row.chat_kind !== "dm" @@ -581,14 +624,21 @@ function projectItem( && row.chat_kind !== "newsletter" && row.chat_kind !== "unknown") ) fail("projection-invalid"); - const chatJid = projectedJid(row.chat_jid)! as string; + const rawChatJid = projectedJid(row.chat_jid, false, true)! as string; + const chatJid = LEGACY_SYSTEM_JID.test(rawChatJid) ? SYSTEM_SENTINEL_JID : rawChatJid; + const fromMe = fromMeValue === 1n; const projected = Object.freeze({ rowid: rowid.toString(), chatJid, messageId: row.msg_id, - senderJid: projectedJid(row.sender_jid, true), + senderJid: legacyInteractionSender({ + chatJid, + chatKind: chatJid === SYSTEM_SENTINEL_JID ? "unknown" : row.chat_kind, + senderJid: projectedJid(row.sender_jid, true), + fromMe, + }, owner), timestamp: new Date(Number(seconds) * 1_000).toISOString(), - fromMe: fromMe === 1n, + fromMe, chatKind: chatJid === SYSTEM_SENTINEL_JID ? "unknown" : row.chat_kind, }); assertMessageDirection(projected, owner, "legacy-interaction");