Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@ Historical entries retain their original delivery coordinates.

## 0.18.66

Select the European PostHog UI only for the exact validated ingestion origin.
Accept the short legacy system JIDs and inconsistent participant fields emitted by supported wacli migrations when projecting body-free WhatsApp interaction counts.

- Map bounded short system JIDs to the existing unsupported sentinel.
- Drop participant fields that contradict the stored message direction instead of failing the complete page.
- Keep the strict schema, owner, cursor, and no-message-body guards intact.
- Select the European PostHog UI only for the exact validated ingestion origin.
- Replace the ambiguous substring match with exact origin equality.
- Preserve fixed trusted PostHog UI URLs.

Expand Down
18 changes: 9 additions & 9 deletions scripts/npm-release-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1220,7 +1220,7 @@ describe("npm publication contract", () => {
(MAX_UNPACKED_BYTES + MAX_PACKED_ENTRIES * 1_023 + 1_024) / 512,
) * 512,
);
expect(MAX_PACKAGE_TAR_BYTES).toBe(24_696_320);
expect(MAX_PACKAGE_TAR_BYTES).toBe(24_698_880);
expect(MAX_PACKAGE_TAR_BYTES % 512).toBe(0);
expect(artifact).toContain("maxOutputLength: MAX_PACKAGE_TAR_BYTES");
expect(artifact).not.toContain("const maximumTarBytes");
Expand Down Expand Up @@ -1430,8 +1430,8 @@ describe("npm publication contract", () => {
expect(budget).toContain("785b8fa60c329d7ac46bc8fcf4d959b5fa9d96455bba9e7cdea63f6a3827c4f6");
expect(Object.isFrozen(repairPackageMeasurement)).toBeTrue();
expect(repairPackageMeasurement).toMatchObject({
archiveSha256: "b462796a666a3f7acccbbf052590f6f7b3db4f956307567dca313c6b13d3393a",
packedBytes: 12_143_254, unpackedBytes: 24_055_132, entryCount: 625,
archiveSha256: "f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc",
packedBytes: 12_144_044, unpackedBytes: 24_057_855, entryCount: 625,
packedPlatformProjection: 12_387, packedPortabilityAllowance: 4_096,
payloadPlatformProjection: 353, payloadAllowance: 65,
});
Expand All @@ -1447,8 +1447,8 @@ describe("npm publication contract", () => {
expect(budget).toContain("24,024,705 + 353 + 65 = 24,025,123 unpacked");
expect(budget).toContain("12,152,562 + 12,387 + 4,096 = 12,169,045 packed");
expect(budget).toContain("24,093,786 + 353 + 65 = 24,094,204 unpacked");
expect(MAX_PACKED_BYTES).toBe(12_159_737);
expect(MAX_PACKED_BYTES).toBe(12_143_254 + 12_387 + 4_096);
expect(MAX_PACKED_BYTES).toBe(12_160_527);
expect(MAX_PACKED_BYTES).toBe(12_144_044 + 12_387 + 4_096);
expect(budget).toContain("aa127b3193c9bb3b0cb5deece5927be60ccb7111a50169320d322ffdeaa13f39");
expect(budget).toContain("0c331bab3ab3df69a108e18f5f29845b0db90c281cbd6455c0d90fa0b24081e2");
expect(budget).toContain("873cad8139fda303e2d19c6afd61cf549cf9b4d1d76b2a1d6d632a6afe6bd0d1");
Expand Down Expand Up @@ -1569,7 +1569,7 @@ describe("npm publication contract", () => {
expect(budget).toContain("47684b3e2eb5cf3ed07fbb520aade8c7251d993f75262fbf1af627d9081a1a5f");
expect(budget).toContain("23,688,277 + 353 + 65 = 23,688,695");
expect(budget).toContain("23,759,283 + 353 + 65 = 23,759,701");
expect(MAX_UNPACKED_BYTES).toBe(24_055_550);
expect(MAX_UNPACKED_BYTES).toBe(24_058_273);
expect(budget).toContain("23,037,873 + 65 = 23,037,938");
expect(budget).toContain("f9f3ab38a682690ceaa2699a7309997512030f0fa500a9dc29dcd108123dc41f");
expect(budget).toContain("23,038,557 + 65 = 23,038,622");
Expand Down Expand Up @@ -1602,7 +1602,7 @@ describe("npm publication contract", () => {
expect(budget).toContain("01875f12ab73a49d6c7d6bf520dc3d318db816addee2fa7981889f35c958cf7c");
expect(budget).toContain("b12909f08f7c19460ced56e30619f4860a1183f4b0106170c07837dae577a937");
expect(budget).toContain("0b212ac291218528dcf979370110a36f10850e046ca90a536057d9a44e807d1d");
expect(MAX_UNPACKED_BYTES).toBe(24_055_132 + 353 + 65);
expect(MAX_UNPACKED_BYTES).toBe(24_057_855 + 353 + 65);
expect(budget).toContain("22,794,052 + 65 = 22,794,117");
expect(budget).toContain("c482efe748f880e3717727d6d39fd92a68953e6eea766642b329ba47ae772d80");
expect(budget).toContain("22,759,423 + 65 = 22,759,488");
Expand Down Expand Up @@ -1638,8 +1638,8 @@ describe("npm publication contract", () => {
expect(packageArtifactBudget).toEqual({
entryCount: { min: 625, max: 625 },
fileCount: { min: 625, max: 625 },
packedBytes: { min: 1_600_000, max: 12_159_737 },
unpackedBytes: { min: 9_000_000, max: 24_055_550 },
packedBytes: { min: 1_600_000, max: 12_160_527 },
unpackedBytes: { min: 9_000_000, max: 24_058_273 },
});
});

Expand Down
26 changes: 12 additions & 14 deletions scripts/package-budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2390,24 +2390,22 @@
// Preserve the same projections and allowances:
// 12,154,912 + 12,387 + 4,096 = 12,171,395 packed;
// 24,054,352 + 353 + 65 = 24,054,770 payload.
// The 0.18.65 editorial release preserves that runtime and file inventory.
// Two byte-identical packs with the same qualified toolchain measured 625
// entries, 12,155,065 packed and 24,054,930 payload bytes. The 578-byte payload
// increase is the new changelog entry; article and image files are website-only.
// Retain every projection and allowance:
// 12,155,065 + 12,387 + 4,096 = 12,171,548 packed;
// 24,054,930 + 353 + 65 = 24,055,348 payload.
// The 0.18.66 analytics-host release measures 625 entries under the same
// Node 24.20.0 / npm 11.19.0 / darwin-arm64 toolchain. Two packs are identical.
// Preserve the reviewed platform projections and portability allowances.
// The 0.18.66 WhatsApp projection and analytics-host repair release retains
// the 625-entry inventory after rebuilding dist. Two byte-identical packs with
// the same qualified toolchain measured 12,144,044 packed and 24,057,855
// payload bytes; archive SHA-256
// f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc.
// Retain the reviewed projections and allowances:
// 12,144,044 + 12,387 + 4,096 = 12,160,527 packed;
// 24,057,855 + 353 + 65 = 24,058,273 payload.
export const repairPackageMeasurement = Object.freeze({
scope: "GhostGet 0.18.66 exact analytics host release over the verified 0.18.65 runtime",
scope: "GhostGet 0.18.66 WhatsApp projection and analytics-host repair release over the verified 0.18.64 runtime",
command: "npm pack --ignore-scripts",
npmVersion: "11.19.0",
platform: "darwin-arm64",
archiveSha256: "b462796a666a3f7acccbbf052590f6f7b3db4f956307567dca313c6b13d3393a",
packedBytes: 12_143_254,
unpackedBytes: 24_055_132,
archiveSha256: "f36aad9ba60d22276020595e353820a2b9140cae428ddd1f61be59ef09f275fc",
packedBytes: 12_144_044,
unpackedBytes: 24_057_855,
entryCount: 625,
packedPlatformProjection: 12_387,
packedPortabilityAllowance: 4_096,
Expand Down
135 changes: 135 additions & 0 deletions src/providers/whatsapp-interaction-projection-helper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,141 @@ describe("WhatsApp content-free interaction projection helper", () => {
}
});

test("maps the short legacy system JID to the fixed unsupported sentinel", async () => {
const path = createStore();
try {
const database = new Database(join(path, "wacli.db"), { strict: true });
try {
database.query("INSERT INTO chats(jid, kind) VALUES (?1, ?2)")
.run("1@s.whatsapp.net", "dm");
database.query(`
INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me)
VALUES (?1,?2,NULL,?3,?4)
`).run("1@s.whatsapp.net", "LEGACY-SYSTEM-1", 1_776_513_603, 0);
} finally {
database.close();
}
chmodSync(join(path, "wacli.db"), 0o600);
expect(await response(path, request(path, {
cursor: "3",
cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830",
limit: 1,
}))).toMatchObject({
status: "succeeded",
interactions: [{ rowid: "4", chatJid: "0@s.whatsapp.net", chatKind: "unknown" }],
});
} finally {
rmSync(path, { recursive: true, force: true });
}
});

test("normalizes only known legacy participant fields and retains valid senders", async () => {
const path = createStore();
try {
const database = new Database(join(path, "wacli.db"), { strict: true });
try {
database.query(`
INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me)
VALUES
('15557654321@s.whatsapp.net','LEGACY-DM-OUT','15557654321@s.whatsapp.net',1776513603,1),
('120363123456789012@g.us','LEGACY-GROUP-OUT','120363123456789012@g.us',1776513604,1),
('120363123456789012@g.us','LEGACY-GROUP-IN-OWNER',?1,1776513605,0),
('15557654321@s.whatsapp.net','LEGACY-DM-OUT-DEVICE','15557654321:7@s.whatsapp.net',1776513606,1),
('120363123456789012@g.us','LEGACY-GROUP-IN-LID','999999999999999:4@lid',1776513607,0),
('120363123456789012@g.us','VALID-GROUP-IN','15559876543:5@s.whatsapp.net',1776513608,0),
('15557654321@s.whatsapp.net','VALID-DM-OUT',?1,1776513609,1),
('120363123456789012@g.us','VALID-GROUP-OUT-LID','999999999999999:4@lid',1776513610,1)
`).run(OWNER_JID);
} finally {
database.close();
}
chmodSync(join(path, "wacli.db"), 0o600);
expect(await response(path, request(path, {
cursor: "3",
cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830",
limit: 8,
}))).toMatchObject({
status: "succeeded",
interactions: [
{ rowid: "4", messageId: "LEGACY-DM-OUT", senderJid: null },
{ rowid: "5", messageId: "LEGACY-GROUP-OUT", senderJid: null },
{ rowid: "6", messageId: "LEGACY-GROUP-IN-OWNER", senderJid: null },
{ rowid: "7", messageId: "LEGACY-DM-OUT-DEVICE", senderJid: null },
{ rowid: "8", messageId: "LEGACY-GROUP-IN-LID", senderJid: null },
{ rowid: "9", messageId: "VALID-GROUP-IN", senderJid: "15559876543:5@s.whatsapp.net" },
{ rowid: "10", messageId: "VALID-DM-OUT", senderJid: OWNER_JID },
{ rowid: "11", messageId: "VALID-GROUP-OUT-LID", senderJid: "999999999999999:4@lid" },
],
});
} finally {
rmSync(path, { recursive: true, force: true });
}
});

for (const invalid of [
{ label: "outgoing DM participant", chat: "15557654321@s.whatsapp.net", sender: "15559876543@s.whatsapp.net", fromMe: 1 },
{ label: "incoming DM participant", chat: "15557654321@s.whatsapp.net", sender: "15559876543@s.whatsapp.net", fromMe: 0 },
{ label: "outgoing group participant", chat: "120363123456789012@g.us", sender: "15559876543@s.whatsapp.net", fromMe: 1 },
{ label: "outgoing group chat", chat: "120363123456789012@g.us", sender: "120363987654321098@g.us", fromMe: 1 },
{ label: "incoming group chat", chat: "120363123456789012@g.us", sender: "120363987654321098@g.us", fromMe: 0 },
{ label: "incoming group same-chat sender", chat: "120363123456789012@g.us", sender: "120363123456789012@g.us", fromMe: 0 },
] as const) {
test(`rejects unrelated or contradictory legacy ${invalid.label}`, async () => {
const path = createStore();
try {
const database = new Database(join(path, "wacli.db"), { strict: true });
try {
database.query(`
INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me)
VALUES (?1,'INVALID-LEGACY-SENDER',?2,1776513603,?3)
`).run(invalid.chat, invalid.sender, invalid.fromMe);
} finally {
database.close();
}
chmodSync(join(path, "wacli.db"), 0o600);
expect(await response(path, request(path, {
cursor: "3",
cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830",
limit: 1,
}))).toMatchObject({ status: "failed", errorCode: "projection-invalid" });
} finally {
rmSync(path, { recursive: true, force: true });
}
});
}

test("rejects unrelated outgoing participant IDs across PN, LID, and device aliases", async () => {
await assertAsyncProperty(fc.asyncProperty(
fc.integer({ min: 10_000, max: 9_999_999 }),
fc.constantFrom("s.whatsapp.net", "lid"),
fc.option(fc.integer({ min: 0, max: 99_999 }), { nil: undefined }),
fc.constantFrom("15557654321@s.whatsapp.net", "120363123456789012@g.us"),
async (participant, server, device, chat) => {
const path = createStore();
try {
const sender = `${participant}${device === undefined ? "" : `:${device}`}@${server}`;
const database = new Database(join(path, "wacli.db"), { strict: true });
try {
database.query(`
INSERT INTO messages(chat_jid,msg_id,sender_jid,ts,from_me)
VALUES (?1,'INVALID-LEGACY-PROPERTY',?2,1776513603,1)
`).run(chat, sender);
} finally {
database.close();
}
chmodSync(join(path, "wacli.db"), 0o600);
expect(await response(path, request(path, {
cursor: "3",
cursorAnchor: "a86ecbd8b98eb6466c2b584a5b3d3ca0458230bbd6ecc86981d57ca4aaa81830",
limit: 1,
}))).toMatchObject({ status: "failed", errorCode: "projection-invalid" });
} finally {
rmSync(path, { recursive: true, force: true });
}
},
), { numRuns: 20 });
});

test("preserves rowids above JavaScript's safe-integer range", async () => {
const path = createStore();
try {
Expand Down
62 changes: 56 additions & 6 deletions src/providers/whatsapp-interaction-projection-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ const MAX_MESSAGE_DATABASE_BYTES = 2n * 1024n * 1024n * 1024n;
const SQLITE_CACHE_KIB = 4_096;
const MAX_UNIX_SECONDS = 253_402_300_799n;
const SYSTEM_SENTINEL_JID = "0@s.whatsapp.net";
// Older wacli stores use short numeric JIDs for system rows. They are not
// person or conversation identifiers and must stay outside the contact
// interaction projection. Keep the public projection's single sentinel
// while accepting the bounded legacy spelling.
const LEGACY_SYSTEM_JID = /^[0-9]@s\.whatsapp\.net$/u;
const MESSAGE_EXPORT_SESSION_MAX_MESSAGES = 500_000;
type MessageExportSessionSelfChatsExcluded = "none-detected" | "present-excluded";

Expand Down Expand Up @@ -505,14 +510,15 @@ function sqliteBigInt(value: unknown): bigint {
return fail("projection-invalid");
}

function projectedJid(value: unknown, nullable = false): string | null {
function projectedJid(value: unknown, nullable = false, allowLegacySystem = false): string | null {
if (nullable && (value === null || value === "")) return null;
if (
typeof value !== "string"
|| value.length < 9
|| value.length > 96
|| !(
value === SYSTEM_SENTINEL_JID
|| (allowLegacySystem && typeof value === "string" && LEGACY_SYSTEM_JID.test(value))
|| /^[0-9]{5,20}(?::[0-9]{1,5})?@s\.whatsapp\.net$/u.test(value)
|| /^[0-9]{5,32}(?::[0-9]{1,5})?@lid$/u.test(value)
|| /^[0-9]{5,32}(?:-[0-9]{5,20})?@g\.us$/u.test(value)
Expand All @@ -532,6 +538,43 @@ function legacyInteractionParticipantJid(value: string): string {
: `${match[1]}@s.whatsapp.net`;
}

function legacyInteractionSender(
item: Readonly<{
chatJid: string;
chatKind: "dm" | "group" | "broadcast" | "newsletter" | "unknown";
senderJid: string | null;
fromMe: boolean;
}>,
owner: WhatsAppMatchedOwnerIdentity,
): string | null {
if (item.senderJid === null || (item.chatKind !== "dm" && item.chatKind !== "group")) {
return item.senderJid;
}
if (item.chatKind === "group" && item.senderJid.endsWith("@g.us")) {
// Some outgoing migrated rows carry this exact chat JID in sender_jid.
// Other group JIDs and incoming chat senders remain invalid participants.
return item.fromMe && item.senderJid === item.chatJid ? null : item.senderJid;
}
const sender = legacyInteractionParticipantJid(item.senderJid);
const self = new Set(owner.selfJids);
const chat = item.chatKind === "dm"
? legacyInteractionParticipantJid(item.chatJid)
: item.chatJid;
if (item.chatKind === "dm" && item.fromMe && sender === chat) {
// Wacli records the recipient in sender_jid for some outgoing direct
// messages. The direction already carries the only fact PeopleBlade
// needs; dropping this inconsistent participant keeps the interaction
// body-free and prevents a false sender identity.
return null;
}
if (item.chatKind === "group" && !item.fromMe && self.has(sender)) {
// Incoming migrated rows may carry an exact session-proved owner alias.
// It identifies no counterparty; unrelated outgoing senders remain invalid.
return null;
}
return item.senderJid;
}

function assertMessageDirection(
item: Pick<WhatsAppInteractionProjectionItem, "chatJid" | "chatKind" | "senderJid" | "fromMe">,
owner: WhatsAppMatchedOwnerIdentity,
Expand Down Expand Up @@ -567,12 +610,12 @@ function projectItem(
exactRowKeys(row, ["rowid", "chat_jid", "msg_id", "sender_jid", "ts", "from_me", "chat_kind"]);
const rowid = sqliteBigInt(row.rowid);
const seconds = sqliteBigInt(row.ts);
const fromMe = sqliteBigInt(row.from_me);
const fromMeValue = sqliteBigInt(row.from_me);
if (
rowid < 1n
|| seconds < 0n
|| seconds > MAX_UNIX_SECONDS
|| (fromMe !== 0n && fromMe !== 1n)
|| (fromMeValue !== 0n && fromMeValue !== 1n)
|| typeof row.msg_id !== "string"
|| !/^[A-Za-z0-9._~:-]{1,256}$/u.test(row.msg_id)
|| (row.chat_kind !== "dm"
Expand All @@ -581,14 +624,21 @@ function projectItem(
&& row.chat_kind !== "newsletter"
&& row.chat_kind !== "unknown")
) fail("projection-invalid");
const chatJid = projectedJid(row.chat_jid)! as string;
const rawChatJid = projectedJid(row.chat_jid, false, true)! as string;
const chatJid = LEGACY_SYSTEM_JID.test(rawChatJid) ? SYSTEM_SENTINEL_JID : rawChatJid;
const fromMe = fromMeValue === 1n;
const projected = Object.freeze({
rowid: rowid.toString(),
chatJid,
messageId: row.msg_id,
senderJid: projectedJid(row.sender_jid, true),
senderJid: legacyInteractionSender({
chatJid,
chatKind: chatJid === SYSTEM_SENTINEL_JID ? "unknown" : row.chat_kind,
senderJid: projectedJid(row.sender_jid, true),
fromMe,
}, owner),
timestamp: new Date(Number(seconds) * 1_000).toISOString(),
fromMe: fromMe === 1n,
fromMe,
chatKind: chatJid === SYSTEM_SENTINEL_JID ? "unknown" : row.chat_kind,
});
assertMessageDirection(projected, owner, "legacy-interaction");
Expand Down
Loading