Skip to content

feat: churn-analysis golden fixtures and deterministic tests #992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
orunganiekan merged 3 commits into from
Aug 31, 2026
Merged

feat: churn-analysis golden fixtures and deterministic tests #992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6e47bef
feat: golden fixtures and deterministic churn-analysis tests (Closes …
godamongstmen897 Aug 31, 2026
23da7a6
Merge branch 'master' into feat/886-churn-golden-fixtures
orunganiekan Aug 31, 2026
858c51b
Merge branch 'master' into feat/886-churn-golden-fixtures
orunganiekan Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions scripts/data/churn-analysis-fixtures.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
{
"$schema": "https://json-schema.org/draft/2027/schema",
"title": "Churn Analysis Golden Fixtures",
"description": "Canonical input events and expected outputs for churn-analysis.ts deterministic tests. Each fixture is a self-contained scenario covering a specific metric or edge case.",
"fixtures": [
{
"name": "cohort-retention-new-logic",
"description": "Tests cohort retention (30d/90d) with 'new' resubscription logic. Covers multiple cohorts, resubscriptions, and insufficient data guards.",
"referenceTime": 1781481600,
"events": [
{ "eventName": "subscribed", "user": "user1", "merchant": "merchA", "timestamp": 1768003200 },
{ "eventName": "subscribed", "user": "user2", "merchant": "merchA", "timestamp": 1768435200 },
{ "eventName": "cancelled", "user": "user2", "timestamp": 1768867200 },
{ "eventName": "subscribed", "user": "user3", "merchant": "merchB", "timestamp": 1768867200 },
{ "eventName": "cancelled", "user": "user3", "timestamp": 1773532800 },
{ "eventName": "subscribed", "user": "user5", "merchant": "merchC", "timestamp": 1767571200 },
{ "eventName": "cancelled", "user": "user5", "timestamp": 1768435200 },
{ "eventName": "subscribed", "user": "user5", "merchant": "merchC", "timestamp": 1769300000 },
{ "eventName": "cancelled", "user": "user5", "timestamp": 1772236800 },
{ "eventName": "subscribed", "user": "user4", "merchant": "merchB", "timestamp": 1775433600 }
],
"expected": {
"cohorts": [
{ "cohort": "2026-01", "initial_size": 5, "active_30_days": 3, "active_90_days": 1, "retention_rate_30d": "60.00%", "retention_rate_90d": "20.00%" },
{ "cohort": "2026-04", "initial_size": 1, "active_30_days": 1, "active_90_days": "insufficient data", "retention_rate_30d": "100.00%", "retention_rate_90d": "insufficient data" }
],
"projection": { "average_monthly_churn_rate": "20.00%", "current_active_subscribers": 2, "projected_churn_count": "0.40" }
}
},
{
"name": "cohort-retention-retention-logic",
"description": "Tests cohort retention with 'retention' resubscription logic (first-subscribe user tracking). Resubscriptions do not create new lifespans.",
"referenceTime": 1781481600,
"events": [
{ "eventName": "subscribed", "user": "user1", "merchant": "merchA", "timestamp": 1768003200 },
{ "eventName": "subscribed", "user": "user2", "merchant": "merchA", "timestamp": 1768435200 },
{ "eventName": "cancelled", "user": "user2", "timestamp": 1768867200 },
{ "eventName": "subscribed", "user": "user3", "merchant": "merchB", "timestamp": 1768867200 },
{ "eventName": "cancelled", "user": "user3", "timestamp": 1773532800 },
{ "eventName": "subscribed", "user": "user5", "merchant": "merchC", "timestamp": 1767571200 },
{ "eventName": "cancelled", "user": "user5", "timestamp": 1768435200 },
{ "eventName": "subscribed", "user": "user5", "merchant": "merchC", "timestamp": 1769300000 },
{ "eventName": "cancelled", "user": "user5", "timestamp": 1772236800 },
{ "eventName": "subscribed", "user": "user4", "merchant": "merchB", "timestamp": 1775433600 }
],
"expected": {
"cohorts": [
{ "cohort": "2026-01", "initial_size": 4, "active_30_days": 3, "active_90_days": 1, "retention_rate_30d": "75.00%", "retention_rate_90d": "25.00%" },
{ "cohort": "2026-04", "initial_size": 1, "active_30_days": 1, "active_90_days": "insufficient data", "retention_rate_30d": "100.00%", "retention_rate_90d": "insufficient data" }
],
"projection": { "average_monthly_churn_rate": "12.50%", "current_active_subscribers": 2, "projected_churn_count": "0.25" }
}
},
{
"name": "merchant-churn-breakdown",
"description": "Tests merchant-level churn breakdown. Filters out single-subscriber merchants. Sorts top 5 by churn rate descending.",
"referenceTime": 500,
"events": [
{ "eventName": "subscribed", "user": "user1", "merchant": "merchA", "timestamp": 100 },
{ "eventName": "cancelled", "user": "user1", "timestamp": 150 },
{ "eventName": "subscribed", "user": "user1", "merchant": "merchB", "timestamp": 200 },
{ "eventName": "subscribed", "user": "user2", "merchant": "merchB", "timestamp": 210 },
{ "eventName": "subscribed", "user": "user3", "merchant": "merchB", "timestamp": 220 },
{ "eventName": "cancelled", "user": "user2", "timestamp": 250 },
{ "eventName": "cancelled", "user": "user3", "timestamp": 260 },
{ "eventName": "subscribed", "user": "user1", "merchant": "merchC", "timestamp": 300 },
{ "eventName": "subscribed", "user": "user2", "merchant": "merchC", "timestamp": 310 },
{ "eventName": "subscribed", "user": "user4", "merchant": "merchD", "timestamp": 400 },
{ "eventName": "cancelled", "user": "user4", "timestamp": 450 }
],
"expected": {
"top_merchants_churn": [
{ "merchant_id": "merchB", "subscribers": 3, "cancellations": 2, "churn_rate": "66.67%" },
{ "merchant_id": "merchC", "subscribers": 2, "cancellations": 0, "churn_rate": "0.00%" }
]
}
},
{
"name": "churn-projection-multiple-completed-cohorts",
"description": "Tests historical churn projection across multiple completed cohorts with varying churn rates. Validates average churn rate calculation and projection formula.",
"referenceTime": 1781481600,
"events": [
{ "eventName": "subscribed", "user": "user1", "merchant": "merchM", "timestamp": 1767571200 },
{ "eventName": "subscribed", "user": "user2", "merchant": "merchM", "timestamp": 1767571210 },
{ "eventName": "subscribed", "user": "user3", "merchant": "merchM", "timestamp": 1767571220 },
{ "eventName": "subscribed", "user": "user4", "merchant": "merchM", "timestamp": 1767571230 },
{ "eventName": "cancelled", "user": "user1", "timestamp": 1768435200 },
{ "eventName": "cancelled", "user": "user2", "timestamp": 1768867200 },
{ "eventName": "subscribed", "user": "feb_user_0", "merchant": "merchM", "timestamp": 1770249600 },
{ "eventName": "subscribed", "user": "feb_user_1", "merchant": "merchM", "timestamp": 1770249601 },
{ "eventName": "subscribed", "user": "feb_user_2", "merchant": "merchM", "timestamp": 1770249602 },
{ "eventName": "subscribed", "user": "feb_user_3", "merchant": "merchM", "timestamp": 1770249603 },
{ "eventName": "subscribed", "user": "feb_user_4", "merchant": "merchM", "timestamp": 1770249604 },
{ "eventName": "subscribed", "user": "feb_user_5", "merchant": "merchM", "timestamp": 1770249605 },
{ "eventName": "subscribed", "user": "feb_user_6", "merchant": "merchM", "timestamp": 1770249606 },
{ "eventName": "subscribed", "user": "feb_user_7", "merchant": "merchM", "timestamp": 1770249607 },
{ "eventName": "subscribed", "user": "feb_user_8", "merchant": "merchM", "timestamp": 1770249608 },
{ "eventName": "subscribed", "user": "feb_user_9", "merchant": "merchM", "timestamp": 1770249609 },
{ "eventName": "cancelled", "user": "feb_user_0", "timestamp": 1770681600 },
{ "eventName": "cancelled", "user": "feb_user_1", "timestamp": 1771113600 }
],
"expected": {
"projection": { "average_monthly_churn_rate": "35.00%", "current_active_subscribers": 10, "projected_churn_count": "3.50" }
}
},
{
"name": "grace-period-cancelled-with-refund",
"description": "Tests that 'cancelled_with_refund' events are treated identically to 'cancelled' for churn metrics (grace lapses).",
"referenceTime": 1781481600,
"events": [
{ "eventName": "subscribed", "user": "userA", "merchant": "merchX", "timestamp": 1768003200 },
{ "eventName": "cancelled_with_refund", "user": "userA", "timestamp": 1768867200 },
{ "eventName": "subscribed", "user": "userB", "merchant": "merchX", "timestamp": 1768003210 },
{ "eventName": "cancelled", "user": "userB", "timestamp": 1768867210 }
],
"expected": {
"cohorts": [
{ "cohort": "2026-01", "initial_size": 2, "active_30_days": 0, "active_90_days": 0, "retention_rate_30d": "0.00%", "retention_rate_90d": "0.00%" }
],
"projection": { "average_monthly_churn_rate": "100.00%", "current_active_subscribers": 0, "projected_churn_count": "0.00" }
}
},
{
"name": "empty-and-single-event-edge-cases",
"description": "Tests empty event list and single event edge cases produce valid reports with zero/insufficient data.",
"referenceTime": 1781481600,
"events": [],
"expected": {
"cohorts": [],
"top_merchants_churn": [],
"projection": { "average_monthly_churn_rate": "0.00%", "current_active_subscribers": 0, "projected_churn_count": "0.00" }
}
},
{
"name": "single-subscriber-merchant-filtered",
"description": "Verifies merchants with <= 1 unique subscriber are excluded from top_merchants_churn to avoid statistical skew.",
"referenceTime": 1000,
"events": [
{ "eventName": "subscribed", "user": "user1", "merchant": "singleSub", "timestamp": 100 },
{ "eventName": "subscribed", "user": "user2", "merchant": "twoSubs", "timestamp": 110 },
{ "eventName": "subscribed", "user": "user3", "merchant": "twoSubs", "timestamp": 120 },
{ "eventName": "cancelled", "user": "user3", "timestamp": 130 }
],
"expected": {
"top_merchants_churn": [
{ "merchant_id": "twoSubs", "subscribers": 2, "cancellations": 1, "churn_rate": "50.00%" }
]
}
}
]
}
1 change: 1 addition & 0 deletions scripts/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"lint": "tsc --noEmit --strict --noUnusedLocals --noUnusedParameters",
"test:event-dedup": "tsx test-event-dedup-integration.ts",
"test:renewal-forecast": "tsx test-renewal-forecast.ts",
"test": "tsx test-churn-analysis.ts"
"test:rpc-failover": "tsx test-rpc-failover.ts"
"test:renewal-forecast": "tsx test-renewal-forecast.ts"
},
Expand Down
192 changes: 175 additions & 17 deletions scripts/test-churn-analysis.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import { calculateChurnAnalysis, ChurnAnalysisReport } from "./churn-analysis.js";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { DatabaseSync } from "node:sqlite";
import { calculateChurnAnalysis } from "./churn-analysis.js";

const __dirname = resolve(fileURLToPath(import.meta.url), "..");
const FIXTURES_PATH = resolve(__dirname, "data/churn-analysis-fixtures.json");

// Load golden fixtures
const fixtures = JSON.parse(readFileSync(FIXTURES_PATH, "utf8")).fixtures;

// Utility to assert values in tests
function assertEquals(actual: any, expected: any, message: string) {
Expand All @@ -13,6 +22,109 @@ function assertEquals(actual: any, expected: any, message: string) {
}
}

function deepEquals(actual: any, expected: any): boolean {
return JSON.stringify(actual) === JSON.stringify(expected);
}

function assertDeepEquals(actual: any, expected: any, message: string) {
if (!deepEquals(actual, expected)) {
console.error(`Assertion Failed: ${message}`);
console.error(` Expected: ${JSON.stringify(expected, null, 2)}`);
console.error(` Actual: ${JSON.stringify(actual, null, 2)}`);
process.exit(1);
} else {
console.log(`[PASS] ${message}`);
}
}

function runFixture(fixture: typeof fixtures[0], logic: "new" | "retention") {
const report = calculateChurnAnalysis(fixture.events, logic, fixture.referenceTime);

// Validate cohorts if expected
if (fixture.expected.cohorts) {
for (const expCohort of fixture.expected.cohorts) {
const actCohort = report.cohorts.find((c) => c.cohort === expCohort.cohort);
assertEquals(
actCohort?.initial_size,
expCohort.initial_size,
`${fixture.name} [${logic}] cohort ${expCohort.cohort} initial_size`
);
assertEquals(
actCohort?.active_30_days,
expCohort.active_30_days,
`${fixture.name} [${logic}] cohort ${expCohort.cohort} active_30_days`
);
assertEquals(
actCohort?.active_90_days,
expCohort.active_90_days,
`${fixture.name} [${logic}] cohort ${expCohort.cohort} active_90_days`
);
assertEquals(
actCohort?.retention_rate_30d,
expCohort.retention_rate_30d,
`${fixture.name} [${logic}] cohort ${expCohort.cohort} retention_rate_30d`
);
assertEquals(
actCohort?.retention_rate_90d,
expCohort.retention_rate_90d,
`${fixture.name} [${logic}] cohort ${expCohort.cohort} retention_rate_90d`
);
}
}

// Validate projection if expected
if (fixture.expected.projection) {
assertEquals(
report.projection.average_monthly_churn_rate,
fixture.expected.projection.average_monthly_churn_rate,
`${fixture.name} [${logic}] avg monthly churn rate`
);
assertEquals(
report.projection.current_active_subscribers,
fixture.expected.projection.current_active_subscribers,
`${fixture.name} [${logic}] current active subscribers`
);
assertEquals(
report.projection.projected_churn_count,
fixture.expected.projection.projected_churn_count,
`${fixture.name} [${logic}] projected churn count`
);
}

// Validate merchant churn if expected
if (fixture.expected.top_merchants_churn) {
const expMerchants = fixture.expected.top_merchants_churn;
const actMerchants = report.top_merchants_churn;

assertEquals(
actMerchants.length,
expMerchants.length,
`${fixture.name} [${logic}] merchant count`
);

for (let i = 0; i < expMerchants.length; i++) {
assertEquals(
actMerchants[i]?.merchant_id,
expMerchants[i].merchant_id,
`${fixture.name} [${logic}] merchant ${i} id`
);
assertEquals(
actMerchants[i]?.subscribers,
expMerchants[i].subscribers,
`${fixture.name} [${logic}] merchant ${i} subscribers`
);
assertEquals(
actMerchants[i]?.cancellations,
expMerchants[i].cancellations,
`${fixture.name} [${logic}] merchant ${i} cancellations`
);
assertEquals(
actMerchants[i]?.churn_rate,
expMerchants[i].churn_rate,
`${fixture.name} [${logic}] merchant ${i} churn rate`
);
}
}
function testCohortAndResubscriptionLogic() {
console.log(
"\n--- Running Churn Analysis Cohort and Resubscription Logic Tests ---",
Expand Down Expand Up @@ -280,18 +392,60 @@ function testMerchantChurnBreakdown() {
assertEquals(merchC?.churn_rate, "0.00%", "Merchant C churn rate");
}

function testChurnProjection() {
console.log("\n--- Running Churn Projection Tests ---");
function runAllFixtures() {
console.log("==========================================");
console.log(" CHURN ANALYSIS GOLDEN FIXTURE TESTS ");
console.log("==========================================\n");

for (const fixture of fixtures) {
console.log(`\n--- Fixture: ${fixture.name} ---`);
console.log(` ${fixture.description}`);

// Test both logic modes unless fixture is logic-specific
const logics: ("new" | "retention")[] = fixture.name.includes("new-logic")
? ["new"]
: fixture.name.includes("retention-logic")
? ["retention"]
: ["new", "retention"];

for (const logic of logics) {
console.log(` Testing logic: '${logic}'`);
runFixture(fixture, logic);
}
}

console.log("\n==========================================");
console.log(" ALL GOLDEN FIXTURE TESTS PASSED! ");
console.log("==========================================");
}

// Mock events with 2 completed cohorts:
// Cohort Jan: size 4, active30 is 2. Monthly churn rate = 50%
// Cohort Feb: size 10, active30 is 8. Monthly churn rate = 20%
// Average monthly churn rate = (50% + 20%) / 2 = 35%
// Currently active subscribers = 8.
// Projected churn for next month = 8 * 35% = 2.80
const REF_TIME = 1781481600; // 2026-06-15
// Additional standalone tests for specific edge cases not covered by fixtures
function runAdditionalTests() {
console.log("\n--- Additional Edge Case Tests ---");

// Test: Data source reporting (sqlite vs rpc) - already covered by integration
// Test: CSV output format
const events = [
{ eventName: "subscribed", user: "user1", merchant: "merchA", timestamp: 1768003200 },
{ eventName: "cancelled", user: "user1", timestamp: 1768867200 },
];
const report = calculateChurnAnalysis(events, "new", 1781481600);

// Verify report structure completeness
assertEquals(typeof report.generated_at, "string", "Report has generated_at");
assertEquals(typeof report.reference_time, "string", "Report has reference_time");
assertEquals(["sqlite", "rpc"].includes(report.data_source), true, "Report has valid data_source");
assertEquals(["new", "retention"].includes(report.resubscription_logic), true, "Report has valid resubscription_logic");
assertEquals(Array.isArray(report.cohorts), true, "Report has cohorts array");
assertEquals(Array.isArray(report.top_merchants_churn), true, "Report has top_merchants_churn array");
assertEquals(typeof report.projection, "object", "Report has projection object");

// Verify projection fields
assertEquals(typeof report.projection.average_monthly_churn_rate, "string", "Projection has avg churn rate string");
assertEquals(typeof report.projection.current_active_subscribers, "number", "Projection has active count");
assertEquals(typeof report.projection.projected_churn_count, "string", "Projection has projected churn string");

console.log("[PASS] Report structure validation");
// Jan cohort (Starts Jan 05: 1767571200)
{
eventName: "subscribed",
Expand Down Expand Up @@ -375,8 +529,8 @@ function testChurnProjection() {
);
}

function testDatabaseAndRpcFallback() {
console.log("\n--- Running SQLite and RPC Fallback Configuration Tests ---");
function runDatabaseAndRpcFallbackTests() {
console.log("\n--- Database and RPC Fallback Configuration Tests ---");

// Create an in-memory SQLite database to test real DatabaseSync querying
const db = new DatabaseSync(":memory:");
Expand All @@ -397,6 +551,11 @@ function testDatabaseAndRpcFallback() {
n: number;
};
assertEquals(row.n, 1, "Mock SQLite event inserted correctly");

// The report must declare which data source fed it, so callers can tell a
// real RPC-backed run from a fallback/empty one.
const report = calculateChurnAnalysis([], "new", 1781481600);
assertEquals(["sqlite", "rpc"].includes(report.data_source), true, "Report advertises sqlite/rpc data source");
db.close();
}

Expand All @@ -405,14 +564,13 @@ function runAllTests() {
console.log(" PAYFLOW CHURN ANALYSIS TEST SUITE ");
console.log("==========================================");

testCohortAndResubscriptionLogic();
testMerchantChurnBreakdown();
testChurnProjection();
testDatabaseAndRpcFallback();
runAllFixtures();
runAdditionalTests();
runDatabaseAndRpcFallbackTests();

console.log("\n==========================================");
console.log(" ALL TESTS COMPLETED SUCCESSFULLY! ");
console.log("==========================================");
}

runAllTests();
runAllTests();
Loading