From 9bc8ef0908877abc0648fc48484b6bc830ced0ce Mon Sep 17 00:00:00 2001 From: Philip Stayetski Date: Mon, 14 Sep 2026 17:21:23 -0700 Subject: [PATCH] Count people on the statistics dashboard, and fix what it mismeasured The dashboard counted events and could not tell a crawler from a customer, a returning user from a new one, or a docs reader from a 404. Fixes first. The web app, CLI, Refstream and platforms documentation pages were counted as "Not found" because the page-view classifier only knew nine paths; every documentation route, current or versioned, now resolves to its own target, and the example Wrangler config routes those paths through the Worker so they are seen at all. Paths the site answers with the landing page are counted as unknown_path, apart from real 404s, which were never recorded and now are. Clicks on the landing page's sign-up links are counted. People, as keyed hashes. The Worker hashes a secret salt with the connecting address and the browser family (version numbers removed, so an update is the same person) and keeps twenty hex characters. The hash goes only to the dashboard's Durable Object, never to Analytics Engine, and only for events with a person behind them. One row per person per day per surface, forgotten 120 days after they were last seen. From that: unique, new and returning visitors, installers, CLI machines and viewers, weekly cohorts of who came back, and a funnel from a first look to a first keystroke in which every step says what it counts and what it is a share of. Without STATS_VISITOR_SALT nobody is hashed and the dashboard says so. Accounts, exactly. The app records the days an account used it and answers aggregates only, to a bearer STATS_TOKEN, which the Worker reads when APP_STATS_URL and APP_STATS_TOKEN are set: totals, new per day and sign-up cohorts, with no uid or email in the response. The privacy policy says what is kept and for how long. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Ueq619VZvd44TY7bF43KDR --- CHANGELOG.md | 12 + app/.env.example | 4 + app/README.md | 1 + app/server/app.test.ts | 28 ++ app/server/app.ts | 26 +- app/server/index.ts | 1 + app/server/lib/config.ts | 12 + .../lib/migrations/012_account_activity.sql | 17 + app/server/lib/orgs.ts | 5 + app/server/lib/store-conformance.test.ts | 37 ++ app/server/lib/store-memory.ts | 35 +- app/server/lib/store-postgres.ts | 41 +++ app/server/lib/store.ts | 20 + app/server/lib/types.ts | 10 + app/server/routes/stats.test.ts | 58 +++ app/server/routes/stats.ts | 132 +++++++ app/src/routes/Privacy.tsx | 22 +- app/worker/index.ts | 3 + docs/self-hosting.md | 14 +- shared/stats-snapshot.ts | 275 ++++++++++++-- shared/stats.ts | 107 +++++- tests/analytics.test.ts | 54 +++ web/main.ts | 37 +- web/stats.css | 279 ++++++++++++++ web/stats.ts | 341 +++++++++++++----- worker/analytics.ts | 100 ++++- worker/index.ts | 159 +++++--- worker/stats-store.test.ts | 130 +++++++ worker/stats-store.ts | 122 ++++++- wrangler.example.jsonc | 4 + 30 files changed, 1903 insertions(+), 183 deletions(-) create mode 100644 app/server/lib/migrations/012_account_activity.sql create mode 100644 app/server/routes/stats.test.ts create mode 100644 app/server/routes/stats.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cb0dc62..fff9a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,24 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve and unlock screens, error notices, the empty sessions list, and the delete account form. Messages are kept by the service and, with `FEEDBACK_TO` set, forwarded by email. Nothing from a terminal is attached. +- The statistics dashboard counts people, not only events: unique, new and + returning visitors, CLI machines, installers and viewers, from keyed hashes + of address and browser family that never leave the Worker and are forgotten + after 120 days. A funnel from a first look to a first browser keystroke says + what each step counts, weekly cohorts show who came back, and the accounts + app can add exact account counts and sign-up retention when the two are + linked. +- Clicks on the landing page's Sign up free and Web app links are counted. ### Fixed - Widened the vault password fields on the vault setup, unlock, and Account pages to twice their previous width, so a password is no longer typed into a box sized for the four-letter recovery-key confirmation. +- The web app, CLI, Refstream and platforms documentation pages were counted + as "Not found". Every documentation route, current or versioned, now has its + own page-view target; unknown paths the site answers with the landing page + are counted apart from real 404s, and real 404s are counted at all. ### Changed diff --git a/app/.env.example b/app/.env.example index ccecd7f..74aaafc 100644 --- a/app/.env.example +++ b/app/.env.example @@ -16,6 +16,10 @@ POSTGRES_PASSWORD= # Set only behind a proxy that replaces X-Forwarded-For TRUST_PROXY=0 +# Optional, 32+ characters. The same value goes on the relay Worker as +# APP_STATS_TOKEN so its statistics dashboard can read account counts. +# STATS_TOKEN= + # Optional invitation email. Defaults to SendGrid. MAIL_PROVIDER=sendgrid MAIL_API_KEY= diff --git a/app/README.md b/app/README.md index 88c038a..506d999 100644 --- a/app/README.md +++ b/app/README.md @@ -43,6 +43,7 @@ The client reads `VITE_FIREBASE_*` at build time. The server uses: | `MAIL_PROVIDER`, `MAIL_API_URL` | Optional non-SendGrid JSON provider | | `FEEDBACK_TO` | Optional address that feedback sent from the app is forwarded to | | `TRUST_PROXY` | Set to `1` only behind a trusted proxy | +| `STATS_TOKEN` | Optional, 32+ characters: lets the relay's statistics dashboard read account counts and sign-up cohorts | See [`.env.example`](.env.example) for the complete development configuration. diff --git a/app/server/app.test.ts b/app/server/app.test.ts index 545254d..8d9ad9b 100644 --- a/app/server/app.test.ts +++ b/app/server/app.test.ts @@ -2581,3 +2581,31 @@ describe("feedback", () => { expect(other.status).toBe(201); }); }); + +describe("account figures for the statistics dashboard", () => { + const TOKEN = "stats-token-with-thirty-two-characters!"; + + it("does not exist until a token is configured", async () => { + const answer = await call("GET", "/api/stats/accounts", { auth: TOKEN }); + expect(answer.status).toBe(404); + }); + + it("answers only the configured token, with counts and no identifiers", async () => { + handle = createApp({ store, verifyIdToken: verifyIdToken as never, allowedOrigins: [ORIGIN], statsToken: TOKEN }); + /* One person signs in, which creates their organization and marks the day. */ + expect((await call("GET", "/api/org", { auth: await idToken() })).status).toBe(200); + + expect((await call("GET", "/api/stats/accounts?range=7d")).status).toBe(401); + expect((await call("GET", "/api/stats/accounts?range=7d", { auth: "stats-token-with-thirty-two-characters?" })).status).toBe(401); + expect((await call("GET", "/api/stats/accounts?range=7d", { auth: await idToken() })).status).toBe(401); + + const answer = await call("GET", "/api/stats/accounts?range=7d", { auth: TOKEN }); + expect(answer.status).toBe(200); + expect(answer.body).toMatchObject({ total: 1, newInRange: 1, activeInRange: 1 }); + expect(answer.body.newByDay).toHaveLength(90); + expect(answer.body.cohorts).toHaveLength(1); + expect(answer.body.cohorts[0].size).toBe(1); + expect(JSON.stringify(answer.body)).not.toContain("uid-1"); + expect(JSON.stringify(answer.body)).not.toContain("ana@example.com"); + }); +}); diff --git a/app/server/app.ts b/app/server/app.ts index a074a34..e3ec77f 100644 --- a/app/server/app.ts +++ b/app/server/app.ts @@ -45,6 +45,8 @@ import { recordAudit, assignSession, auditCsv, SEALED_KINDS } from "./routes/aud import { addComment, inbox, notifyAssigned, notifySessionStarted } from "./routes/social"; import { deleteAccount } from "./routes/account"; import { submitFeedback } from "./routes/feedback"; +import { accountStats, isStatsRange } from "./routes/stats"; +import { timingSafeEqual } from "node:crypto"; import { callerAddress, rateLimiter } from "./lib/rate-limit"; import { logMailer, type Mailer } from "./lib/mail"; @@ -76,6 +78,12 @@ export interface AppOptions { * store only, which is still the record; the mail is for whoever reads it. */ feedbackTo?: string; + /** + * Lets the statistics dashboard on the relay read account figures: counts + * and sign-up cohorts, never a person. Absent means the route does not + * exist. At least 32 characters; see readConfig. + */ + statsToken?: string; /** * Serves the built client for anything that is not an API route. Present * only in a deployment that serves the app and the API together; in @@ -364,7 +372,10 @@ export function createApp(options: AppOptions) { async function requireMember(request: IncomingMessage, inviteId?: string) { const identity = await requireUser(request); if (!identity) return null; - return (await ensureMembership(store, identity, inviteId))?.membership ?? null; + const membership = (await ensureMembership(store, identity, inviteId))?.membership ?? null; + /* A day with a request on it is a day the account was active. */ + if (membership) await store.touchMembership(membership.uid); + return membership; } /* The CLI authenticates with an opaque access token issued by this service. */ @@ -1460,6 +1471,19 @@ export function createApp(options: AppOptions) { return send(response, 201, { feedback: { id: result.value.id, at: result.value.at } }); } + /* ---- Account figures for the statistics dashboard ---- */ + + if (route === "GET /api/stats/accounts") { + const expected = options.statsToken; + if (!expected) return send(response, 404, { error: "not found" }); + const presented = bearer(request); + const matches = presented.length === expected.length && + timingSafeEqual(Buffer.from(presented), Buffer.from(expected)); + if (!matches) return send(response, 401, { error: "sign in first" }); + const range = url.searchParams.get("range"); + return send(response, 200, accountStats(await store.accountActivity(), isStatsRange(range) ? range : "7d")); + } + /* ---- Inbox ---- */ if (route === "GET /api/notifications") { diff --git a/app/server/index.ts b/app/server/index.ts index 12bdd52..54f7c58 100644 --- a/app/server/index.ts +++ b/app/server/index.ts @@ -50,6 +50,7 @@ const server = createAccountsServer({ trustProxy: config.trustProxy, mailer: createMailer(config.mail), feedbackTo: config.feedbackTo, + statsToken: config.statsToken, serveClient: config.clientDir ? staticFiles(config.clientDir) : undefined, relay: forward ?? undefined, sessionLiveness: config.relayUrl ? relaySessionLiveness(config.relayUrl) : undefined, diff --git a/app/server/lib/config.ts b/app/server/lib/config.ts index 1a1280c..5bc7214 100644 --- a/app/server/lib/config.ts +++ b/app/server/lib/config.ts @@ -50,6 +50,11 @@ export interface Config { * in the database only, which is still the record. */ feedbackTo?: string; + /** + * The bearer token the relay's statistics dashboard presents for account + * figures. Absent means that route does not exist. + */ + statsToken?: string; } export class ConfigError extends Error {} @@ -133,6 +138,12 @@ export function readConfig(env: NodeJS.ProcessEnv = process.env): Config { throw new ConfigError("set DATABASE_URL: the file store cannot back a deployment"); } + const statsToken = env.STATS_TOKEN?.trim() || undefined; + /* A short token is a guessable one, and this one opens business figures. */ + if (statsToken !== undefined && statsToken.length < 32) { + throw new ConfigError("STATS_TOKEN must be at least 32 characters"); + } + const clientDir = env.CLIENT_DIR?.trim() || undefined; const relayUrl = env.RELAY_URL?.trim() || undefined; if (relayUrl) { @@ -175,5 +186,6 @@ export function readConfig(env: NodeJS.ProcessEnv = process.env): Config { from: env.MAIL_FROM?.trim() || undefined, }, feedbackTo: env.FEEDBACK_TO?.trim() || undefined, + statsToken, }; } diff --git a/app/server/lib/migrations/012_account_activity.sql b/app/server/lib/migrations/012_account_activity.sql new file mode 100644 index 0000000..f64f7de --- /dev/null +++ b/app/server/lib/migrations/012_account_activity.sql @@ -0,0 +1,17 @@ +-- The days on which an account used the app, so the statistics dashboard can +-- say how many accounts are active and how many come back after signing up. +-- +-- One row per account per day and nothing else: no route, no action, no +-- address. last_seen_at on the membership is the same fact at finer grain, +-- and is what keeps the write cheap: a request only touches these when the +-- membership has not been touched for a while. Both are deleted with the +-- account; the days are dropped after 400 days regardless. +ALTER TABLE memberships ADD COLUMN IF NOT EXISTS last_seen_at BIGINT; + +CREATE TABLE IF NOT EXISTS account_activity ( + uid TEXT NOT NULL, + day BIGINT NOT NULL, + PRIMARY KEY (uid, day) +); + +CREATE INDEX IF NOT EXISTS account_activity_day ON account_activity (day); diff --git a/app/server/lib/orgs.ts b/app/server/lib/orgs.ts index 1d3c813..5843b70 100644 --- a/app/server/lib/orgs.ts +++ b/app/server/lib/orgs.ts @@ -24,6 +24,11 @@ export interface Membership { name: string; role: Role; joinedAt: number; + /** + * When this account last used the app, to the hour. Moved by + * Store.touchMembership; see the account_activity migration for why. + */ + lastSeenAt?: number; /** * This person's browser key, published so colleagues can seal a session * password to them. Absent until they have signed in somewhere. diff --git a/app/server/lib/store-conformance.test.ts b/app/server/lib/store-conformance.test.ts index 712c816..069f22c 100644 --- a/app/server/lib/store-conformance.test.ts +++ b/app/server/lib/store-conformance.test.ts @@ -148,6 +148,7 @@ function feedback(overrides: Partial = {}): Feedback { const TABLES = [ "feedback", + "account_activity", "deleted_accounts", "account_keys", "session_key_shares", @@ -906,6 +907,42 @@ for (const implementation of implementations) { }); }); + describe("account activity", () => { + const day = 24 * 60 * 60_000; + const noon = 10 * day + 12 * 60 * 60_000; + + it("marks a day once and moves last seen at most once an hour", async () => { + await store.putOrganization(organization()); + await store.putMembership(membership()); + await store.touchMembership("uid-1", noon); + await store.touchMembership("uid-1", noon + 10 * 60_000); + expect((await store.membershipOf("uid-1"))?.lastSeenAt).toBe(noon); + await store.touchMembership("uid-1", noon + 2 * 60 * 60_000); + expect((await store.membershipOf("uid-1"))?.lastSeenAt).toBe(noon + 2 * 60 * 60_000); + await store.touchMembership("uid-1", noon + day); + await store.touchMembership("nobody", noon); + expect(await store.accountActivity()).toEqual([{ joinedAt: 1000, days: [10 * day, 11 * day] }]); + }); + + it("keeps the days through a membership rewrite and drops them with the account", async () => { + await store.putOrganization(organization()); + await store.putMembership(membership()); + await store.touchMembership("uid-1", noon); + await store.putMembership(membership({ name: "Ana R." })); + expect((await store.accountActivity())[0].days).toEqual([10 * day]); + await store.deleteAccount("uid-1", { orgId: "org_1", dissolve: true }, noon + 1); + expect(await store.accountActivity()).toEqual([]); + }); + + it("forgets days older than the memory window when purging", async () => { + await store.putOrganization(organization()); + await store.putMembership(membership()); + await store.touchMembership("uid-1", noon); + await store.purgeExpired(noon + 401 * day); + expect((await store.accountActivity())[0].days).toEqual([]); + }); + }); + describe("claiming an organization on first sight", () => { /* * Signing in fires several requests at once. On a new account none of diff --git a/app/server/lib/store-memory.ts b/app/server/lib/store-memory.ts index 6387f72..ed11874 100644 --- a/app/server/lib/store-memory.ts +++ b/app/server/lib/store-memory.ts @@ -3,6 +3,8 @@ import { randomBytes } from "node:crypto"; import { dirname, join } from "node:path"; import type { Invite, Membership, Organization, Role } from "./orgs"; import { + ACCOUNT_ACTIVITY_MEMORY_MS, + DAY_MS, DELETED_ACCOUNT_MEMORY_MS, DELETED_ACTOR_EMAIL, type AccountDeletion, @@ -11,6 +13,7 @@ import { type Store, } from "./store"; import type { + AccountActivity, AccountKey, AgentCommand, AuditEvent, @@ -69,6 +72,7 @@ interface Shape { feedback: Feedback[]; accountKeys: AccountKey[]; deletedAccounts: { uid: string; deletedAt: number }[]; + accountActivity: { uid: string; day: number }[]; teamKeys: TeamKey[]; teamKeyShares: TeamKeyShare[]; } @@ -77,7 +81,7 @@ const EMPTY: Shape = { codes: [], tokens: [], sessions: [], commands: [], organizations: [], memberships: [], invites: [], audit: [], comments: [], notifications: [], feedback: [], accountKeys: [], deletedAccounts: [], - teamKeys: [], teamKeyShares: [], + accountActivity: [], teamKeys: [], teamKeyShares: [], }; /** @@ -143,6 +147,7 @@ export class MemoryStore implements Store { feedback: parsed.feedback ?? [], accountKeys: parsed.accountKeys ?? [], deletedAccounts: parsed.deletedAccounts ?? [], + accountActivity: parsed.accountActivity ?? [], teamKeys: parsed.teamKeys ?? [], teamKeyShares: parsed.teamKeyShares ?? [], }; @@ -337,6 +342,7 @@ export class MemoryStore implements Store { if (invite.acceptedBy === uid) delete invite.email; } data.accountKeys = data.accountKeys.filter((entry) => entry.uid !== uid); + data.accountActivity = data.accountActivity.filter((entry) => entry.uid !== uid); data.comments = data.comments.filter((entry) => entry.authorUid !== uid); data.notifications = data.notifications.filter( (entry) => entry.uid !== uid && entry.actorUid !== uid, @@ -814,6 +820,9 @@ export class MemoryStore implements Store { } async purgeExpired(now = Date.now()): Promise { + this.data.accountActivity = this.data.accountActivity.filter( + (entry) => entry.day >= now - ACCOUNT_ACTIVITY_MEMORY_MS, + ); const before = this.data.codes.length; this.data.codes = this.data.codes.filter((entry) => entry.expiresAt > now); /* Finished commands are only kept long enough to be reported back. */ @@ -860,6 +869,30 @@ export class MemoryStore implements Store { .slice(0, limit); } + /* ---- Account activity ---- */ + + async touchMembership(uid: string, now = Date.now(), resolutionMs = 60 * 60_000): Promise { + const membership = this.data.memberships.find((entry) => entry.uid === uid); + if (!membership) return; + if (membership.lastSeenAt !== undefined && now - membership.lastSeenAt < resolutionMs) return; + membership.lastSeenAt = now; + const day = Math.floor(now / DAY_MS) * DAY_MS; + if (!this.data.accountActivity.some((entry) => entry.uid === uid && entry.day === day)) { + this.data.accountActivity.push({ uid, day }); + } + this.flush(); + } + + async accountActivity(): Promise { + return this.data.memberships.map((membership) => ({ + joinedAt: membership.joinedAt, + days: this.data.accountActivity + .filter((entry) => entry.uid === membership.uid) + .map((entry) => entry.day) + .sort((left, right) => left - right), + })); + } + async tokensForImport(): Promise { return this.data.tokens; } diff --git a/app/server/lib/store-postgres.ts b/app/server/lib/store-postgres.ts index 7c5ce01..4331c47 100644 --- a/app/server/lib/store-postgres.ts +++ b/app/server/lib/store-postgres.ts @@ -5,6 +5,8 @@ import { fileURLToPath } from "node:url"; import pg from "pg"; import type { Invite, Membership, Organization, Role } from "./orgs"; import { + ACCOUNT_ACTIVITY_MEMORY_MS, + DAY_MS, DELETED_ACCOUNT_MEMORY_MS, DELETED_ACTOR_EMAIL, type AccountDeletion, @@ -13,6 +15,7 @@ import { type Store, } from "./store"; import type { + AccountActivity, AccountKey, AgentCommand, AuditEvent, @@ -223,6 +226,7 @@ function toMembership(row: Row): Membership { name: row.name, role: row.role, joinedAt: row.joined_at, + lastSeenAt: row.last_seen_at ?? undefined, publicKey: row.public_key, accountKey: row.account_key, }) as unknown as Membership; @@ -993,6 +997,7 @@ export class PostgresStore implements Store { "DELETE FROM sessions WHERE uid = $1", "DELETE FROM session_key_shares WHERE uid = $1", "DELETE FROM account_keys WHERE uid = $1", + "DELETE FROM account_activity WHERE uid = $1", "DELETE FROM comments WHERE author_uid = $1", "DELETE FROM notifications WHERE uid = $1 OR actor_uid = $1", /* The address an invite was sent to is theirs once they accepted it. */ @@ -1575,9 +1580,45 @@ export class PostgresStore implements Store { return rows.map(toFeedback); } + /* ---- Account activity ---- */ + + /* + * One UPDATE per request, which does nothing until the hour is up; the day + * row is written only when the update did something, so a busy account + * costs one extra statement an hour and an idle one costs nothing. + */ + async touchMembership(uid: string, now = Date.now(), resolutionMs = 60 * 60_000): Promise { + const moved = await this.pool.query( + `UPDATE memberships SET last_seen_at = $2 + WHERE uid = $1 AND (last_seen_at IS NULL OR $2 - last_seen_at >= $3)`, + [uid, now, resolutionMs], + ); + if ((moved.rowCount ?? 0) === 0) return; + await this.pool.query( + "INSERT INTO account_activity (uid, day) VALUES ($1, $2) ON CONFLICT DO NOTHING", + [uid, Math.floor(now / DAY_MS) * DAY_MS], + ); + } + + async accountActivity(): Promise { + const members = await this.rows("SELECT uid, joined_at FROM memberships"); + const days = await this.rows("SELECT uid, day FROM account_activity ORDER BY day"); + const byUid = new Map(); + for (const row of days) { + const list = byUid.get(row.uid as string) ?? []; + list.push(row.day as number); + byUid.set(row.uid as string, list); + } + return members.map((row) => ({ + joinedAt: row.joined_at as number, + days: byUid.get(row.uid as string) ?? [], + })); + } + /* ---- Housekeeping ---- */ async purgeExpired(now = Date.now()): Promise { + await this.pool.query("DELETE FROM account_activity WHERE day < $1", [now - ACCOUNT_ACTIVITY_MEMORY_MS]); await this.pool.query("DELETE FROM auth_codes WHERE expires_at <= $1", [now]); /* Finished commands are only kept long enough to be reported back. */ await this.pool.query("DELETE FROM agent_commands WHERE done_at IS NOT NULL AND done_at < $1", [ diff --git a/app/server/lib/store.ts b/app/server/lib/store.ts index 0545cfc..36c30ab 100644 --- a/app/server/lib/store.ts +++ b/app/server/lib/store.ts @@ -1,5 +1,6 @@ import type { Invite, Membership, Organization, Role } from "./orgs"; import type { + AccountActivity, AccountKey, AgentCommand, AuditEvent, @@ -39,6 +40,15 @@ export interface AuditPage { */ export const DELETED_ACCOUNT_MEMORY_MS = 2 * 60 * 60_000; +export const DAY_MS = 24 * 60 * 60_000; + +/** + * How long the days an account used the app are kept. Long enough for a + * year's sign-up cohorts to be read back, short enough that the table stays + * a footnote. + */ +export const ACCOUNT_ACTIVITY_MEMORY_MS = 400 * DAY_MS; + /** What the activity trail shows in place of a deleted account's email. */ export const DELETED_ACTOR_EMAIL = "deleted account"; @@ -238,6 +248,16 @@ export interface Store { */ feedback(limit?: number): Promise; + /* ---- Account activity ---- */ + /** + * Records that an account used the app. Cheap enough for every request: the + * membership's lastSeenAt moves at most once per `resolutionMs`, and the + * day is written only when it does. + */ + touchMembership(uid: string, now?: number, resolutionMs?: number): Promise; + /** Every account's sign-up time and active days, with no identifiers. */ + accountActivity(): Promise; + /* ---- Housekeeping ---- */ purgeExpired(now?: number): Promise; close(): Promise; diff --git a/app/server/lib/types.ts b/app/server/lib/types.ts index b301cc4..24c43fb 100644 --- a/app/server/lib/types.ts +++ b/app/server/lib/types.ts @@ -196,6 +196,16 @@ export interface Feedback { at: number; } +/** + * One account's sign-up time and the days it used the app, with nothing that + * says which account. What the statistics dashboard's account figures are + * computed from. + */ +export interface AccountActivity { + joinedAt: number; + days: number[]; +} + export interface SessionRecord { id: string; uid: string; diff --git a/app/server/routes/stats.test.ts b/app/server/routes/stats.test.ts new file mode 100644 index 0000000..f548833 --- /dev/null +++ b/app/server/routes/stats.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { accountStats, buildRetentionCohorts, DAY_MS, WEEK_MS, weekStart } from "./stats"; + +/* A Monday, so the weeks are easy to read. */ +const monday = Date.UTC(2026, 8, 7); +const now = monday + 2 * WEEK_MS + 3 * DAY_MS + 9 * 60 * 60_000; + +describe("accountStats", () => { + const accounts = [ + /* signed up week 0, used it in weeks 1 and 2 */ + { joinedAt: monday + 60_000, days: [monday, monday + WEEK_MS + DAY_MS, monday + 2 * WEEK_MS] }, + /* signed up week 0, never came back */ + { joinedAt: monday + 3 * DAY_MS, days: [monday + 3 * DAY_MS] }, + /* signed up yesterday */ + { joinedAt: now - DAY_MS, days: [Math.floor((now - DAY_MS) / DAY_MS) * DAY_MS] }, + /* signed up long ago, active today */ + { joinedAt: monday - 30 * WEEK_MS, days: [Math.floor(now / DAY_MS) * DAY_MS] }, + ]; + + it("counts accounts, new ones, and active ones for the range", () => { + const week = accountStats(accounts, "7d", now); + expect(week.total).toBe(4); + expect(week.newInRange).toBe(1); + expect(week.activeInRange).toBe(3); + const all = accountStats(accounts, "all", now); + expect(all.newInRange).toBe(4); + expect(all.activeInRange).toBe(4); + }); + + it("sends new accounts per day, with zeros for quiet days", () => { + const stats = accountStats(accounts, "30d", now); + expect(stats.newByDay).toHaveLength(90); + expect(stats.newByDay.reduce((sum, point) => sum + point.count, 0)).toBe(3); + expect(stats.newByDay.at(-2)?.count).toBe(1); + }); + + it("builds sign-up cohorts without any identifier", () => { + const stats = accountStats(accounts, "7d", now); + expect(stats.cohorts).toEqual([ + { weekStart: monday, size: 2, active: [1, 1] }, + { weekStart: monday + 2 * WEEK_MS, size: 1, active: [] }, + ]); + expect(JSON.stringify(stats)).not.toMatch(/uid|email/); + }); +}); + +describe("buildRetentionCohorts", () => { + it("starts weeks on Monday and counts a person once per later week", () => { + expect(weekStart(monday + 6 * DAY_MS + 5 * 60 * 60_000)).toBe(monday); + const rows = [ + { visitor: "a", first_day: monday, day: monday }, + { visitor: "a", first_day: monday, day: monday + WEEK_MS }, + { visitor: "a", first_day: monday, day: monday + WEEK_MS + DAY_MS }, + { visitor: "b", first_day: monday + DAY_MS, day: monday + DAY_MS }, + ]; + expect(buildRetentionCohorts(rows, now, 8)).toEqual([{ weekStart: monday, size: 2, active: [1, 0] }]); + }); +}); diff --git a/app/server/routes/stats.ts b/app/server/routes/stats.ts new file mode 100644 index 0000000..6b3ad4a --- /dev/null +++ b/app/server/routes/stats.ts @@ -0,0 +1,132 @@ +import type { AccountActivity } from "../lib/store"; + +/* + * Account figures for the statistics dashboard on stats.shell.online. + * + * The relay's dashboard counts people by keyed hashes, which is the best it + * can do without accounts. This app has the exact thing: an account signed up + * on a day and used the app on some days after. What leaves here is + * aggregates only, computed from rows that carry no identifier, so the + * dashboard learns how many and never who. + */ + +export const STATS_RANGES = ["24h", "7d", "30d", "all"] as const; +export type StatsRange = (typeof STATS_RANGES)[number]; + +export const DAY_MS = 24 * 60 * 60_000; +export const WEEK_MS = 7 * DAY_MS; +/** How many weekly sign-up cohorts the dashboard shows. Matches the relay's. */ +export const RETENTION_WEEKS = 8; +/** How many days of new-account counts are sent, whatever the range. */ +export const NEW_BY_DAY_LIMIT = 90; +/** How long an account's activity days are kept. */ +export const ACCOUNT_ACTIVITY_MEMORY_MS = 400 * DAY_MS; + +export interface RetentionCohort { + weekStart: number; + size: number; + /** active[0] is the week after the sign-up week, and so on. */ + active: number[]; +} + +export interface AccountStats { + total: number; + newInRange: number; + activeInRange: number; + newByDay: { day: number; count: number }[]; + cohorts: RetentionCohort[]; +} + +export function isStatsRange(value: unknown): value is StatsRange { + return typeof value === "string" && (STATS_RANGES as readonly string[]).includes(value); +} + +export function dayStart(at: number): number { + return Math.floor(at / DAY_MS) * DAY_MS; +} + +/** Monday 00:00 UTC of the week containing `at`. */ +export function weekStart(at: number): number { + const day = dayStart(at); + const sinceMonday = (new Date(day).getUTCDay() + 6) % 7; + return day - sinceMonday * DAY_MS; +} + +export function rangeStart(range: StatsRange, now: number): number { + if (range === "24h") return now - DAY_MS; + if (range === "7d") return now - 7 * DAY_MS; + if (range === "30d") return now - 30 * DAY_MS; + return 0; +} + +export function accountStats(activity: AccountActivity[], range: StatsRange, now = Date.now()): AccountStats { + const start = rangeStart(range, now); + const startDay = dayStart(start); + const newByDay = new Map(); + const firstDay = dayStart(now) - (NEW_BY_DAY_LIMIT - 1) * DAY_MS; + for (let day = firstDay; day <= dayStart(now); day += DAY_MS) newByDay.set(day, 0); + + let newInRange = 0; + let activeInRange = 0; + const rows: { visitor: string; first_day: number; day: number }[] = []; + activity.forEach((account, index) => { + const joinedDay = dayStart(account.joinedAt); + if (account.joinedAt >= start) newInRange += 1; + /* Signing up is using it, so a brand-new account is active on its first day. */ + if (account.joinedAt >= start || account.days.some((day) => day >= startDay)) activeInRange += 1; + if (newByDay.has(joinedDay)) newByDay.set(joinedDay, (newByDay.get(joinedDay) ?? 0) + 1); + const visitor = String(index); + rows.push({ visitor, first_day: joinedDay, day: joinedDay }); + for (const day of account.days) rows.push({ visitor, first_day: joinedDay, day }); + }); + + return { + total: activity.length, + newInRange, + activeInRange, + newByDay: [...newByDay.entries()].map(([day, count]) => ({ day, count })), + cohorts: buildRetentionCohorts(rows, now), + }; +} + +/* + * Weekly cohorts, the same way the relay's dashboard builds them from visitor + * hashes: everyone who signed up in one week, and how many of them used the + * app in each later week. The current week is included as it stands. + */ +export function buildRetentionCohorts( + rows: { visitor: string; first_day: number; day: number }[], + now: number, + weeks = RETENTION_WEEKS, +): RetentionCohort[] { + const thisWeek = weekStart(now); + const earliest = thisWeek - (weeks - 1) * WEEK_MS; + const cohorts = new Map>>(); + for (const row of rows) { + const cohortWeek = weekStart(row.first_day); + if (cohortWeek < earliest || cohortWeek > thisWeek) continue; + let members = cohorts.get(cohortWeek); + if (!members) { + members = new Map(); + cohorts.set(cohortWeek, members); + } + let active = members.get(row.visitor); + if (!active) { + active = new Set(); + members.set(row.visitor, active); + } + const later = Math.round((weekStart(row.day) - cohortWeek) / WEEK_MS); + if (later >= 1) active.add(later); + } + return [...cohorts.entries()] + .sort(([left], [right]) => left - right) + .map(([cohortWeek, members]) => { + const span = Math.min(weeks - 1, Math.round((thisWeek - cohortWeek) / WEEK_MS)); + const active = Array.from({ length: span }, (_, index) => { + let count = 0; + for (const weeksActive of members.values()) if (weeksActive.has(index + 1)) count += 1; + return count; + }); + return { weekStart: cohortWeek, size: members.size, active }; + }); +} diff --git a/app/src/routes/Privacy.tsx b/app/src/routes/Privacy.tsx index a81f404..02e91e2 100644 --- a/app/src/routes/Privacy.tsx +++ b/app/src/routes/Privacy.tsx @@ -120,7 +120,8 @@ export function Privacy() { Your email address, your name if you give one, and how you sign in: email and password, or Google. Google’s Firebase Authentication holds the account and your password. We never - see the password. + see the password. We also keep the days on which your account + used the app, so we can count how many accounts are active.
@@ -333,10 +334,15 @@ export function Privacy() {

- The web app runs no analytics. The shell.online site and the relay - count events such as page views, installer downloads and sessions - opened, with a device class, a client name and the referring site. - They record no IP addresses, session identifiers, URLs, commands, + The web app runs no analytics of its own; the days your account + used it are account data, described above, and leave the app only + as counts. The shell.online site and the relay count events such as + page views, installer downloads and sessions opened, with a device + class, a client name and the referring site. To tell one visitor + from another they keep, for 120 days, a keyed hash of the network + address and browser family. The key never leaves the server, the + address itself is not stored, and nothing in that record says who + a visitor is. They record no session identifiers, URLs, commands, terminal content or full user-agent strings.

@@ -364,6 +370,10 @@ export function Privacy() { Feedback you send stays with us, with your identity removed once you delete your account. +
  • + The days your account used the app: 400 days, or until you delete + your account. +
  • When an account is deleted, its user identifier alone is kept for two hours, so a browser still signed in to it cannot bring it @@ -395,7 +405,7 @@ export function Privacy() { your session records, and the session passwords sealed to you, are deleted;
    • -
  • your vault, comments and notifications are deleted;
    • +
  • your vault, comments, notifications and activity days are deleted;
  • feedback you sent is kept, no longer linked to your account or email address; diff --git a/app/worker/index.ts b/app/worker/index.ts index 0ac948c..dc48e5e 100644 --- a/app/worker/index.ts +++ b/app/worker/index.ts @@ -40,6 +40,8 @@ export interface Env { MAIL_API_KEY?: string; /** Where in-app feedback is forwarded. Absent keeps it in the database only. */ FEEDBACK_TO?: string; + /** Secret: what the relay's statistics dashboard presents for account figures. */ + STATS_TOKEN?: string; } interface ExecutionContext { @@ -136,6 +138,7 @@ function routerFor(env: Env): NodeHandler { from: env.MAIL_FROM, }), feedbackTo: env.FEEDBACK_TO?.trim() || undefined, + statsToken: env.STATS_TOKEN && env.STATS_TOKEN.length >= 32 ? env.STATS_TOKEN : undefined, log: (message, error) => console.error(message, error), sessionLiveness: liveness, })); diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 1a0eec1..8920167 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -66,7 +66,19 @@ npx wrangler deploy --config wrangler.local.jsonc Set `SHELL_ONLINE_SERVER` to the URL Wrangler prints. Add a `routes` entry to the copied config for a custom domain. The Worker path requires Durable -Objects, Rate Limiting, Analytics Engine, and static assets. +Objects, Rate Limiting, Analytics Engine, and static assets. Keep every +documentation path in `run_worker_first`: a page served straight from the +assets binding is never counted. + +The private statistics dashboard is optional and configured with Worker +secrets (`npx wrangler secret put `): + +| Secret | Purpose | +|---|---| +| `STATS_PASSWORD` | Opens the dashboard on the stats hostname; at least 12 characters | +| `STATS_VISITOR_SALT` | Lets the dashboard count people as keyed hashes of address and browser family; at least 16 characters. Without it every unique, new, returning and retention figure stays at zero | +| `APP_STATS_URL` | The accounts app's origin, for exact account counts and sign-up cohorts | +| `APP_STATS_TOKEN` | The token the accounts app expects; the same value as its `STATS_TOKEN` | ## Accounts app diff --git a/shared/stats-snapshot.ts b/shared/stats-snapshot.ts index 7ed3f3a..386f3b5 100644 --- a/shared/stats-snapshot.ts +++ b/shared/stats-snapshot.ts @@ -1,13 +1,25 @@ import { + RETENTION_WEEKS, + UNIQUE_SURFACES, + VISITOR_MEMORY_DAYS, + isUniqueSurface, + type StatsAccounts, type StatsBreakdownItem, + type StatsFunnelStep, type StatsRange, + type StatsRetentionCohort, type StatsSeriesPoint, type StatsSnapshot, type StatsTargetMetric, + type StatsUniqueCount, + type StatsUniqueDay, + type StatsUniques, + type UniqueSurface, } from "./stats"; -const HOUR_MS = 60 * 60 * 1_000; -const DAY_MS = 24 * HOUR_MS; +export const HOUR_MS = 60 * 60 * 1_000; +export const DAY_MS = 24 * HOUR_MS; +export const WEEK_MS = 7 * DAY_MS; export const STATS_PRESENCE_REFRESH_MS = 45_000; export const STATS_PRESENCE_LEASE_MS = 3 * 60 * 1_000; @@ -37,6 +49,27 @@ export interface LivePresenceRow extends Record active_viewers: number; } +/** Distinct visitor hashes seen on one surface in the range, and how many were first seen in it. */ +export interface UniqueSummaryRow extends Record { + surface: string; + unique_count: number; + new_count: number; +} + +export interface UniqueDayRow extends Record { + day: number; + surface: string; + unique_count: number; +} + +/** One visitor on one day, with the day they were first seen, for the cohort grids. */ +export interface RetentionRow extends Record { + surface: string; + visitor: string; + first_day: number; + day: number; +} + export interface StatsSnapshotRows { summary: MetricSummaryRow[]; trend: MetricTrendRow[]; @@ -45,6 +78,22 @@ export interface StatsSnapshotRows { clients: BreakdownRow[]; live: LivePresenceRow; collectingSince: number | null; + uniques: UniqueSummaryRow[]; + uniqueDays: UniqueDayRow[]; + retention: RetentionRow[]; + uniquesConfigured: boolean; +} + +/** Midnight UTC of the day that contains `at`. */ +export function dayStart(at: number): number { + return Math.floor(at / DAY_MS) * DAY_MS; +} + +/** Monday 00:00 UTC of the week that contains `at`. */ +export function weekStart(at: number): number { + const day = dayStart(at); + const sinceMonday = (new Date(day).getUTCDay() + 6) % 7; + return day - sinceMonday * DAY_MS; } export function buildStatsSnapshot( @@ -52,6 +101,7 @@ export function buildStatsSnapshot( range: StatsRange, now: number, rangeStart: number, + accounts: StatsAccounts = null, ): StatsSnapshot { const total = (event: string, target?: string): number => rows.summary .filter((row) => row.event === event && (target === undefined || row.target === target)) @@ -64,45 +114,68 @@ export function buildStatsSnapshot( const sessionsStarted = total("session_started"); const sharesOpened = total("share_opened"); const collaborations = total("collaboration_started"); + const landingViews = total("page_view", "landing"); + const docsViews = rows.summary + .filter((row) => row.event === "page_view" && row.target.startsWith("docs")) + .reduce((sum, row) => sum + Number(row.count), 0); + const ctaClicks = total("cta_click"); + const binaryDownloads = total("binary_download"); const trendStepMs = statsTrendStep(range, now - rangeStart); + const uniques = buildUniques(rows); + + const metrics: StatsSnapshot["metrics"] = { + activeSessions: Math.max(0, Number(rows.live.active_sessions)), + activeViewers: Math.max(0, Number(rows.live.active_viewers)), + sessionsCreated, + sessionsStarted, + sharesOpened, + viewerConnections: total("viewer_connected"), + collaborations, + landingViews, + docsViews, + terminalViews: total("page_view", "session"), + notFoundViews: total("page_view", "not_found"), + unknownPaths: total("page_view", "unknown_path"), + ctaClicks, + installs: total("installer_download"), + skillDownloads: total("skill_download"), + binaryDownloads, + copies: total("copy"), + averageDurationSeconds: endedCount === 0 ? 0 : durationSum / endedCount, + longestDurationSeconds: ended.reduce( + (maximum, row) => Math.max(maximum, Number(row.value_max)), + 0, + ), + averagePeakViewers: endedCount === 0 ? 0 : peakViewerSum / endedCount, + maximumPeakViewers: ended.reduce( + (maximum, row) => Math.max(maximum, Number(row.auxiliary_max)), + 0, + ), + }; return { - version: 1, + version: 2, generatedAt: now, collectingSince: rows.collectingSince, range, rangeStart, trendStepMs, - metrics: { - activeSessions: Math.max(0, Number(rows.live.active_sessions)), - activeViewers: Math.max(0, Number(rows.live.active_viewers)), - sessionsCreated, - sessionsStarted, - sharesOpened, - viewerConnections: total("viewer_connected"), - collaborations, - landingViews: total("page_view", "landing"), - terminalViews: total("page_view", "session"), - installs: total("installer_download"), - skillDownloads: total("skill_download"), - binaryDownloads: total("binary_download"), - copies: total("copy"), - averageDurationSeconds: endedCount === 0 ? 0 : durationSum / endedCount, - longestDurationSeconds: ended.reduce( - (maximum, row) => Math.max(maximum, Number(row.value_max)), - 0, - ), - averagePeakViewers: endedCount === 0 ? 0 : peakViewerSum / endedCount, - maximumPeakViewers: ended.reduce( - (maximum, row) => Math.max(maximum, Number(row.auxiliary_max)), - 0, - ), - }, + metrics, rates: { started: ratio(sessionsStarted, sessionsCreated), shared: ratio(sharesOpened, sessionsCreated), collaborated: ratio(collaborations, sessionsCreated), + signup: ratio(ctaClicks, landingViews), + installed: ratio(binaryDownloads, landingViews), }, + funnel: buildFunnel(metrics, uniques), + uniques, + retention: { + weeks: RETENTION_WEEKS, + site: buildRetentionCohorts(rows.retention.filter((row) => row.surface === "site"), now), + cli: buildRetentionCohorts(rows.retention.filter((row) => row.surface === "cli"), now), + }, + accounts, trend: buildTrend(rows.trend, rangeStart, now, trendStepMs), breakdowns: { devices: breakdown(rows.devices), @@ -115,6 +188,7 @@ export function buildStatsSnapshot( ...targetBreakdown(rows.summary, "binary_download"), ].sort((left, right) => right.value - left.value), outcomes: targetBreakdown(rows.summary, "session_ended"), + pages: targetBreakdown(rows.summary, "page_view"), }, targets: rows.summary.map((row): StatsTargetMetric => ({ event: row.event, @@ -128,6 +202,151 @@ export function buildStatsSnapshot( }; } +/* + * The path from a first look to a first keystroke, one row per step, each + * saying what it counts. A step's count is an event total; its unique figure + * is how many distinct people were behind it, on the surfaces that count + * people. The two are shown side by side rather than blended, because a + * hundred page views from one crawler and a hundred visitors are different + * news. + */ +export function buildFunnel( + metrics: StatsSnapshot["metrics"], + uniques: StatsUniques, +): StatsFunnelStep[] { + const people = (surface: UniqueSurface): number | null => + uniques.configured ? uniques.surfaces[surface].unique : null; + return [ + { + key: "visited", + label: "Visited the site", + count: metrics.landingViews + metrics.docsViews, + unique: people("site"), + note: "Landing and documentation page views. Crawlers count as views, not as people.", + basis: null, + }, + { + key: "signup", + label: "Clicked Sign up", + count: metrics.ctaClicks, + unique: null, + note: "Any Sign up free or Web app link on the landing page.", + basis: "visited", + }, + { + key: "installer", + label: "Fetched the installer", + count: metrics.installs, + unique: people("install"), + note: "Requests for the install script. Reading it counts; so does piping it to sh.", + basis: "visited", + }, + { + key: "installed", + label: "Completed an install", + count: metrics.binaryDownloads, + unique: null, + note: "Release binaries served, the installer's last step. Homebrew and source builds are not in this number.", + basis: "installer", + }, + { + key: "session", + label: "Started a session", + count: metrics.sessionsStarted, + unique: people("cli"), + note: "A shell command connected its process to the relay. Not a share of the step before: sessions come from every install to date.", + basis: null, + }, + { + key: "opened", + label: "Opened it in a browser", + count: metrics.sharesOpened, + unique: people("viewer"), + note: "Sessions whose link was opened at least once, by anyone, the owner included.", + basis: "session", + }, + { + key: "typed", + label: "Typed from a browser", + count: metrics.collaborations, + unique: null, + note: "Sessions that received at least one keystroke from a browser.", + basis: "opened", + }, + ]; +} + +/** + * Weekly cohorts from (visitor, first day, day) rows. + * + * A cohort is everyone first seen in one week. For each later week, the count + * is how many of them were seen at all in that week, so the grid reads as + * "of the 40 who arrived that week, 12 were back the week after". The current + * week is included as it stands and grows until it ends. + */ +export function buildRetentionCohorts( + rows: Pick[], + now: number, + weeks = RETENTION_WEEKS, +): StatsRetentionCohort[] { + const thisWeek = weekStart(now); + const earliest = thisWeek - (weeks - 1) * WEEK_MS; + const cohorts = new Map>>(); + for (const row of rows) { + const cohortWeek = weekStart(Number(row.first_day)); + if (cohortWeek < earliest || cohortWeek > thisWeek) continue; + let members = cohorts.get(cohortWeek); + if (!members) { + members = new Map(); + cohorts.set(cohortWeek, members); + } + let active = members.get(row.visitor); + if (!active) { + active = new Set(); + members.set(row.visitor, active); + } + const later = Math.round((weekStart(Number(row.day)) - cohortWeek) / WEEK_MS); + if (later >= 1) active.add(later); + } + return [...cohorts.entries()] + .sort(([left], [right]) => left - right) + .map(([cohortWeek, members]) => { + const span = Math.min(weeks - 1, Math.round((thisWeek - cohortWeek) / WEEK_MS)); + const active = Array.from({ length: span }, (_, index) => { + let count = 0; + for (const weeksActive of members.values()) if (weeksActive.has(index + 1)) count += 1; + return count; + }); + return { weekStart: cohortWeek, size: members.size, active }; + }); +} + +function buildUniques(rows: StatsSnapshotRows): StatsUniques { + const surfaces = Object.fromEntries( + UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0 }]), + ) as Record; + for (const row of rows.uniques) { + if (!isUniqueSurface(row.surface)) continue; + const unique = Number(row.unique_count); + const fresh = Math.min(unique, Number(row.new_count)); + surfaces[row.surface] = { unique, new: fresh, returning: unique - fresh }; + } + const days = new Map(); + for (const row of rows.uniqueDays) { + if (!isUniqueSurface(row.surface)) continue; + const day = Number(row.day); + const point = days.get(day) ?? { day, site: 0, cli: 0, viewer: 0, install: 0 }; + point[row.surface] = Number(row.unique_count); + days.set(day, point); + } + return { + configured: rows.uniquesConfigured, + memoryDays: VISITOR_MEMORY_DAYS, + surfaces, + daily: [...days.values()].sort((left, right) => left.day - right.day), + }; +} + export function statsRangeStart( range: StatsRange, now: number, diff --git a/shared/stats.ts b/shared/stats.ts index 345f66b..3f7e075 100644 --- a/shared/stats.ts +++ b/shared/stats.ts @@ -1,6 +1,20 @@ export const STATS_RANGES = ["24h", "7d", "30d", "all"] as const; export type StatsRange = typeof STATS_RANGES[number]; +/* + * The four places a person can be counted once. A visitor to the site, a + * machine running the CLI, a browser opening a shared terminal, and a machine + * fetching the installer are different people often enough to be kept apart. + */ +export const UNIQUE_SURFACES = ["site", "cli", "viewer", "install"] as const; +export type UniqueSurface = typeof UNIQUE_SURFACES[number]; + +/** How long the dashboard remembers a visitor hash, so "new" has a meaning. */ +export const VISITOR_MEMORY_DAYS = 120; + +/** How many weekly cohorts the retention grids show. */ +export const RETENTION_WEEKS = 8; + export interface StatsSeriesPoint { at: number; sessions: number; @@ -24,8 +38,77 @@ export interface StatsTargetMetric { auxiliaryMaximum: number; } +export interface StatsUniqueCount { + /** Distinct people seen in the range. */ + unique: number; + /** Of those, first seen in the range. */ + new: number; + /** Of those, seen before the range began. */ + returning: number; +} + +export interface StatsUniqueDay { + day: number; + site: number; + cli: number; + viewer: number; + install: number; +} + +export interface StatsUniques { + /** False until the Worker has a visitor salt; every count is then zero. */ + configured: boolean; + memoryDays: number; + surfaces: Record; + daily: StatsUniqueDay[]; +} + +export interface StatsFunnelStep { + key: string; + label: string; + count: number; + /** Distinct people behind the count, when the surface is counted. */ + unique: number | null; + /** What the count is, in one sentence, so nobody has to guess. */ + note: string; + /** + * The step this one is a share of, or null when it is its own population: + * sessions in a range come from every install ever made, not from this + * range's installs, so a percentage there would mislead. + */ + basis: string | null; +} + +/** + * One weekly cohort: everyone first seen in the week starting at weekStart, + * and how many of them were seen again in each later week. active[0] is the + * week after the first; the current, unfinished week is included as it stands. + */ +export interface StatsRetentionCohort { + weekStart: number; + size: number; + active: number[]; +} + +export interface StatsRetention { + weeks: number; + site: StatsRetentionCohort[]; + cli: StatsRetentionCohort[]; +} + +/** Aggregates the accounts app answers with, when the dashboard is linked to it. */ +export interface StatsAccountStats { + total: number; + newInRange: number; + activeInRange: number; + newByDay: { day: number; count: number }[]; + cohorts: StatsRetentionCohort[]; +} + +export type StatsAccounts = StatsAccountStats | { error: string } | null; + export interface StatsSnapshot { - version: 1; + version: 2; generatedAt: number; collectingSince: number | null; range: StatsRange; @@ -40,9 +123,18 @@ export interface StatsSnapshot { viewerConnections: number; collaborations: number; landingViews: number; + docsViews: number; terminalViews: number; + /** Documents answered with a 404. */ + notFoundViews: number; + /** Paths the site did not know but answered anyway, with the landing page. */ + unknownPaths: number; + /** Sign up free and Web app links clicked on the landing page. */ + ctaClicks: number; + /** Requests for the install script, which humans and crawlers both make. */ installs: number; skillDownloads: number; + /** Release binaries served: the installer's last step, so a completed install. */ binaryDownloads: number; copies: number; averageDurationSeconds: number; @@ -54,7 +146,15 @@ export interface StatsSnapshot { started: number; shared: number; collaborated: number; + /** Sign-up clicks per landing view. */ + signup: number; + /** Completed installs per landing view. */ + installed: number; }; + funnel: StatsFunnelStep[]; + uniques: StatsUniques; + retention: StatsRetention; + accounts: StatsAccounts; trend: StatsSeriesPoint[]; breakdowns: { devices: StatsBreakdownItem[]; @@ -63,6 +163,7 @@ export interface StatsSnapshot { copies: StatsBreakdownItem[]; downloads: StatsBreakdownItem[]; outcomes: StatsBreakdownItem[]; + pages: StatsBreakdownItem[]; }; targets: StatsTargetMetric[]; } @@ -70,3 +171,7 @@ export interface StatsSnapshot { export function isStatsRange(value: string | null): value is StatsRange { return STATS_RANGES.includes(value as StatsRange); } + +export function isUniqueSurface(value: string): value is UniqueSurface { + return UNIQUE_SURFACES.includes(value as UniqueSurface); +} diff --git a/tests/analytics.test.ts b/tests/analytics.test.ts index cfa9908..f55717d 100644 --- a/tests/analytics.test.ts +++ b/tests/analytics.test.ts @@ -4,9 +4,14 @@ import { classifyClient, classifyDevice, classifyReferrer, + documentTarget, + hasVisitorSalt, isDocumentNavigation, normalizeAnalyticsRecord, requestAnalyticsContext, + requestVisitor, + uniqueSurface, + visitorKey, writeAnalytics, } from "../worker/analytics"; @@ -92,4 +97,53 @@ describe("analytics", () => { expect(binaryDownloadTarget("/downloads/shell-linux-mips64le")).toBe("linux-mips64le"); expect(binaryDownloadTarget("/downloads/shell-linux-amd64.sha256")).toBeNull(); }); + + it("names every documentation page, and keeps unknown paths apart from 404s", () => { + expect(documentTarget("/", 200, "0.15.1")).toBe("landing"); + expect(documentTarget("/docs/", 200, "0.15.1")).toBe("docs"); + expect(documentTarget("/app/", 200, "0.15.1")).toBe("docs_app"); + expect(documentTarget("/cli/", 200, "0.15.1")).toBe("docs_cli"); + expect(documentTarget("/refstream/", 200, "0.15.1")).toBe("docs_refstream"); + expect(documentTarget("/self-hosting/", 200, "0.15.1")).toBe("docs_self_hosting"); + expect(documentTarget("/docs/v0.15.1/app/", 200, "0.15.1")).toBe("docs_app"); + expect(documentTarget("/docs/v0.15.1/", 200, "0.15.1")).toBe("docs"); + expect(documentTarget("/s/abcdefghijklmnopqrstuvwxyz012345", 200, "0.15.1")).toBe("session"); + expect(documentTarget("/docs/contributing/", 200, "0.15.1")).toBe("unknown_path"); + expect(documentTarget("/docs/contributing/", 404, "0.15.1")).toBe("not_found"); + }); + + it("counts people on the surfaces where a person is behind the event", () => { + expect(uniqueSurface("page_view", "landing")).toBe("site"); + expect(uniqueSurface("page_view", "docs_app")).toBe("site"); + expect(uniqueSurface("page_view", "session")).toBe("viewer"); + expect(uniqueSurface("page_view", "unknown_path")).toBeNull(); + expect(uniqueSurface("cta_click", "signup_hero")).toBe("site"); + expect(uniqueSurface("binary_download", "darwin-arm64")).toBe("install"); + expect(uniqueSurface("session_created", "cli")).toBe("cli"); + expect(uniqueSurface("viewer_connected", "viewer")).toBe("viewer"); + expect(uniqueSurface("session_ended", "task_exit")).toBeNull(); + expect(uniqueSurface("viewer_disconnected", "viewer")).toBeNull(); + }); + + it("hashes a visitor so the address cannot be read back and a browser update is the same person", async () => { + const salt = "a-salt-long-enough-to-count"; + const one = await visitorKey(salt, "203.0.113.7", "Mozilla/5.0 (Macintosh) Chrome/129.0.0.0 Safari/537.36"); + expect(one).toMatch(/^[a-f0-9]{20}$/); + expect(one).not.toContain("203"); + await expect(visitorKey(salt, "203.0.113.7", "Mozilla/5.0 (Macintosh) Chrome/130.0.0.0 Safari/537.36")).resolves.toBe(one); + await expect(visitorKey(salt, "203.0.113.8", "Mozilla/5.0 (Macintosh) Chrome/129.0.0.0 Safari/537.36")).resolves.not.toBe(one); + await expect(visitorKey("another-salt-of-some-length", "203.0.113.7", "Mozilla/5.0 (Macintosh) Chrome/129.0.0.0 Safari/537.36")).resolves.not.toBe(one); + }); + + it("counts nobody without a salt worth the name or without an address", async () => { + const request = new Request("https://shell.online/", { + headers: { "CF-Connecting-IP": "203.0.113.7", "User-Agent": "curl/8.4.0" }, + }); + expect(hasVisitorSalt(undefined)).toBe(false); + expect(hasVisitorSalt("short")).toBe(false); + await expect(requestVisitor(undefined, request)).resolves.toBeUndefined(); + await expect(requestVisitor("short", request)).resolves.toBeUndefined(); + await expect(requestVisitor("a-salt-long-enough-to-count", new Request("https://shell.online/"))).resolves.toBeUndefined(); + await expect(requestVisitor("a-salt-long-enough-to-count", request)).resolves.toMatch(/^[a-f0-9]{20}$/); + }); }); diff --git a/web/main.ts b/web/main.ts index f1de8b6..08c3437 100644 --- a/web/main.ts +++ b/web/main.ts @@ -174,7 +174,7 @@ function renderLanding(): void { - + @@ -184,7 +184,7 @@ function renderLanding(): void {

    Run it here.
    Open it anywhere.

    Run shell <command> on your machine. It gives you a link and password to the same encrypted terminal—open it from any desktop or phone to watch or type.

    - @@ -296,7 +296,7 @@ function renderLanding(): void {
    @@ -415,7 +415,7 @@ function renderLanding(): void {

    Live browser terminals for the work your machine is already doing.Developed by Pilot Protocol.

    • - Counts only. No commands, terminal contents, IPs, session IDs, or user profiles. + Counts and keyed hashes only. No commands, terminal contents, IP addresses, session IDs, or user profiles. v${RELEASE_VERSION} · SHA-256
    @@ -264,7 +265,7 @@ function renderAuthenticatedDashboard(root: HTMLElement): () => void { } if (!response.ok) throw new Error(`Statistics unavailable (${response.status})`); const snapshot = await response.json() as StatsSnapshot; - if (snapshot.version !== 1) throw new Error("Unsupported statistics response"); + if (snapshot.version !== 2) throw new Error("Unsupported statistics response"); renderSnapshot(content, snapshot); content.setAttribute("aria-busy", "false"); const generated = new Date(snapshot.generatedAt); @@ -329,7 +330,20 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { const rangeLabel = snapshot.range === "all" ? "all time" : `last ${snapshot.range}`; const outcomes = snapshot.breakdowns.outcomes; const endedSessions = outcomes.reduce((sum, item) => sum + item.value, 0); + const people = snapshot.uniques; + const site = people.surfaces.site; + const cli = people.surfaces.cli; + const ctaByLink = snapshot.targets + .filter((metric) => metric.event === "cta_click") + .map((metric) => ({ label: metric.target, value: metric.count })) + .sort((left, right) => right.value - left.value); container.innerHTML = ` + ${people.configured ? "" : ` +

    + + People are not being counted yet. Set STATS_VISITOR_SALT (16 or more characters) on the Worker and every unique, new, returning and retention figure below fills in from that moment. Event counts are unaffected. +

    + `}
    ${renderKpi( "Active now", @@ -340,51 +354,64 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { "blue", )} ${renderKpi( - "Sessions created", - metrics.sessionsCreated, - metrics.sessionsCreated === 0 - ? "None created in this range" - : `${formatPercent(snapshot.rates.started)} reached the process`, + "Unique visitors", + people.configured ? site.unique : "—", + people.configured + ? `${integerFormatter.format(site.new)} new · ${integerFormatter.format(site.returning)} returning` + : `${integerFormatter.format(metrics.landingViews)} landing views, people not counted`, + snapshot.trend, + "pageViews", + "silver", + )} + ${renderKpi( + "Installs completed", + metrics.binaryDownloads, + `${integerFormatter.format(metrics.installs)} installer fetch${metrics.installs === 1 ? "" : "es"}`, + snapshot.trend, + "sessions", + "amber", + )} + ${renderKpi( + "Sessions started", + metrics.sessionsStarted, + people.configured + ? `from ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"}, ${integerFormatter.format(cli.new)} new` + : `${integerFormatter.format(metrics.sessionsCreated)} created`, snapshot.trend, "sessions", "violet", )} ${renderKpi( - "Viewer connections", - metrics.viewerConnections, - `${integerFormatter.format(metrics.sharesOpened)} newly shared terminal${metrics.sharesOpened === 1 ? "" : "s"}`, + "Opened in a browser", + metrics.sharesOpened, + metrics.sessionsStarted === 0 + ? "No sessions in this range" + : `${formatPercent(ratio(metrics.sharesOpened, metrics.sessionsStarted))} of sessions started`, snapshot.trend, "shares", "green", )} ${renderKpi( - "Collaborations", + "Typed from a browser", metrics.collaborations, - metrics.sessionsCreated === 0 - ? "No new-session cohort yet" - : `${formatPercent(snapshot.rates.collaborated)} of created sessions`, + metrics.sharesOpened === 0 + ? "No opened sessions yet" + : `${formatPercent(ratio(metrics.collaborations, metrics.sharesOpened))} of opened sessions`, snapshot.trend, "collaborations", "pink", )} - ${renderKpi( - "Terminal opens", - metrics.terminalViews, - `${integerFormatter.format(metrics.landingViews)} landing views`, - snapshot.trend, - "pageViews", - "silver", - )} - ${renderKpi( - "Average lifetime", - formatDuration(metrics.averageDurationSeconds), - `Longest ${formatDuration(metrics.longestDurationSeconds)}`, - snapshot.trend, - "sessions", - "amber", - )}
    +
    +
    +
    From a first look to a first keystroke

    Funnel

    + ${escapeHtml(rangeLabel)} +
    + ${renderFunnel(snapshot)} + ${renderUniquesStrip(snapshot)} +
    +
    @@ -396,28 +423,6 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${renderTimeChart("activity", snapshot.trend, ACTIVITY_SERIES, snapshot.range)}
    -
    -
    -
    New-session cohort

    From command to collaboration

    - ${escapeHtml(rangeLabel)} -
    - ${renderFunnel(snapshot)} -
    -
    - -
    -
    -
    -
    Attention

    Traffic pulse

    - ${integerFormatter.format(metrics.landingViews + metrics.terminalViews)} views -
    - ${renderTimeChart("traffic", snapshot.trend, TRAFFIC_SERIES, snapshot.range, true)} -
    - Landing ${integerFormatter.format(metrics.landingViews)} - Shared terminals ${integerFormatter.format(metrics.terminalViews)} -
    -
    -
    Reliability

    Session outcomes

    @@ -425,29 +430,57 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void {
    ${renderDonut(outcomes)}
    + Average lifetime${formatDuration(metrics.averageDurationSeconds)} + Longest lifetime${formatDuration(metrics.longestDurationSeconds)} Avg peak audience${numberFormatter.format(metrics.averagePeakViewers)} Largest audience${integerFormatter.format(metrics.maximumPeakViewers)}
    +
    + ${renderCohorts( + "Machines that came back", + "CLI hosts by the week they first started a session", + snapshot.retention.cli, + snapshot.retention.weeks, + people.configured, + "No machine has started a session in these weeks yet.", + )} + ${renderCohorts( + "Visitors that came back", + "Site visitors by the week they first arrived", + snapshot.retention.site, + snapshot.retention.weeks, + people.configured, + "No visitor has been counted in these weeks yet.", + )} +
    + + ${renderAccounts(snapshot)} + +
    +
    +
    Attention

    Traffic pulse

    + ${integerFormatter.format(metrics.landingViews + metrics.docsViews + metrics.terminalViews)} views +
    + ${renderTimeChart("traffic", snapshot.trend, TRAFFIC_SERIES, snapshot.range, true)} +
    + Landing ${integerFormatter.format(metrics.landingViews)} + Docs ${integerFormatter.format(metrics.docsViews)} + Shared terminals ${integerFormatter.format(metrics.terminalViews)} + Unknown paths ${integerFormatter.format(metrics.unknownPaths)} + 404s ${integerFormatter.format(metrics.notFoundViews)} +
    +
    +
    ${renderBreakdown("Acquisition", "Where landing visits came from", snapshot.breakdowns.referrers, "referrer")} + ${renderBreakdown("Pages", "Document views by page", snapshot.breakdowns.pages, "page")} + ${renderBreakdown("Sign-up clicks", "Which link was clicked", ctaByLink, "cta")} ${renderBreakdown("Devices", "Browsers opening shell.online", snapshot.breakdowns.devices, "device")} ${renderBreakdown("CLI clients", "Versions creating sessions", snapshot.breakdowns.clients, "client")} - ${renderBreakdown("Copy actions", "Commands and links copied", snapshot.breakdowns.copies, "copy")} - ${renderBreakdown("Delivery requests", "Install script, skill, and binary requests", snapshot.breakdowns.downloads, "download")} -
    -
    -
    Distribution

    Delivery totals

    -
    -
    - Install script requests${integerFormatter.format(metrics.installs)} - Agent skill${integerFormatter.format(metrics.skillDownloads)} - Release binaries${integerFormatter.format(metrics.binaryDownloads)} - All copies${integerFormatter.format(metrics.copies)} -
    -
    + ${renderBreakdown("Delivery", "Installer, skill and binary requests", snapshot.breakdowns.downloads, "download")}
    @@ -473,6 +506,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void {
    Showing ${escapeHtml(rangeLabel)}. ${snapshot.collectingSince ? `Collecting exact dashboard metrics since ${formatDate(snapshot.collectingSince)}.` : "Waiting for the first event."} + People are keyed hashes of address and browser family, forgotten ${people.memoryDays} days after they were last seen; a person seen again after that counts as new.
    `; @@ -568,24 +602,26 @@ function renderTimeChart( } function renderFunnel(snapshot: StatsSnapshot): string { - const values = [ - { label: "Created", value: snapshot.metrics.sessionsCreated, color: "#8eafff" }, - { label: "Started", value: snapshot.metrics.sessionsStarted, color: "#819de5" }, - { label: "First opened", value: snapshot.metrics.sharesOpened, color: "#75dac2" }, - { label: "Collaborated", value: snapshot.metrics.collaborations, color: "#d7a6ff" }, - ]; - const maximum = Math.max(1, values[0].value, ...values.map((item) => item.value)); + const steps = snapshot.funnel; + const colors = ["#9ab7e8", "#8eafff", "#819de5", "#f4bd78", "#8eafff", "#75dac2", "#d7a6ff"]; + const maximum = Math.max(1, ...steps.map((step) => step.count)); return ` -
    - ${values.map((item, index) => { - const width = item.value === 0 ? 2 : Math.max(7, item.value / maximum * 100); - const previous = index === 0 ? item.value : values[index - 1].value; +
    + ${steps.map((step, index) => { + const width = step.count === 0 ? 1.5 : Math.max(4, step.count / maximum * 100); + const basis = step.basis === null ? null : steps.find((candidate) => candidate.key === step.basis) ?? null; + const share = basis === null + ? (index === 0 ? "the whole path starts here" : "its own population") + : basis.count === 0 + ? `no ${basis.label.toLowerCase()} to compare with` + : `${formatPercent(ratio(step.count, basis.count))} of ${basis.label.toLowerCase()}`; return ` -
    - ${item.label} -
    - ${integerFormatter.format(item.value)} - ${index === 0 ? "baseline" : `${formatPercent(ratio(item.value, previous))} step`} +
    + ${escapeHtml(step.label)} + ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}`} +
    + ${escapeHtml(share)} + ${escapeHtml(step.note)}
    `; }).join("")} @@ -593,6 +629,115 @@ function renderFunnel(snapshot: StatsSnapshot): string { `; } +function renderUniquesStrip(snapshot: StatsSnapshot): string { + const people = snapshot.uniques; + const cell = (label: string, surface: keyof typeof people.surfaces): string => { + const count = people.surfaces[surface]; + return ` + + ${escapeHtml(label)} + ${people.configured ? integerFormatter.format(count.unique) : "—"} + ${people.configured ? `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} back` : "not counted"} + + `; + }; + return ` +
    + ${cell("Visitors", "site")} + ${cell("Installers", "install")} + ${cell("CLI machines", "cli")} + ${cell("Viewers", "viewer")} +
    + `; +} + +/* + * A cohort grid: one row per week of first arrivals, one column per later + * week, each cell the share of that cohort seen in that week. Cells for weeks + * that have not happened yet are left blank rather than drawn as zero. + */ +function renderCohorts( + title: string, + kicker: string, + cohorts: StatsRetentionCohort[], + weeks: number, + configured: boolean, + empty: string, +): string { + const later = weeks - 1; + const columns = `78px 54px repeat(${later}, minmax(34px, 1fr))`; + const head = [ + 'Week of', + 'People', + ...Array.from({ length: later }, (_, index) => `+${index + 1}`), + ].join(""); + const rows = cohorts.map((cohort) => { + const cells = Array.from({ length: later }, (_, index) => { + if (index >= cohort.active.length) return ''; + const share = cohort.size === 0 ? 0 : cohort.active[index] / cohort.size; + if (cohort.active[index] === 0) return '0%'; + return `${formatPercent(share)}`; + }).join(""); + return `${escapeHtml(formatWeek(cohort.weekStart))}${integerFormatter.format(cohort.size)}${cells}`; + }).join(""); + return ` +
    +
    +
    ${escapeHtml(kicker)}

    ${escapeHtml(title)}

    + ${weeks} weeks +
    + ${!configured + ? '

    Not counted until the Worker has a visitor salt.

    ' + : cohorts.length === 0 + ? `

    ${escapeHtml(empty)}

    ` + : `
    ${head}${rows}
    `} +
    + `; +} + +function renderAccounts(snapshot: StatsSnapshot): string { + const accounts = snapshot.accounts; + if (accounts === null) return ""; + if ("error" in accounts) { + return ` +

    + + Account figures are linked but unavailable: ${escapeHtml(accounts.error)}. Check APP_STATS_URL and APP_STATS_TOKEN on the Worker and STATS_TOKEN on the app. +

    + `; + } + const rangeLabel = snapshot.range === "all" ? "all time" : `last ${snapshot.range}`; + return ` +
    +
    +
    +
    Exact, from the accounts app

    Accounts

    + ${escapeHtml(rangeLabel)} +
    +
    + Accounts${integerFormatter.format(accounts.total)} + New${integerFormatter.format(accounts.newInRange)} + Active${integerFormatter.format(accounts.activeInRange)} +
    +
    ${renderSparkline(accounts.newByDay.map((point) => point.count))}
    +

    New accounts per day. Active means the account used the app in the range.

    +
    + ${renderCohorts( + "Accounts that came back", + "Accounts by the week they signed up", + accounts.cohorts, + snapshot.retention.weeks, + true, + "No account has signed up in these weeks yet.", + )} +
    + `; +} + +function formatWeek(weekStart: number): string { + return new Date(weekStart).toLocaleDateString([], { month: "short", day: "numeric", timeZone: "UTC" }); +} + function renderDonut(items: StatsBreakdownItem[]): string { const total = items.reduce((sum, item) => sum + item.value, 0); const colors = ["#8eafff", "#75dac2", "#d7a6ff", "#f4bd78", "#6d7992"]; @@ -760,6 +905,40 @@ function humanize(value: string): string { disconnected_timeout: "Disconnected timeout", never_started: "Never started", remote_input: "Remote input", + cta_click: "Sign-up click", + signup_nav: "Sign up free (nav)", + signup_hero: "Sign up free (hero)", + signup_team: "Manage a team", + signup_footer: "Web app (footer)", + not_found: "Not found (404)", + unknown_path: "Unknown path (served the landing page)", + landing: "Landing", + docs: "Docs", + docs_app: "Docs · Web app", + docs_cli: "Docs · CLI", + docs_platforms: "Docs · Platforms", + docs_mobile: "Docs · Mobile", + docs_refstream: "Docs · Refstream", + docs_reliability: "Docs · Reliability", + docs_security: "Docs · Security", + docs_e2ee: "Docs · E2EE", + docs_docker: "Docs · Docker", + docs_self_hosting: "Docs · Self-hosting", + session: "Shared terminal", + installer_download: "Installer fetched", + binary_download: "Install completed", + share_opened: "Opened in a browser", + viewer_connected: "Viewer connected", + viewer_disconnected: "Viewer disconnected", + collaboration_started: "Typed from a browser", + session_created: "Session created", + session_started: "Session started", + session_ended: "Session ended", + page_view: "Page view", + skill_download: "Skill fetched", + stats_view: "Dashboard view", + posix: "Install script (POSIX)", + powershell: "Install script (PowerShell)", darwin_arm64: "macOS arm64", darwin_amd64: "macOS amd64", linux_arm64: "Linux arm64", diff --git a/worker/analytics.ts b/worker/analytics.ts index 872b5dd..fc7da5a 100644 --- a/worker/analytics.ts +++ b/worker/analytics.ts @@ -1,6 +1,10 @@ +import { resolveDocumentationRoute } from "../shared/documentation"; +import type { UniqueSurface } from "../shared/stats"; + export type AnalyticsEvent = | "page_view" | "copy" + | "cta_click" | "installer_download" | "binary_download" | "skill_download" @@ -29,6 +33,13 @@ export interface AnalyticsContext { referrer?: string; value?: number; auxiliary?: number; + /** + * A keyed hash standing in for the person behind the request, so the + * private dashboard can count how many there were and how many came back. + * Goes to the dashboard's own store only; see writeAnalytics, which never + * sends it to Analytics Engine. + */ + visitor?: string; } export interface AnalyticsRecord { @@ -42,14 +53,26 @@ export interface AnalyticsRecord { auxiliary: number; } +/** What the landing page reports when one of its sign-up links is clicked. */ +export const CTA_TARGETS: ReadonlySet = new Set(["signup_nav", "signup_hero", "signup_team", "signup_footer"]); + +/** What the landing page reports when a command or link is copied. */ +export const COPY_TARGETS: ReadonlySet = new Set(["install", "brew_install", "source_build", "run", "share", "skill"]); + +/** A share page: /s/ and a session id. */ +const SESSION_PATH = /^\/s\/[A-Za-z0-9_-]{32}\/?$/; + +/** Shorter salts are guessable, and a guessable salt turns hashes back into addresses. */ +export const MIN_VISITOR_SALT_LENGTH = 16; + /** * Analytics Engine fields are deliberately fixed and low-cardinality: * * blob1 event, blob2 target, blob3 device, blob4 client, blob5 referrer * double1 count, double2 value/duration, double3 auxiliary/peak viewers * - * No session IDs, terminal contents, commands, URLs, IPs, or raw user agents - * are written to analytics. + * No session IDs, terminal contents, commands, URLs, IPs, raw user agents, or + * visitor hashes are written to analytics. */ export function writeAnalytics( dataset: AnalyticsDataset, @@ -107,6 +130,79 @@ export function requestAnalyticsContext(request: Request): AnalyticsContext { }; } +export function hasVisitorSalt(salt: unknown): salt is string { + return typeof salt === "string" && salt.length >= MIN_VISITOR_SALT_LENGTH; +} + +/** + * The hash that stands in for a visitor: SHA-256 over a secret, the address + * and the browser family. The family is the user agent with its version + * numbers removed, so a browser update does not make a new visitor. Without + * the secret the hash cannot be turned back into an address, and the secret + * never leaves the Worker. Twenty hex characters is eighty bits: enough to + * keep visitors apart, small enough to keep by the thousand. + */ +export async function visitorKey(salt: string, address: string, userAgent: string): Promise { + const family = userAgent.replace(/\d+/g, "").replace(/\s+/g, " ").trim().slice(0, 200); + const bytes = new TextEncoder().encode(`${salt}\n${address}\n${family}`); + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); + return Array.from(digest.subarray(0, 10), (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +/** The visitor hash for a request, or nothing when the Worker has no salt or the edge sent no address. */ +export async function requestVisitor(salt: unknown, request: Request): Promise { + if (!hasVisitorSalt(salt)) return undefined; + const address = request.headers.get("CF-Connecting-IP"); + if (!address) return undefined; + return visitorKey(salt, address, request.headers.get("User-Agent") ?? ""); +} + +/** + * Which surface an event counts a person on, or null for events that do not + * count people: a session ending is the same machine that started it, and a + * viewer disconnecting is the same browser that connected. + */ +export function uniqueSurface(event: AnalyticsEvent, target: string): UniqueSurface | null { + switch (event) { + case "page_view": + if (target === "session") return "viewer"; + if (target === "not_found" || target === "unknown_path") return null; + return "site"; + case "copy": + case "cta_click": + return "site"; + case "installer_download": + case "binary_download": + return "install"; + case "session_created": + return "cli"; + case "viewer_connected": + return "viewer"; + default: + return null; + } +} + +/** + * What a document request counts as. + * + * Every documentation route, current or versioned, has its own target, so a + * page added to the docs is counted the day it ships rather than falling + * through. A 404 is "not_found". Anything else the site answered without + * knowing the path is "unknown_path": the assets binding serves the landing + * page for those, so they are not lost visitors but they are not the landing + * page either, and the count says how often a link points somewhere the site + * should answer properly. + */ +export function documentTarget(pathname: string, status: number, currentVersion: string): string { + if (status === 404) return "not_found"; + if (pathname === "/") return "landing"; + const route = resolveDocumentationRoute(pathname, currentVersion); + if (route) return route.kind === "docs" ? "docs" : `docs_${route.kind.replace(/-/g, "_")}`; + if (SESSION_PATH.test(pathname)) return "session"; + return "unknown_path"; +} + export function isDocumentNavigation(request: Request): boolean { if (request.method !== "GET") return false; if (request.headers.get("Sec-Fetch-Dest") === "document") return true; diff --git a/worker/index.ts b/worker/index.ts index f614d98..7c53680 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -4,12 +4,17 @@ import { binaryDownloadTarget, isDocumentNavigation, requestAnalyticsContext, + requestVisitor, + hasVisitorSalt, + documentTarget, + CTA_TARGETS, + COPY_TARGETS, writeAnalytics, type AnalyticsContext, type AnalyticsEvent, type DeviceClass, } from "./analytics"; -import { isStatsRange } from "../shared/stats"; +import { isStatsRange, type StatsAccountStats, type StatsAccounts, type StatsRange } from "../shared/stats"; import { RELEASE_VERSION } from "../shared/release"; import { downloadAssetIsSpaFallback } from "../shared/download-assets"; import { viewerFrameAction } from "../shared/session-access"; @@ -30,7 +35,7 @@ import { readGitHubApiStarCount, } from "../shared/github"; import { STATS_PRESENCE_REFRESH_MS } from "../shared/stats-snapshot"; -import { isVersionedDocumentationPath } from "../shared/documentation"; +import { isVersionedDocumentationPath, resolveDocumentationRoute } from "../shared/documentation"; import { fetchStatsSnapshot, removeStatsPresence, @@ -83,6 +88,17 @@ interface Env { STATS_PASSWORD: string; ANALYTICS: AnalyticsEngineDataset; ASSETS: Fetcher; + /** + * Secret behind the visitor hashes the dashboard counts people with. Absent + * or short, nobody is counted and the dashboard says so. See requestVisitor. + */ + STATS_VISITOR_SALT?: string; + /** + * The accounts app, and the token it expects, for the account figures on the + * dashboard. Both absent means the accounts panel is left out. + */ + APP_STATS_URL?: string; + APP_STATS_TOKEN?: string; } interface SessionMeta { @@ -221,7 +237,7 @@ export default { const skillUrl = new URL("/skill/shell-online/SKILL.md", url.origin); const skillAsset = await env.ASSETS.fetch(skillUrl); const response = secureAssetResponse(skillAsset, "/skill", url.hostname); - recordAssetAnalytics(request, env, url, response, executionContext); + executionContext.waitUntil(recordAssetAnalytics(request, env, url, response, executionContext)); return response; } @@ -267,7 +283,7 @@ export default { }); } const response = secureAssetResponse(assetResponse, url.pathname, url.hostname); - recordAssetAnalytics(request, env, url, response, executionContext); + executionContext.waitUntil(recordAssetAnalytics(request, env, url, response, executionContext)); return response; }, } satisfies ExportedHandler; @@ -453,16 +469,53 @@ async function handleStatsRequest(request: Request, env: Env, url: URL): Promise return secureStatsResponse(json({ error: "authentication required" }, 401)); } const requestedRange = url.searchParams.get("range"); - const response = await fetchStatsSnapshot( - env.STATS, - isStatsRange(requestedRange) ? requestedRange : "7d", - ); - return secureStatsResponse(response); + const range: StatsRange = isStatsRange(requestedRange) ? requestedRange : "7d"; + const [snapshotResponse, accounts] = await Promise.all([ + fetchStatsSnapshot(env.STATS, range, hasVisitorSalt(env.STATS_VISITOR_SALT)), + fetchAccountStats(env, range), + ]); + if (!snapshotResponse.ok) return secureStatsResponse(snapshotResponse); + const snapshot = await snapshotResponse.json>(); + return secureStatsResponse(json({ ...snapshot, accounts })); } return secureStatsResponse(json({ error: "not found" }, 404)); } +/* + * The accounts app keeps the only exact count of people: accounts. It answers + * aggregates -- how many, how many new, how many back each week -- to a bearer + * token, and nothing per person. Left out when it is not linked, and reported + * as unavailable rather than left out when it is linked and does not answer, + * so a broken link is visible on the dashboard rather than silent. + */ +async function fetchAccountStats(env: Env, range: StatsRange): Promise { + const base = env.APP_STATS_URL?.trim(); + const token = env.APP_STATS_TOKEN?.trim(); + if (!base || !token) return null; + try { + const response = await fetch(`${base.replace(/\/+$/, "")}/api/stats/accounts?range=${range}`, { + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + signal: AbortSignal.timeout(4_000), + }); + if (!response.ok) return { error: `accounts app answered ${response.status}` }; + const body = await response.json(); + return isAccountStats(body) ? body : { error: "accounts app answered in an unexpected shape" }; + } catch { + return { error: "accounts app did not answer" }; + } +} + +function isAccountStats(value: unknown): value is StatsAccountStats { + if (typeof value !== "object" || value === null) return false; + const candidate = value as Record; + return typeof candidate.total === "number" && + typeof candidate.newInRange === "number" && + typeof candidate.activeInRange === "number" && + Array.isArray(candidate.newByDay) && + Array.isArray(candidate.cohorts); +} + function statsPassword(env: Env): string | null { return typeof env.STATS_PASSWORD === "string" && env.STATS_PASSWORD.length >= 12 ? env.STATS_PASSWORD @@ -484,19 +537,32 @@ function recordAnalytics( waitUntilContext.waitUntil(submitStatsEvent(env.STATS, event, target, analyticsContext)); } -function recordAssetAnalytics( +async function recordAssetAnalytics( request: Request, env: Env, url: URL, response: Response, executionContext: ExecutionContext, -): void { - if (request.method !== "GET" || !response.ok) return; +): Promise { + if (request.method !== "GET") return; const context = requestAnalyticsContext(request); + const withVisitor = async (): Promise => ({ + ...context, + visitor: await requestVisitor(env.STATS_VISITOR_SALT, request), + }); + + if (!response.ok) { + /* A document that got a 404 is worth counting; a missing asset is noise. */ + if (response.status === 404 && isDocumentNavigation(request) && !isStatsHostname(url.hostname)) { + recordAnalytics(env, executionContext, "page_view", "not_found", context); + } + return; + } + if (url.pathname === "/install" || url.pathname === "/install.ps1") { const target = url.pathname === "/install.ps1" ? "powershell" : "posix"; - recordAnalytics(env, executionContext, "installer_download", target, context); + recordAnalytics(env, executionContext, "installer_download", target, await withVisitor()); return; } if (url.pathname === "/skill" || url.pathname === "/skill/") { @@ -506,7 +572,7 @@ function recordAssetAnalytics( const binaryTarget = binaryDownloadTarget(url.pathname); if (binaryTarget) { - recordAnalytics(env, executionContext, "binary_download", binaryTarget, context); + recordAnalytics(env, executionContext, "binary_download", binaryTarget, await withVisitor()); return; } @@ -515,23 +581,9 @@ function recordAssetAnalytics( recordAnalytics(env, executionContext, "stats_view", "dashboard", context); return; } - const documentTarget = new Map([ - ["/", "landing"], - ["/docs/", "docs"], - ["/platforms/", "docs_platforms"], - ["/mobile/", "docs_mobile"], - ["/reliability/", "docs_reliability"], - ["/security/", "docs_security"], - ["/e2ee/", "docs_e2ee"], - ["/docker/", "docs_docker"], - ["/self-hosting/", "docs_self_hosting"], - ]).get(url.pathname); - const target = documentTarget ?? ( - SESSION_ID_PATTERN.test(url.pathname.replace(/^\/s\//, "").replace(/\/$/, "")) - ? "session" - : "not_found" - ); - recordAnalytics(env, executionContext, "page_view", target, context); + const target = documentTarget(url.pathname, response.status, RELEASE_VERSION); + /* An unknown path is as likely a crawler as a person, so it counts no visitor. */ + recordAnalytics(env, executionContext, "page_view", target, target === "unknown_path" ? context : await withVisitor()); } async function recordEvent( @@ -561,26 +613,22 @@ async function recordEvent( return json({ error: "invalid event" }, 400); } - if ( - body.event !== "copy" || - ( - body.target !== "install" && - body.target !== "brew_install" && - body.target !== "source_build" && - body.target !== "run" && - body.target !== "share" && - body.target !== "skill" - ) - ) { + const event = body.event; + const target = body.target; + const known = typeof target === "string" && ( + (event === "copy" && COPY_TARGETS.has(target)) || + (event === "cta_click" && CTA_TARGETS.has(target)) + ); + if (!known) { return json({ error: "invalid event" }, 400); } recordAnalytics( env, executionContext, - "copy", - body.target, - requestAnalyticsContext(request), + event, + target, + { ...requestAnalyticsContext(request), visitor: await requestVisitor(env.STATS_VISITOR_SALT, request) }, ); return new Response(null, { @@ -658,7 +706,7 @@ async function createSession( executionContext, "session_created", "cli", - requestAnalyticsContext(request), + { ...requestAnalyticsContext(request), visitor: await requestVisitor(env.STATS_VISITOR_SALT, request) }, ); const origin = requestOrigin(request, url); @@ -726,7 +774,10 @@ async function resumeSession( if (!resumed.ok) return json({ error: resumed.status === 403 ? "persistent credentials rejected" : "could not resume session" }, resumed.status); const resumeResult = await resumed.json<{ created?: unknown }>(); if (resumeResult.created === true) { - recordAnalytics(env, executionContext, "session_created", "persistent_cli", requestAnalyticsContext(request)); + recordAnalytics(env, executionContext, "session_created", "persistent_cli", { + ...requestAnalyticsContext(request), + visitor: await requestVisitor(env.STATS_VISITOR_SALT, request), + }); } const origin = requestOrigin(request, url); return json({ @@ -904,6 +955,13 @@ export class TerminalSession extends DurableObject { const server = pair[1]; const guestNumber = role === "viewer" ? this.nextGuestNumber() : undefined; const analyticsContext = requestAnalyticsContext(request); + /* + * A viewer is a person to count once; the host is the machine already + * counted when its session was created, so it carries no visitor hash. + */ + const viewerContext: AnalyticsContext = role === "viewer" + ? { ...analyticsContext, visitor: await requestVisitor(this.env.STATS_VISITOR_SALT, request) } + : analyticsContext; const attachment: SocketAttachment = { role, id: role === "viewer" ? randomUint32() : 0, @@ -948,9 +1006,9 @@ export class TerminalSession extends DurableObject { if (firstShareOpen) this.meta.shareOpenedAt = Date.now(); this.meta.peakViewers = Math.max(this.meta.peakViewers ?? 0, viewerCount, 1); await this.persistMeta(); - recordAnalytics(this.env, this.state, "viewer_connected", "viewer", analyticsContext); + recordAnalytics(this.env, this.state, "viewer_connected", "viewer", viewerContext); if (firstShareOpen) { - recordAnalytics(this.env, this.state, "share_opened", "viewer", analyticsContext); + recordAnalytics(this.env, this.state, "share_opened", "viewer", viewerContext); } await this.refreshLivePresence(true); await this.scheduleNextAlarm(); @@ -1699,7 +1757,8 @@ function secureAssetResponse(response: Response, pathname: string, hostname: str } function isPublicDocumentPath(pathname: string): boolean { - return pathname === "/" || pathname === "/docs/" || pathname === "/mobile/" || pathname === "/reliability/" || pathname === "/security/" || pathname === "/e2ee/" || pathname === "/docker/" || pathname === "/self-hosting/"; + if (pathname === "/") return true; + return !isVersionedDocumentationPath(pathname) && resolveDocumentationRoute(pathname, RELEASE_VERSION) !== null; } function secureStatsResponse(response: Response): Response { diff --git a/worker/stats-store.test.ts b/worker/stats-store.test.ts index 4d39367..628d83d 100644 --- a/worker/stats-store.test.ts +++ b/worker/stats-store.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; import { + buildRetentionCohorts, buildStatsSnapshot, + DAY_MS, STATS_PRESENCE_LEASE_MS, STATS_PRESENCE_REFRESH_MS, + WEEK_MS, + weekStart, } from "../shared/stats-snapshot"; describe("statistics live presence", () => { @@ -21,6 +25,10 @@ describe("statistics live presence", () => { clients: [], live: { active_sessions: 1, active_viewers: 3 }, collectingSince, + uniques: [], + uniqueDays: [], + retention: [], + uniquesConfigured: false, }, "all", now, collectingSince); expect(snapshot.metrics).toMatchObject({ @@ -44,6 +52,10 @@ describe("statistics live presence", () => { clients: [], live: { active_sessions: 0, active_viewers: 0 }, collectingSince: now - 60_000, + uniques: [], + uniqueDays: [], + retention: [], + uniquesConfigured: false, }, "24h", now, now - 24 * 60 * 60 * 1_000); expect(snapshot.metrics.activeSessions).toBe(0); @@ -52,6 +64,124 @@ describe("statistics live presence", () => { }); }); +describe("people and the funnel", () => { + const now = Date.UTC(2026, 8, 14, 12); + const rangeStart = now - 7 * DAY_MS; + const rows = { + summary: [ + metric("page_view", "landing", 969), + metric("page_view", "docs_app", 30), + metric("page_view", "docs", 18), + metric("page_view", "unknown_path", 57), + metric("page_view", "not_found", 3), + metric("cta_click", "signup_hero", 12), + metric("installer_download", "posix", 62), + metric("binary_download", "darwin-arm64", 4), + metric("session_created", "cli", 48), + metric("session_started", "cli", 48), + metric("share_opened", "viewer", 6), + metric("collaboration_started", "remote_input", 5), + ], + trend: [], + devices: [], + referrers: [], + clients: [], + live: { active_sessions: 0, active_viewers: 0 }, + collectingSince: now - 30 * DAY_MS, + uniques: [ + { surface: "site", unique_count: 400, new_count: 350 }, + { surface: "cli", unique_count: 9, new_count: 2 }, + { surface: "viewer", unique_count: 5, new_count: 5 }, + { surface: "install", unique_count: 40, new_count: 38 }, + ], + uniqueDays: [ + { day: now - 2 * DAY_MS, surface: "site", unique_count: 120 }, + { day: now - 2 * DAY_MS, surface: "cli", unique_count: 4 }, + { day: now - DAY_MS, surface: "site", unique_count: 140 }, + ], + retention: [], + uniquesConfigured: true, + }; + + it("keeps docs, unknown paths and 404s apart and counts people beside events", () => { + const snapshot = buildStatsSnapshot(rows, "7d", now, rangeStart); + expect(snapshot.version).toBe(2); + expect(snapshot.metrics).toMatchObject({ + landingViews: 969, + docsViews: 48, + unknownPaths: 57, + notFoundViews: 3, + ctaClicks: 12, + binaryDownloads: 4, + }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50 }); + expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7 }); + expect(snapshot.uniques.daily.map((day) => [day.site, day.cli])).toEqual([[120, 4], [140, 0]]); + expect(snapshot.rates.signup).toBeCloseTo(12 / 969); + expect(snapshot.breakdowns.pages.map((page) => page.label)).toEqual(["landing", "unknown_path", "docs_app", "docs", "not_found"]); + }); + + it("lays the funnel out from a first look to a first keystroke", () => { + const snapshot = buildStatsSnapshot(rows, "7d", now, rangeStart); + expect(snapshot.funnel.map((step) => [step.key, step.count, step.unique])).toEqual([ + ["visited", 1017, 400], + ["signup", 12, null], + ["installer", 62, 40], + ["installed", 4, null], + ["session", 48, 9], + ["opened", 6, 5], + ["typed", 5, null], + ]); + for (const step of snapshot.funnel) expect(step.note.length).toBeGreaterThan(20); + expect(snapshot.funnel.map((step) => step.basis)).toEqual([null, "visited", "visited", "installer", null, "session", "opened"]); + }); + + it("says when nobody is being counted", () => { + const snapshot = buildStatsSnapshot({ ...rows, uniques: [], uniquesConfigured: false }, "7d", now, rangeStart); + expect(snapshot.uniques.configured).toBe(false); + expect(snapshot.funnel[0].unique).toBeNull(); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0 }); + }); +}); + +describe("retention cohorts", () => { + /* A Monday, so the weeks are easy to read. */ + const monday = Date.UTC(2026, 8, 7); + const now = monday + 2 * WEEK_MS + 3 * DAY_MS; + + it("starts weeks on Monday", () => { + expect(weekStart(monday + 6 * DAY_MS + 5 * 60 * 60_000)).toBe(monday); + expect(weekStart(monday - 1)).toBe(monday - WEEK_MS); + }); + + it("counts each person once per later week, and leaves unfinished weeks to grow", () => { + const rows = [ + /* a: arrived week 0, back in week 1 twice and week 2 */ + { visitor: "a", first_day: monday, day: monday }, + { visitor: "a", first_day: monday, day: monday + WEEK_MS }, + { visitor: "a", first_day: monday, day: monday + WEEK_MS + DAY_MS }, + { visitor: "a", first_day: monday, day: monday + 2 * WEEK_MS + DAY_MS }, + /* b: arrived week 0, never back */ + { visitor: "b", first_day: monday + 2 * DAY_MS, day: monday + 2 * DAY_MS }, + /* c: arrived week 1, back in week 2 */ + { visitor: "c", first_day: monday + WEEK_MS + 3 * DAY_MS, day: monday + WEEK_MS + 3 * DAY_MS }, + { visitor: "c", first_day: monday + WEEK_MS + 3 * DAY_MS, day: monday + 2 * WEEK_MS }, + /* d: arrived this week */ + { visitor: "d", first_day: now, day: now }, + ]; + expect(buildRetentionCohorts(rows, now, 8)).toEqual([ + { weekStart: monday, size: 2, active: [1, 1] }, + { weekStart: monday + WEEK_MS, size: 1, active: [1] }, + { weekStart: monday + 2 * WEEK_MS, size: 1, active: [] }, + ]); + }); + + it("ignores cohorts older than the grid shows", () => { + const old = { visitor: "z", first_day: monday - 12 * WEEK_MS, day: monday }; + expect(buildRetentionCohorts([old], now, 8)).toEqual([]); + }); +}); + function metric( event: string, target: string, diff --git a/worker/stats-store.ts b/worker/stats-store.ts index 209ffc7..a5f68cd 100644 --- a/worker/stats-store.ts +++ b/worker/stats-store.ts @@ -1,19 +1,29 @@ import { DurableObject } from "cloudflare:workers"; import { + RETENTION_WEEKS, + VISITOR_MEMORY_DAYS, isStatsRange, type StatsRange, } from "../shared/stats"; import { buildStatsSnapshot, + DAY_MS, + dayStart, STATS_PRESENCE_LEASE_MS, statsRangeStart, + WEEK_MS, + weekStart, type BreakdownRow, type LivePresenceRow, type MetricSummaryRow, type MetricTrendRow, + type RetentionRow, + type UniqueDayRow, + type UniqueSummaryRow, } from "../shared/stats-snapshot"; import { normalizeAnalyticsRecord, + uniqueSurface, type AnalyticsContext, type AnalyticsEvent, type AnalyticsRecord, @@ -24,6 +34,7 @@ const STATS_OBJECT_NAME = "shell-online-global-stats"; const ANALYTICS_EVENTS = new Set([ "page_view", "copy", + "cta_click", "installer_download", "binary_download", "skill_download", @@ -54,7 +65,7 @@ export async function submitStatsEvent( { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ ...record, at: Date.now() }), + body: JSON.stringify({ ...record, at: Date.now(), visitor: context.visitor }), }, ); if (!response.ok) throw new Error(`Stats store rejected event: ${response.status}`); @@ -66,9 +77,10 @@ export async function submitStatsEvent( export function fetchStatsSnapshot( namespace: DurableObjectNamespace, range: StatsRange, + uniquesConfigured: boolean, ): Promise { return namespace.getByName(STATS_OBJECT_NAME).fetch( - `https://stats.internal/internal/stats?range=${range}`, + `https://stats.internal/internal/stats?range=${range}&uniques=${uniquesConfigured ? "1" : "0"}`, ); } @@ -144,6 +156,31 @@ export class StatsStore extends DurableObject> { ) `); this.sql.exec("CREATE INDEX IF NOT EXISTS live_presence_expiry ON live_presence(expires_at)"); + /* + * Who was seen, as keyed hashes only: one row per person per day per + * surface, and one row per person with their first and last day. Both are + * forgotten VISITOR_MEMORY_DAYS after the person was last seen, so "new" + * means "not seen in that long" and nothing older than that is kept. + */ + this.sql.exec(` + CREATE TABLE IF NOT EXISTS visitor_days ( + surface TEXT NOT NULL, + visitor TEXT NOT NULL, + day INTEGER NOT NULL, + PRIMARY KEY (surface, visitor, day) + ) + `); + this.sql.exec("CREATE INDEX IF NOT EXISTS visitor_days_day ON visitor_days(day)"); + this.sql.exec(` + CREATE TABLE IF NOT EXISTS visitors ( + surface TEXT NOT NULL, + visitor TEXT NOT NULL, + first_day INTEGER NOT NULL, + last_day INTEGER NOT NULL, + PRIMARY KEY (surface, visitor) + ) + `); + this.sql.exec("CREATE INDEX IF NOT EXISTS visitors_last_day ON visitors(last_day)"); } async fetch(request: Request): Promise { @@ -153,7 +190,10 @@ export class StatsStore extends DurableObject> { } if (url.pathname === "/internal/stats" && request.method === "GET") { const requestedRange = url.searchParams.get("range"); - return this.snapshot(isStatsRange(requestedRange) ? requestedRange : "7d"); + return this.snapshot( + isStatsRange(requestedRange) ? requestedRange : "7d", + url.searchParams.get("uniques") === "1", + ); } if (url.pathname === "/internal/presence" && request.method === "POST") { return this.updatePresence(request); @@ -228,12 +268,35 @@ export class StatsStore extends DurableObject> { record.auxiliary, record.auxiliary, ); + const surface = record.visitor ? uniqueSurface(record.event, record.target) : null; + if (surface && record.visitor) { + const day = dayStart(record.at); + this.sql.exec( + "INSERT OR IGNORE INTO visitor_days (surface, visitor, day) VALUES (?, ?, ?)", + surface, + record.visitor, + day, + ); + this.sql.exec( + `INSERT INTO visitors (surface, visitor, first_day, last_day) VALUES (?, ?, ?, ?) + ON CONFLICT (surface, visitor) DO UPDATE SET + first_day = MIN(visitors.first_day, excluded.first_day), + last_day = MAX(visitors.last_day, excluded.last_day)`, + surface, + record.visitor, + day, + day, + ); + } return new Response(null, { status: 204 }); } - private snapshot(range: StatsRange): Response { + private snapshot(range: StatsRange, uniquesConfigured: boolean): Response { const now = Date.now(); this.sql.exec("DELETE FROM live_presence WHERE expires_at <= ?", now); + const forgetBefore = dayStart(now) - VISITOR_MEMORY_DAYS * DAY_MS; + this.sql.exec("DELETE FROM visitor_days WHERE day < ?", forgetBefore); + this.sql.exec("DELETE FROM visitors WHERE last_day < ?", forgetBefore); const collectingSince = this.sql.exec( "SELECT MIN(bucket) AS minimum FROM metric_hourly", ).one().minimum; @@ -269,9 +332,52 @@ export class StatsStore extends DurableObject> { COALESCE(SUM(active_viewers), 0) AS active_viewers FROM live_presence`, ).one(); + /* + * People are counted by day, so a range that starts mid-day includes the + * whole of that day: a day is the finest grain the visitor tables keep. + */ + const uniqueStart = dayStart(rangeStart); + const uniques = this.sql.exec( + `SELECT seen.surface AS surface, + COUNT(*) AS unique_count, + SUM(CASE WHEN visitors.first_day >= ? THEN 1 ELSE 0 END) AS new_count + FROM (SELECT DISTINCT surface, visitor FROM visitor_days WHERE day >= ?) AS seen + JOIN visitors ON visitors.surface = seen.surface AND visitors.visitor = seen.visitor + GROUP BY seen.surface`, + uniqueStart, + uniqueStart, + ).toArray(); + const uniqueDays = this.sql.exec( + `SELECT day, surface, COUNT(*) AS unique_count + FROM visitor_days + WHERE day >= ? + GROUP BY day, surface + ORDER BY day`, + uniqueStart, + ).toArray(); + const retention = this.sql.exec( + `SELECT visitors.surface AS surface, visitors.visitor AS visitor, + visitors.first_day AS first_day, visitor_days.day AS day + FROM visitors + JOIN visitor_days ON visitor_days.surface = visitors.surface AND visitor_days.visitor = visitors.visitor + WHERE visitors.first_day >= ? AND visitors.surface IN ('site', 'cli')`, + weekStart(now) - (RETENTION_WEEKS - 1) * WEEK_MS, + ).toArray(); const snapshot = buildStatsSnapshot( - { summary, trend, devices, referrers, clients, live, collectingSince }, + { + summary, + trend, + devices, + referrers, + clients, + live, + collectingSince, + uniques, + uniqueDays, + retention, + uniquesConfigured, + }, range, now, rangeStart, @@ -328,7 +434,7 @@ async function parsePresenceRequest( }; } -function parseStatsRecord(candidate: unknown): (AnalyticsRecord & { at: number }) | null { +function parseStatsRecord(candidate: unknown): (AnalyticsRecord & { at: number; visitor?: string }) | null { if (typeof candidate !== "object" || candidate === null) return null; const value = candidate as Record; if ( @@ -345,7 +451,11 @@ function parseStatsRecord(candidate: unknown): (AnalyticsRecord & { at: number } ) return null; const at = Number(value.at); if (Math.abs(Date.now() - at) > 10 * 60 * 1_000) return null; + if (value.visitor !== undefined && (typeof value.visitor !== "string" || !/^[a-f0-9]{20}$/.test(value.visitor))) { + return null; + } return { + ...(typeof value.visitor === "string" ? { visitor: value.visitor } : {}), event: value.event as AnalyticsEvent, target: value.target, device: value.device as AnalyticsRecord["device"], diff --git a/wrangler.example.jsonc b/wrangler.example.jsonc index 4a29ea6..ec7d185 100644 --- a/wrangler.example.jsonc +++ b/wrangler.example.jsonc @@ -18,6 +18,10 @@ "/skill", "/skill/*", "/docs/*", + "/app/*", + "/cli/*", + "/refstream/*", + "/platforms/*", "/mobile/*", "/reliability/*", "/security/*",