diff --git a/.github/workflows/downloads.yml b/.github/workflows/downloads.yml index a919ec6..58af418 100644 --- a/.github/workflows/downloads.yml +++ b/.github/workflows/downloads.yml @@ -58,21 +58,25 @@ jobs: script: powershell runs-on: ${{ matrix.os }} timeout-minutes: 10 + # Forty-eight installs a day from fresh runner addresses would read as + # people on the statistics dashboard. The check agent on every request, + # the scripts' own included, tells the site this is a monitor. env: ORIGIN: ${{ inputs.origin || 'https://shell.online' }} + SHELL_ONLINE_INSTALL_CHECK: "1" steps: - name: Install with the documented command if: matrix.script == 'posix' run: | export SHELL_ONLINE_INSTALL_DIR="$RUNNER_TEMP/shell-online" - curl -fsSL "$ORIGIN/install" | sh + curl -fsSL -A shell.online-install-check "$ORIGIN/install" | sh "$SHELL_ONLINE_INSTALL_DIR/shell" --version - name: Install with the documented command if: matrix.script == 'powershell' shell: pwsh run: | $env:SHELL_ONLINE_INSTALL_DIR = Join-Path $env:RUNNER_TEMP "shell-online" - Invoke-RestMethod "$env:ORIGIN/install.ps1" | Invoke-Expression + Invoke-RestMethod -UserAgent shell.online-install-check "$env:ORIGIN/install.ps1" | Invoke-Expression & (Join-Path $env:SHELL_ONLINE_INSTALL_DIR "shell.exe") --version report: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad69e4..c3301dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,46 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve ## Unreleased +### Added + +- The statistics dashboard now carries the accounts, laid out to be read in + one pass: how many there are and how that moved against the period before, + how many signed up and how many opened the app, how many of those had + signed up earlier, a line of sign-ups and use for every day since the first + account, what accounts did in the app, how many days each of them has been + in it, and which sign-up weeks came back. It sits directly under the funnel, + which ends at a first keystroke, because an account is what the funnel is + for. +- Accounts of your own are left out of every account figure, named by + `STATS_EXCLUDE` on the accounts app: addresses, or domains and their + subdomains. The team's accounts are the most active there are and were + always going to use the product, so leaving them in makes a quiet week look + like a good one. The dashboard says how many it left out, so the figure can + be checked rather than taken on trust. What those accounts do in the app is + counted apart from what customers do and never reported. + +### Fixed + +- The scheduled download check installed on three fresh GitHub runners every + half hour and was counted as installs, installer runs, installer outcomes + and new machines, which is where a dashboard day of ninety installs and + fifty-six new machines came from. Both install scripts now take + `SHELL_ONLINE_INSTALL_CHECK=1`, which puts a check user agent on every + request and reports nothing; the workflow sets it, and the site counts + that agent, and monitors in general, as crawlers. +- HTTP libraries and PowerShell's web cmdlets were classified as desktop + browsers, so a Node script or a Windows install read as a person reading + the installer. They are tools now, and a Windows install counts as a run. +- The installs tile drew the sessions line. The trend now carries installs + and started sessions as their own series, crawlers left out of every line. +- "New" people are measured from the day a surface's people were first + counted, per surface, and while a whole range has not yet passed since + that day the split is replaced by the day it becomes meaningful. Machine + rows keyed the old way are dropped once, so they do not sit in the cohorts + as machines that never came back. +- Pages a browser fetched ahead of time (prefetch, prerender) are not views. +- The 24h range says that people are counted by UTC day, so over two days. + ## [0.16.0] — 2026-09-15 ### Changed diff --git a/app/.env.example b/app/.env.example index 4980308..3b1cfe6 100644 --- a/app/.env.example +++ b/app/.env.example @@ -30,6 +30,9 @@ TRUST_PROXY=0 # Optional, 32+ characters. The same value goes on the relay Worker as # APP_STATS_TOKEN so its statistics dashboard can read account counts. # STATS_TOKEN= +# Accounts left out of every statistics figure: your own, not customers'. +# Addresses, or domains and their subdomains, separated by commas or spaces. +# STATS_EXCLUDE= # Optional invitation email. Defaults to SendGrid. MAIL_PROVIDER=sendgrid diff --git a/app/README.md b/app/README.md index 61646e5..3e8a120 100644 --- a/app/README.md +++ b/app/README.md @@ -60,6 +60,7 @@ The server uses: | `FEEDBACK_TO` | Optional address that feedback sent from the app is forwarded to | | `TRUST_PROXY` | Set to `1` only behind a trusted proxy | | `STATS_TOKEN` | Optional, 32+ characters: lets the relay's statistics dashboard read account counts and sign-up cohorts | +| `STATS_EXCLUDE` | Optional: addresses and domains whose accounts are left out of every statistics figure, for your own team's accounts. Comma or space separated; an entry with a local part matches that address, one without matches the domain and its subdomains. The dashboard reports how many accounts it left out | See [`.env.example`](.env.example) for the complete development configuration. diff --git a/app/server/app.test.ts b/app/server/app.test.ts index beaced1..0945f51 100644 --- a/app/server/app.test.ts +++ b/app/server/app.test.ts @@ -2728,6 +2728,26 @@ describe("feedback", () => { expect(kept.orgId).toBeTruthy(); }); + /* + * The dashboard reports what customers did. A thing done by one of our own + * accounts is counted apart, so a week of our own testing cannot read as a + * week of use. + */ + it("counts a thing one of our own accounts did apart from the rest", async () => { + handle = createApp({ + store, + verifyIdToken: verifyIdToken as never, + allowedOrigins: [ORIGIN], + excludedAccounts: ["ours.example"], + }); + expect((await call("POST", "/api/feedback", { auth: await idToken(), body: message })).status).toBe(201); + expect((await call("POST", "/api/feedback", { + auth: await idToken({ sub: "uid-ours", email: "dev@ours.example", name: "Dev" }), + body: message, + })).status).toBe(201); + expect(await store.appEvents(0)).toEqual([{ event: "feedback_sent", count: 1 }]); + }); + it("refuses without a signed-in user", async () => { const posted = await call("POST", "/api/feedback", { body: message }); expect(posted.status).toBe(401); @@ -2788,4 +2808,29 @@ describe("account figures for the statistics dashboard", () => { expect(JSON.stringify(answer.body)).not.toContain("uid-1"); expect(JSON.stringify(answer.body)).not.toContain("ana@example.com"); }); + + /* + * Our own accounts are the most active there are and were always going to + * use the product. Left in, a quiet week reads as a good one, so they are + * out of the counts and only their number is reported. + */ + it("leaves our own accounts out of the figures, and says how many it left out", async () => { + handle = createApp({ + store, + verifyIdToken: verifyIdToken as never, + allowedOrigins: [ORIGIN], + statsToken: TOKEN, + excludedAccounts: ["ours.example"], + }); + expect((await call("GET", "/api/org", { auth: await idToken() })).status).toBe(200); + expect((await call("GET", "/api/org", { + auth: await idToken({ sub: "uid-ours", email: "dev@ours.example", name: "Dev" }), + })).status).toBe(200); + + const answer = await call("GET", "/api/stats/accounts?range=7d", { auth: TOKEN }); + expect(answer.status).toBe(200); + expect(answer.body).toMatchObject({ total: 1, newInRange: 1, activeInRange: 1, excluded: 1 }); + expect(answer.body.cohorts[0].size).toBe(1); + expect(JSON.stringify(answer.body)).not.toContain("ours.example"); + }); }); diff --git a/app/server/app.ts b/app/server/app.ts index 61f15d0..3329b0e 100644 --- a/app/server/app.ts +++ b/app/server/app.ts @@ -49,6 +49,7 @@ import { recordAudit, assignSession, auditCsv, SEALED_KINDS } from "./routes/aud import { addComment, inbox, notifyAssigned, notifySessionStarted } from "./routes/social"; import { deleteAccount } from "./routes/account"; import { submitFeedback } from "./routes/feedback"; +import { excludedAccountFilter } from "./lib/internal-accounts"; import { accountStats, dayStart, isStatsRange, rangeStart } from "./routes/stats"; import { timingSafeEqual } from "node:crypto"; import { callerAddress, rateLimiter } from "./lib/rate-limit"; @@ -88,6 +89,11 @@ export interface AppOptions { * exist. At least 32 characters; see readConfig. */ statsToken?: string; + /** + * Accounts the statistics dashboard leaves out of every figure: ours, not + * customers'. Addresses and domains; see internal-accounts.ts. + */ + excludedAccounts?: string[]; /** * Serves the built client for anything that is not an API route. Present * only in a deployment that serves the app and the API together; in @@ -336,6 +342,8 @@ export function createApp(options: AppOptions) { const mailer = options.mailer ?? logMailer(); /* The first allowed origin is the web app's; see readConfig. */ const webOrigin = options.webOrigin ?? allowedOrigins[0] ?? ""; + /* Whether an address is one of ours, for the statistics only. */ + const isInternalAccount = excludedAccountFilter(options.excludedAccounts ?? []); const credentialLimit = rateLimiter(CREDENTIAL_BUCKET); const generalLimit = rateLimiter(GENERAL_BUCKET); const feedbackLimit = rateLimiter(FEEDBACK_BUCKET); @@ -393,9 +401,12 @@ export function createApp(options: AppOptions) { * One count per thing done, never who did it, for the statistics * dashboard. Counting must never change an answer, so a store that cannot * count is nobody's problem here. + * + * The address is passed so the store can record whether this was one of + * ours; it is read for that and nothing else, and never stored. */ - const track = (event: AppEvent): void => { - void store.recordAppEvent(event).catch(() => undefined); + const track = (event: AppEvent, email: string): void => { + void store.recordAppEvent(event, undefined, isInternalAccount(email)).catch(() => undefined); }; /* The CLI authenticates with an opaque access token issued by this service. */ @@ -503,7 +514,7 @@ export function createApp(options: AppOptions) { label: String(body.label ?? "shell cli").slice(0, 80), machineId, }); - track("machine_linked"); + track("machine_linked", result.email); return send(response, 200, { access_token: tokens.accessToken, refresh_token: tokens.refreshToken, @@ -580,7 +591,7 @@ export function createApp(options: AppOptions) { return send(response, 400, { error: "invalid browser public key" }); } if (publicKey) await store.setMemberKey(identity.uid, publicKey); - if (resolved.joined && invite) track("invite_accepted"); + if (resolved.joined && invite) track("invite_accepted", identity.email); const described = await describeOrganization(store, resolved.membership); return send(response, described.status, { ...(described.body as Record), @@ -621,7 +632,7 @@ export function createApp(options: AppOptions) { const invite = (result.body as { invite?: Invite }).invite; if (invite) await notifyInvited(store, mailer, webOrigin, membership, invite, log); } - if (result.status < 300) track("invite_created"); + if (result.status < 300) track("invite_created", membership.email); return send(response, result.status, result.body); } @@ -993,7 +1004,7 @@ export function createApp(options: AppOptions) { }); if (!created) return send(response, 409, { error: "this account already has a vault" }); const stored = await store.accountKey(identity.uid); - track("vault_created"); + track("vault_created", identity.email); return send(response, 201, { vault: stored ? vaultForApi(stored) : null }); } @@ -1146,7 +1157,7 @@ export function createApp(options: AppOptions) { result.session.name || result.session.command, ); } - track("session_registered"); + track("session_registered", token.email); return send(response, 201, { session: sessionForApi(result.session) }); } @@ -1377,7 +1388,7 @@ export function createApp(options: AppOptions) { createdAt: Date.now(), }; await store.putCommand(queued); - track("command_sent"); + track("command_sent", identity.email); return send(response, 202, { command: queued }); } @@ -1564,7 +1575,7 @@ export function createApp(options: AppOptions) { log, ); if (!result.ok) return send(response, result.status, { error: result.error }); - track("feedback_sent"); + track("feedback_sent", membership.email); return send(response, 201, { feedback: { id: result.value.id, at: result.value.at } }); } @@ -1581,7 +1592,7 @@ export function createApp(options: AppOptions) { const range = isStatsRange(requested) ? requested : "7d"; const now = Date.now(); return send(response, 200, accountStats( - await store.accountActivity(), + await store.accountActivity(isInternalAccount), range, now, await store.appEvents(dayStart(rangeStart(range, now))), diff --git a/app/server/index.ts b/app/server/index.ts index 1bdca8a..4311546 100644 --- a/app/server/index.ts +++ b/app/server/index.ts @@ -51,6 +51,7 @@ const server = createAccountsServer({ mailer: createMailer(config.mail), feedbackTo: config.feedbackTo, statsToken: config.statsToken, + excludedAccounts: config.excludedAccounts, serveClient: config.clientDir ? staticFiles(config.clientDir, config.identity.issuer) : undefined, diff --git a/app/server/lib/config.test.ts b/app/server/lib/config.test.ts index 932e099..1c1f346 100644 --- a/app/server/lib/config.test.ts +++ b/app/server/lib/config.test.ts @@ -171,6 +171,19 @@ describe("withoutCredentials", () => { }); }); +describe("readConfig statistics settings", () => { + it("reads the accounts the statistics leave out, and excludes nobody by default", () => { + expect(readConfig(MINIMAL).excludedAccounts).toEqual([]); + expect(readConfig({ ...MINIMAL, STATS_EXCLUDE: "Ours.example, someone@mail.example" }).excludedAccounts) + .toEqual(["ours.example", "someone@mail.example"]); + }); + + it("refuses a statistics token short enough to guess", () => { + expect(() => readConfig({ ...MINIMAL, STATS_TOKEN: "short" })).toThrow(/STATS_TOKEN/); + expect(readConfig({ ...MINIMAL, STATS_TOKEN: "t".repeat(32) }).statsToken).toBe("t".repeat(32)); + }); +}); + describe("allowedOriginsFor", () => { /* * A page on somebody's own machine must not be able to call production with diff --git a/app/server/lib/config.ts b/app/server/lib/config.ts index eb77798..50b9216 100644 --- a/app/server/lib/config.ts +++ b/app/server/lib/config.ts @@ -1,3 +1,4 @@ +import { parseExcludedAccounts } from "./internal-accounts"; import type { IssuerSettings } from "./oidc-token"; /** @@ -65,6 +66,12 @@ export interface Config { * figures. Absent means that route does not exist. */ statsToken?: string; + /** + * Accounts the statistics dashboard leaves out of every figure: ours, not + * customers'. Addresses and domains, read from STATS_EXCLUDE; see + * internal-accounts.ts for why it is configuration rather than source. + */ + excludedAccounts: string[]; } export class ConfigError extends Error {} @@ -260,5 +267,6 @@ export function readConfig(env: NodeJS.ProcessEnv = process.env): Config { }, feedbackTo: env.FEEDBACK_TO?.trim() || undefined, statsToken, + excludedAccounts: parseExcludedAccounts(env.STATS_EXCLUDE), }; } diff --git a/app/server/lib/internal-accounts.test.ts b/app/server/lib/internal-accounts.test.ts new file mode 100644 index 0000000..6608a0c --- /dev/null +++ b/app/server/lib/internal-accounts.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { excludedAccountFilter, isExcludedAccount, parseExcludedAccounts } from "./internal-accounts"; + +describe("parseExcludedAccounts", () => { + it("reads a list however it was written, and nothing from an unset variable", () => { + expect(parseExcludedAccounts("a@example.com, b.com; @c.org\nD@Example.com")).toEqual([ + "a@example.com", + "b.com", + "@c.org", + "d@example.com", + ]); + expect(parseExcludedAccounts(undefined)).toEqual([]); + expect(parseExcludedAccounts(" ")).toEqual([]); + }); +}); + +describe("isExcludedAccount", () => { + const rules = parseExcludedAccounts("ours.example, @tools.example, someone@mail.example"); + + it("excludes a domain and its subdomains, written either way", () => { + expect(isExcludedAccount(rules, "a@ours.example")).toBe(true); + expect(isExcludedAccount(rules, "a@eng.ours.example")).toBe(true); + expect(isExcludedAccount(rules, "a@tools.example")).toBe(true); + expect(isExcludedAccount(rules, "a@notours.example")).toBe(false); + /* A domain rule must not match a company whose name merely ends the same way. */ + expect(isExcludedAccount(rules, "a@theirours.example")).toBe(false); + }); + + it("excludes one named address without touching the rest of its provider", () => { + expect(isExcludedAccount(rules, "someone@mail.example")).toBe(true); + expect(isExcludedAccount(rules, "SomeOne@Mail.Example")).toBe(true); + expect(isExcludedAccount(rules, "someone.else@mail.example")).toBe(false); + }); + + it("sees through a +tag on either side", () => { + expect(isExcludedAccount(rules, "someone+shell@mail.example")).toBe(true); + expect(isExcludedAccount(parseExcludedAccounts("someone+old@mail.example"), "someone@mail.example")).toBe(true); + }); + + it("excludes nobody on an empty list or a missing address", () => { + expect(isExcludedAccount([], "a@ours.example")).toBe(false); + expect(isExcludedAccount(rules, "")).toBe(false); + expect(isExcludedAccount(rules, undefined)).toBe(false); + expect(isExcludedAccount(rules, "not-an-address")).toBe(false); + }); +}); + +describe("excludedAccountFilter", () => { + it("is a predicate, and excludes nobody when nothing is configured", () => { + expect(excludedAccountFilter(parseExcludedAccounts("ours.example"))("a@ours.example")).toBe(true); + expect(excludedAccountFilter([])("a@ours.example")).toBe(false); + }); +}); diff --git a/app/server/lib/internal-accounts.ts b/app/server/lib/internal-accounts.ts new file mode 100644 index 0000000..ea649c8 --- /dev/null +++ b/app/server/lib/internal-accounts.ts @@ -0,0 +1,62 @@ +/* + * Which accounts are ours rather than a customer's. + * + * The statistics dashboard is read to answer one question -- how is the + * product doing -- and our own accounts answer it wrongly. Founders, + * colleagues, the addresses the end-to-end tests sign up with and the people + * who build the thing are the most active accounts there are, and they were + * always going to use it. Left in, they make a quiet week look like a good + * one. So they are taken out of every account figure, and the dashboard says + * how many it took out rather than quietly showing a smaller number. + * + * The list is configuration, not source. It names individual people, and this + * repository is public: STATS_EXCLUDE carries it, and an unset variable + * excludes nobody. + */ + +/** + * Reads the list. Entries are separated by commas, semicolons or whitespace, + * so a value can be pasted in whichever shape it was written in. + * + * Each entry is either a whole address (`someone@example.com`) or a domain + * (`example.com` or `@example.com`), which also covers its subdomains. + */ +export function parseExcludedAccounts(value: string | undefined): string[] { + return (value ?? "") + .split(/[\s,;]+/) + .map((entry) => entry.trim().toLowerCase()) + .filter((entry) => entry.length > 0); +} + +/** + * Whether this address is one of ours. + * + * A `+tag` is dropped from both sides before comparing: every mail provider + * we use delivers `name+anything@` to `name@`, so a rule that did not would + * be walked around by the first person who signed up with a tagged address. + */ +export function isExcludedAccount(rules: readonly string[], email: string | undefined): boolean { + const address = normalizeAddress(email); + if (address === "") return false; + const domain = address.slice(address.indexOf("@") + 1); + return rules.some((rule) => { + if (rule.includes("@") && !rule.startsWith("@")) return normalizeAddress(rule) === address; + const suffix = rule.startsWith("@") ? rule.slice(1) : rule; + return suffix !== "" && (domain === suffix || domain.endsWith(`.${suffix}`)); + }); +} + +/** The predicate the stores tag rows with, so a caller never handles an address. */ +export function excludedAccountFilter(rules: readonly string[]): (email: string) => boolean { + if (rules.length === 0) return () => false; + return (email) => isExcludedAccount(rules, email); +} + +function normalizeAddress(email: string | undefined): string { + const address = (email ?? "").trim().toLowerCase(); + const at = address.indexOf("@"); + if (at <= 0 || at === address.length - 1) return ""; + const local = address.slice(0, at); + const tag = local.indexOf("+"); + return `${tag === -1 ? local : local.slice(0, tag)}@${address.slice(at + 1)}`; +} diff --git a/app/server/lib/migrations/014_app_events_internal.sql b/app/server/lib/migrations/014_app_events_internal.sql new file mode 100644 index 0000000..22d14fd --- /dev/null +++ b/app/server/lib/migrations/014_app_events_internal.sql @@ -0,0 +1,30 @@ +-- Whether the account that did the thing was one of ours. +-- +-- The statistics dashboard reports what customers did, and until now it could +-- not: app_events counted a machine linked by an end-to-end test exactly the +-- same as one linked by a customer, and the team's own accounts do far more in +-- the app than anybody else. Splitting the count by who did it is the only way +-- the totals can be read as product signal. +-- +-- Rows written before this column existed cannot be split any more, so they +-- are attributed to us: the column arrives defaulting to TRUE, which backfills +-- what is already here, and then defaults to FALSE for everything written +-- afterwards. That way the change can only ever understate what customers did, +-- never overstate it, and no row is thrown away to get there. +ALTER TABLE app_events ADD COLUMN IF NOT EXISTS internal BOOLEAN NOT NULL DEFAULT TRUE; +ALTER TABLE app_events ALTER COLUMN internal SET DEFAULT FALSE; + +-- (event, day) is no longer unique: the same kind on the same day now lands in +-- one row for us and one for everyone else. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_index index_ + JOIN pg_class table_ ON table_.oid = index_.indrelid + WHERE table_.relname = 'app_events' AND index_.indisprimary AND index_.indnatts = 3 + ) THEN + ALTER TABLE app_events DROP CONSTRAINT IF EXISTS app_events_pkey; + ALTER TABLE app_events ADD PRIMARY KEY (event, day, internal); + END IF; +END $$; diff --git a/app/server/lib/store-conformance.test.ts b/app/server/lib/store-conformance.test.ts index 60e867e..aafc144 100644 --- a/app/server/lib/store-conformance.test.ts +++ b/app/server/lib/store-conformance.test.ts @@ -952,7 +952,23 @@ for (const implementation of implementations) { expect((await store.membershipOf("uid-1"))?.lastSeenAt).toBe(noon + 2 * 60 * 60_000); await store.touchMembership("uid-1", noon + day); await store.touchMembership("nobody", noon); - expect(await store.accountActivity()).toEqual([{ joinedAt: 1000, days: [10 * day, 11 * day] }]); + expect(await store.accountActivity()).toEqual([ + { joinedAt: 1000, days: [10 * day, 11 * day], internal: false }, + ]); + }); + + it("tags our own accounts from the address, and hands back no address", async () => { + await store.putOrganization(organization()); + await store.putMembership(membership()); + await store.touchMembership("uid-1", noon); + const asked: string[] = []; + const rows = await store.accountActivity((email) => { + asked.push(email); + return email.endsWith("@example.com"); + }); + expect(asked).toEqual(["ana@example.com"]); + expect(rows).toEqual([{ joinedAt: 1000, days: [10 * day], internal: true }]); + expect(JSON.stringify(rows)).not.toContain("example.com"); }); it("keeps the days through a membership rewrite and drops them with the account", async () => { @@ -990,6 +1006,14 @@ for (const implementation of implementations) { expect(await store.appEvents(12 * day)).toEqual([]); }); + it("counts what we did ourselves apart, and never reports it", async () => { + await store.recordAppEvent("machine_linked", noon, false); + await store.recordAppEvent("machine_linked", noon, true); + await store.recordAppEvent("machine_linked", noon, true); + await store.recordAppEvent("vault_created", noon, true); + expect(await store.appEvents(0)).toEqual([{ event: "machine_linked", count: 1 }]); + }); + it("forgets counts older than the memory window when purging", async () => { await store.recordAppEvent("vault_created", noon); await store.purgeExpired(noon + 401 * day); diff --git a/app/server/lib/store-memory.ts b/app/server/lib/store-memory.ts index 060ef63..1439216 100644 --- a/app/server/lib/store-memory.ts +++ b/app/server/lib/store-memory.ts @@ -75,7 +75,7 @@ interface Shape { accountKeys: AccountKey[]; deletedAccounts: { uid: string; deletedAt: number }[]; accountActivity: { uid: string; day: number }[]; - appEvents: { event: AppEvent; day: number; count: number }[]; + appEvents: { event: AppEvent; day: number; count: number; internal?: boolean }[]; teamKeys: TeamKey[]; teamKeyShares: TeamKeyShare[]; } @@ -914,29 +914,33 @@ export class MemoryStore implements Store { this.flush(); } - async accountActivity(): Promise { + async accountActivity(isInternal: (email: string) => boolean = () => false): Promise { return this.data.memberships.map((membership) => ({ joinedAt: membership.joinedAt, days: this.data.accountActivity .filter((entry) => entry.uid === membership.uid) .map((entry) => entry.day) .sort((left, right) => left - right), + internal: isInternal(membership.email), })); } /* ---- App events ---- */ - async recordAppEvent(event: AppEvent, now = Date.now()): Promise { + async recordAppEvent(event: AppEvent, now = Date.now(), internal = false): Promise { const day = Math.floor(now / DAY_MS) * DAY_MS; - const row = this.data.appEvents.find((entry) => entry.event === event && entry.day === day); + const row = this.data.appEvents.find( + (entry) => entry.event === event && entry.day === day && (entry.internal ?? false) === internal, + ); if (row) row.count += 1; - else this.data.appEvents.push({ event, day, count: 1 }); + else this.data.appEvents.push({ event, day, count: 1, internal }); this.flush(); } async appEvents(sinceDay: number): Promise { const totals = new Map(); for (const entry of this.data.appEvents) { + if (entry.internal ?? false) continue; if (entry.day >= sinceDay) totals.set(entry.event, (totals.get(entry.event) ?? 0) + entry.count); } return [...totals.entries()] diff --git a/app/server/lib/store-postgres.ts b/app/server/lib/store-postgres.ts index fc7a8f7..6b858b7 100644 --- a/app/server/lib/store-postgres.ts +++ b/app/server/lib/store-postgres.ts @@ -1628,8 +1628,8 @@ export class PostgresStore implements Store { ); } - async accountActivity(): Promise { - const members = await this.rows("SELECT uid, joined_at FROM memberships"); + async accountActivity(isInternal: (email: string) => boolean = () => false): Promise { + const members = await this.rows("SELECT uid, email, joined_at FROM memberships"); const days = await this.rows("SELECT uid, day FROM account_activity ORDER BY day"); const byUid = new Map(); for (const row of days) { @@ -1640,22 +1640,24 @@ export class PostgresStore implements Store { return members.map((row) => ({ joinedAt: row.joined_at as number, days: byUid.get(row.uid as string) ?? [], + internal: isInternal((row.email as string | null) ?? ""), })); } /* ---- App events ---- */ - async recordAppEvent(event: AppEvent, now = Date.now()): Promise { + async recordAppEvent(event: AppEvent, now = Date.now(), internal = false): Promise { await this.pool.query( - `INSERT INTO app_events (event, day, count) VALUES ($1, $2, 1) - ON CONFLICT (event, day) DO UPDATE SET count = app_events.count + 1`, - [event, Math.floor(now / DAY_MS) * DAY_MS], + `INSERT INTO app_events (event, day, internal, count) VALUES ($1, $2, $3, 1) + ON CONFLICT (event, day, internal) DO UPDATE SET count = app_events.count + 1`, + [event, Math.floor(now / DAY_MS) * DAY_MS, internal], ); } async appEvents(sinceDay: number): Promise { const rows = await this.rows( - "SELECT event, SUM(count) AS count FROM app_events WHERE day >= $1 GROUP BY event ORDER BY event", + `SELECT event, SUM(count) AS count FROM app_events + WHERE day >= $1 AND internal = FALSE GROUP BY event ORDER BY event`, [sinceDay], ); return rows.map((row) => ({ event: row.event as AppEvent, count: Number(row.count) })); diff --git a/app/server/lib/store.ts b/app/server/lib/store.ts index 2ff83c8..8062e75 100644 --- a/app/server/lib/store.ts +++ b/app/server/lib/store.ts @@ -265,13 +265,27 @@ export interface Store { * day is written only when it does. */ touchMembership(uid: string, now?: number, resolutionMs?: number): Promise; - /** Every account's sign-up time and active days, with no identifiers. */ - accountActivity(): Promise; + /** + * Every account's sign-up time and active days, with no identifiers. + * + * `isInternal` is applied to each address while the store still holds it, + * and only its answer leaves: the caller can drop our own accounts from the + * figures without ever being handed an address to drop them by. + */ + accountActivity(isInternal?: (email: string) => boolean): Promise; /* ---- App events ---- */ - /** Counts one thing an account did, on the day it did it. Nothing about who. */ - recordAppEvent(event: AppEvent, now?: number): Promise; - /** Totals per kind over days on or after `sinceDay` (midnight UTC), for the dashboard. */ + /** + * Counts one thing an account did, on the day it did it. Nothing about who, + * beyond whether the account was one of ours: counts of what the team did + * while testing are kept apart from counts of what customers did, because + * the dashboard reports the second and not the first. + */ + recordAppEvent(event: AppEvent, now?: number, internal?: boolean): Promise; + /** + * Customers' totals per kind over days on or after `sinceDay` (midnight + * UTC), for the dashboard. Our own are never included. + */ appEvents(sinceDay: number): Promise; /* ---- Housekeeping ---- */ diff --git a/app/server/lib/types.ts b/app/server/lib/types.ts index 248d990..b4bc6d8 100644 --- a/app/server/lib/types.ts +++ b/app/server/lib/types.ts @@ -204,6 +204,12 @@ export interface Feedback { export interface AccountActivity { joinedAt: number; days: number[]; + /** + * One of ours rather than a customer's. Decided from the address while the + * store still has it, so the figures can leave us out without anything + * outside the store being handed an identifier. + */ + internal: boolean; } /** diff --git a/app/server/routes/stats.test.ts b/app/server/routes/stats.test.ts index 050fdfe..1ca57c1 100644 --- a/app/server/routes/stats.test.ts +++ b/app/server/routes/stats.test.ts @@ -1,9 +1,14 @@ import { describe, expect, it } from "vitest"; +import type { AccountActivity } from "../lib/store"; import { accountStats, buildRetentionCohorts, DAY_MS, WEEK_MS, weekStart } from "./stats"; /* A Monday, so the weeks are easy to read. */ const monday = Date.UTC(2026, 8, 7); const now = monday + 2 * WEEK_MS + 3 * DAY_MS + 9 * 60 * 60_000; +const today = Math.floor(now / DAY_MS) * DAY_MS; + +const customer = (account: Omit): AccountActivity => ({ ...account, internal: false }); +const ours = (account: Omit): AccountActivity => ({ ...account, internal: true }); describe("accountStats events", () => { it("passes what accounts did through as counts by kind, and an empty object when nothing was counted", () => { @@ -16,23 +21,36 @@ describe("accountStats events", () => { describe("accountStats", () => { const accounts = [ /* signed up week 0, used it in weeks 1 and 2 */ - { joinedAt: monday + 60_000, days: [monday, monday + WEEK_MS + DAY_MS, monday + 2 * WEEK_MS] }, + customer({ joinedAt: monday + 60_000, days: [monday, monday + WEEK_MS + DAY_MS, monday + 2 * WEEK_MS] }), /* signed up week 0, never came back */ - { joinedAt: monday + 3 * DAY_MS, days: [monday + 3 * DAY_MS] }, + customer({ joinedAt: monday + 3 * DAY_MS, days: [monday + 3 * DAY_MS] }), /* signed up yesterday */ - { joinedAt: now - DAY_MS, days: [Math.floor((now - DAY_MS) / DAY_MS) * DAY_MS] }, + customer({ joinedAt: now - DAY_MS, days: [today - DAY_MS] }), /* signed up long ago, active today */ - { joinedAt: monday - 30 * WEEK_MS, days: [Math.floor(now / DAY_MS) * DAY_MS] }, + customer({ joinedAt: monday - 30 * WEEK_MS, days: [today] }), ]; - it("counts accounts, new ones, and active ones for the range", () => { + it("counts accounts, new ones, active ones, and the ones that came back", () => { const week = accountStats(accounts, "7d", now); expect(week.total).toBe(4); expect(week.newInRange).toBe(1); expect(week.activeInRange).toBe(3); + /* Active and not new: the account from week 0 and the one from long ago. */ + expect(week.returningInRange).toBe(2); const all = accountStats(accounts, "all", now); expect(all.newInRange).toBe(4); expect(all.activeInRange).toBe(4); + expect(all.returningInRange).toBe(0); + }); + + it("counts the period before the range the same way, and nothing before all time", () => { + const week = accountStats(accounts, "7d", now); + /* + * The three that had signed up by the start of this week's range; two of + * them used it in the week before, both having signed up before that. + */ + expect(week.previous).toEqual({ total: 3, newAccounts: 0, active: 2, returning: 2 }); + expect(accountStats(accounts, "all", now).previous).toBeNull(); }); it("sends new accounts per day, with zeros for quiet days", () => { @@ -42,6 +60,26 @@ describe("accountStats", () => { expect(stats.newByDay.at(-2)?.count).toBe(1); }); + it("sends active accounts per day over the same days, counting the sign-up day as use", () => { + const stats = accountStats(accounts, "30d", now); + expect(stats.activeByDay.map((point) => point.day)).toEqual(stats.newByDay.map((point) => point.day)); + expect(stats.activeByDay.at(-1)).toEqual({ day: today, count: 1 }); + expect(stats.activeByDay.at(-2)).toEqual({ day: today - DAY_MS, count: 1 }); + }); + + it("buckets accounts by how many days they used it, over the ones it could know about", () => { + const stats = accountStats(accounts, "7d", now); + /* The first day any account was recorded as active; the fourth signed up long before it. */ + expect(stats.activeSince).toBe(monday); + expect(stats.engagementBase).toBe(3); + expect(stats.engagement).toEqual([ + { label: "one_day", value: 2 }, + { label: "two_days", value: 0 }, + { label: "three_to_six_days", value: 1 }, + { label: "seven_or_more_days", value: 0 }, + ]); + }); + it("builds sign-up cohorts without any identifier", () => { const stats = accountStats(accounts, "7d", now); expect(stats.cohorts).toEqual([ @@ -50,6 +88,47 @@ describe("accountStats", () => { ]); expect(JSON.stringify(stats)).not.toMatch(/uid|email/); }); + + it("has nothing to say about an empty app rather than failing", () => { + const stats = accountStats([], "7d", now); + expect(stats.total).toBe(0); + expect(stats.activeSince).toBeNull(); + expect(stats.engagementBase).toBe(0); + expect(stats.engagement.every((bucket) => bucket.value === 0)).toBe(true); + expect(stats.cohorts).toEqual([]); + }); +}); + +describe("accountStats leaves our own accounts out", () => { + const accounts = [ + customer({ joinedAt: now - 2 * DAY_MS, days: [today - 2 * DAY_MS, today] }), + ours({ joinedAt: now - 2 * DAY_MS, days: [today - 2 * DAY_MS, today - DAY_MS, today] }), + ours({ joinedAt: monday, days: [monday, today] }), + ]; + + it("counts them out of every figure and reports how many it left out", () => { + const stats = accountStats(accounts, "7d", now); + expect(stats.excluded).toBe(2); + expect(stats.total).toBe(1); + expect(stats.newInRange).toBe(1); + expect(stats.activeInRange).toBe(1); + expect(stats.engagementBase).toBe(1); + expect(stats.engagement).toEqual([ + { label: "one_day", value: 0 }, + { label: "two_days", value: 1 }, + { label: "three_to_six_days", value: 0 }, + { label: "seven_or_more_days", value: 0 }, + ]); + expect(stats.cohorts).toEqual([{ weekStart: weekStart(now), size: 1, active: [] }]); + expect(stats.activeByDay.at(-1)).toEqual({ day: today, count: 1 }); + }); + + it("leaves the figures alone when none of the accounts are ours", () => { + const outside = accounts.map((account) => ({ ...account, internal: false })); + const stats = accountStats(outside, "7d", now); + expect(stats.excluded).toBe(0); + expect(stats.total).toBe(3); + }); }); describe("buildRetentionCohorts", () => { diff --git a/app/server/routes/stats.ts b/app/server/routes/stats.ts index 9528143..5b2e9fb 100644 --- a/app/server/routes/stats.ts +++ b/app/server/routes/stats.ts @@ -30,11 +30,53 @@ export interface RetentionCohort { active: number[]; } +/** How many accounts have used the app on how many separate days. */ +export const ENGAGEMENT_BUCKETS = [ + { label: "one_day", from: 1, to: 1 }, + { label: "two_days", from: 2, to: 2 }, + { label: "three_to_six_days", from: 3, to: 6 }, + { label: "seven_or_more_days", from: 7, to: Number.POSITIVE_INFINITY }, +] as const; + +/** The headline counts over one period, so a range can be read against the one before it. */ +export interface AccountPeriod { + /** Accounts in existence at the end of the period. */ + total: number; + /** Of those, that signed up during it. */ + newAccounts: number; + /** Accounts that used the app during it. */ + active: number; + /** Of the active, that had signed up before it began: the ones that came back. */ + returning: number; +} + export interface AccountStats { + /** Accounts there are now. */ total: number; + /** Of those, that signed up in the range. */ newInRange: number; + /** Accounts that used the app in the range. */ activeInRange: number; + /** Of the active, that had signed up before the range began. */ + returningInRange: number; + /** The same four over the period of equal length before the range, or null over all time. */ + previous: AccountPeriod | null; newByDay: { day: number; count: number }[]; + /** Accounts that used the app on each day, over the same days as newByDay. */ + activeByDay: { day: number; count: number }[]; + /** + * Accounts by how many separate days they have ever used the app, over the + * accounts that signed up since activity was first recorded. An account + * that signed up before that has days missing through no fault of its own, + * and would read as a bounce. + */ + engagement: { label: string; value: number }[]; + /** How many accounts the engagement figures are over. */ + engagementBase: number; + /** Midnight UTC of the first day any account was recorded as active, or null. */ + activeSince: number | null; + /** Accounts left out of every figure above because they are ours. */ + excluded: number; cohorts: RetentionCohort[]; /** Things done in the range, by kind: machines linked, commands sent. Counts of things, not of accounts. */ events: Record; @@ -62,42 +104,139 @@ export function rangeStart(range: StatsRange, now: number): number { return 0; } +/** + * The account figures for one range. + * + * Our own accounts are dropped before anything is counted, and only the + * number of them dropped is reported: the dashboard is read as product + * signal, and the team's accounts are the most active there are. See + * internal-accounts.ts. + */ export function accountStats( activity: AccountActivity[], range: StatsRange, now = Date.now(), events: AppEventCount[] = [], ): AccountStats { + const excluded = activity.filter((account) => account.internal).length; + const accounts = activity.filter((account) => !account.internal); const start = rangeStart(range, now); - const startDay = dayStart(start); + const current = countPeriod(accounts, start, now); + /* + * The period of equal length before this one, so a headline figure can say + * how it moved. All time has nothing before it. + */ + const previous = range === "all" ? null : countPeriod(accounts, start - (now - start), start); + + const today = dayStart(now); + const firstDay = today - (NEW_BY_DAY_LIMIT - 1) * DAY_MS; const newByDay = new Map(); - const firstDay = dayStart(now) - (NEW_BY_DAY_LIMIT - 1) * DAY_MS; - for (let day = firstDay; day <= dayStart(now); day += DAY_MS) newByDay.set(day, 0); + const activeByDay = new Map(); + for (let day = firstDay; day <= today; day += DAY_MS) { + newByDay.set(day, 0); + activeByDay.set(day, 0); + } - let newInRange = 0; - let activeInRange = 0; + let activeSince: number | null = null; const rows: { visitor: string; first_day: number; day: number }[] = []; - activity.forEach((account, index) => { + accounts.forEach((account, index) => { const joinedDay = dayStart(account.joinedAt); - if (account.joinedAt >= start) newInRange += 1; - /* Signing up is using it, so a brand-new account is active on its first day. */ - if (account.joinedAt >= start || account.days.some((day) => day >= startDay)) activeInRange += 1; if (newByDay.has(joinedDay)) newByDay.set(joinedDay, (newByDay.get(joinedDay) ?? 0) + 1); + /* The recorded days only, so this is when the app started keeping them. */ + for (const day of account.days) { + const recorded = dayStart(day); + if (activeSince === null || recorded < activeSince) activeSince = recorded; + } + for (const day of activeDays(account)) { + if (activeByDay.has(day)) activeByDay.set(day, (activeByDay.get(day) ?? 0) + 1); + } + /* + * Cohorts are built from rows shaped like the relay's visitor rows, with + * the index standing in for a person: it never leaves this function, and + * two runs of the same data give it to different accounts. + */ const visitor = String(index); rows.push({ visitor, first_day: joinedDay, day: joinedDay }); for (const day of account.days) rows.push({ visitor, first_day: joinedDay, day }); }); + /* + * Days are only known from the day activity was first recorded, so an + * account that signed up before it looks like one that never came back. + * Engagement is over the accounts that signed up since, and says how many + * that is. + */ + const measurable = activeSince === null + ? [] + : accounts.filter((account) => dayStart(account.joinedAt) >= (activeSince as number)); + return { - total: activity.length, - newInRange, - activeInRange, + total: current.total, + newInRange: current.newAccounts, + activeInRange: current.active, + returningInRange: current.returning, + previous, newByDay: [...newByDay.entries()].map(([day, count]) => ({ day, count })), + activeByDay: [...activeByDay.entries()].map(([day, count]) => ({ day, count })), + engagement: ENGAGEMENT_BUCKETS.map((bucket) => ({ + label: bucket.label, + value: measurable.filter((account) => { + const days = activeDays(account).length; + return days >= bucket.from && days <= bucket.to; + }).length, + })), + engagementBase: measurable.length, + activeSince, + excluded, cohorts: buildRetentionCohorts(rows, now), events: Object.fromEntries(events.map((entry) => [entry.event, entry.count])), }; } +/** + * The days an account was in the app. + * + * Signing up is using it, so the sign-up day counts even where no activity + * row was written for it -- which is every account that signed up before + * activity was recorded at all, and any whose first request predated the + * first hourly touch. + */ +function activeDays(account: AccountActivity): number[] { + const days = new Set(account.days.map(dayStart)); + days.add(dayStart(account.joinedAt)); + return [...days].sort((left, right) => left - right); +} + +/** + * The four counts over one period: what existed, what arrived, what was used, + * and how much of the use came from accounts that were already here. + * + * Sign-ups are placed by their exact time, in [start, end). Activity only has + * a UTC day, so it is matched by day with both ends inclusive; that is what + * makes a period and the one before it the same number of days, at the cost + * of the two sharing the day they meet on. + */ +function countPeriod(accounts: AccountActivity[], start: number, end: number): AccountPeriod { + const startDay = dayStart(start); + const endDay = dayStart(end); + let total = 0; + let newAccounts = 0; + let active = 0; + let returning = 0; + for (const account of accounts) { + if (account.joinedAt >= end) continue; + total += 1; + const isNew = account.joinedAt >= start; + if (isNew) newAccounts += 1; + const used = isNew || + activeDays(account).some((day) => day >= startDay && day <= endDay); + if (!used) continue; + active += 1; + if (!isNew) returning += 1; + } + return { total, newAccounts, active, returning }; +} + /* * Weekly cohorts, the same way the relay's dashboard builds them from visitor * hashes: everyone who signed up in one week, and how many of them used the diff --git a/app/worker/index.ts b/app/worker/index.ts index 0a21453..91e383a 100644 --- a/app/worker/index.ts +++ b/app/worker/index.ts @@ -7,6 +7,7 @@ import { PostgresStore } from "../server/lib/store-postgres"; import { callNodeHandler, type NodeHandler } from "./node-adapter"; import { allowedOriginsFor, readIdentity, type Env as Settings } from "../server/lib/config"; import { browserSecurityHeaders } from "../server/lib/browser-headers"; +import { parseExcludedAccounts } from "../server/lib/internal-accounts"; import { relaySessionLiveness, type SessionLivenessSource } from "../server/lib/session-liveness"; /** @@ -50,6 +51,8 @@ export interface Env { FEEDBACK_TO?: string; /** Secret: what the relay's statistics dashboard presents for account figures. */ STATS_TOKEN?: string; + /** Accounts the statistics leave out: ours, not customers'. See internal-accounts.ts. */ + STATS_EXCLUDE?: string; } interface ExecutionContext { @@ -148,6 +151,7 @@ function routerFor(env: Env): NodeHandler { }), feedbackTo: env.FEEDBACK_TO?.trim() || undefined, statsToken: env.STATS_TOKEN && env.STATS_TOKEN.length >= 32 ? env.STATS_TOKEN : undefined, + excludedAccounts: parseExcludedAccounts(env.STATS_EXCLUDE), log: (message, error) => console.error(message, error), sessionLiveness: liveness, })); diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 853a223..ee30ad3 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -89,3 +89,18 @@ Accounts are optional and do not participate in terminal transport. The app in PostgreSQL. OIDC requires an issuer and public client id together; leaving both unset keeps the Firebase flow. Its local Docker deployment is documented in [`app/README.md`](../app/README.md). + +Your own accounts would otherwise dominate the account figures: they are the +most active accounts there are, and they were always going to use the product. +`STATS_EXCLUDE` on the accounts app names them, and what they do in the app is +counted apart from what customers do rather than shown. It is configuration +rather than a list in the source because it names individual people: + +```sh +npx wrangler secret put STATS_EXCLUDE # example.com, @tools.example, someone@mail.example +``` + +An entry with a local part excludes that one address, and one without excludes +the domain and its subdomains; a `+tag` is ignored on both sides. The dashboard +says how many accounts were left out, so the figure can be checked rather than +taken on trust. Unset, nobody is excluded. diff --git a/public/install b/public/install index 0f94ffb..6ff8378 100755 --- a/public/install +++ b/public/install @@ -6,6 +6,17 @@ set -eu # checksum_mismatch, ...) and the binary name, nothing else, so that a # platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +curl_agent= +wget_agent= +if [ "${SHELL_ONLINE_INSTALL_CHECK:-0}" = 1 ]; then + curl_agent="-A shell.online-install-check" + wget_agent="-U shell.online-install-check" +fi fail() { printf 'shell.online: %s\n' "$1" >&2 @@ -15,6 +26,7 @@ fail() { report() { [ "${SHELL_ONLINE_INSTALL_REPORT:-1}" != 0 ] || return 0 + [ -z "$curl_agent" ] || return 0 case "${base_url:-}" in http://*|https://*) ;; *) return 0 ;; @@ -101,12 +113,15 @@ binary_url="$base_url/downloads/$binary_name" download() { source_url=$1 destination=$2 + # The agent flags are two words or none, on purpose unquoted. if command -v curl >/dev/null 2>&1; then - if ! curl -fsSL "$source_url" -o "$destination"; then + # shellcheck disable=SC2086 + if ! curl -fsSL $curl_agent "$source_url" -o "$destination"; then fail "download failed: $source_url" download_failed fi elif command -v wget >/dev/null 2>&1; then - if ! wget -q "$source_url" -O "$destination"; then + # shellcheck disable=SC2086 + if ! wget -q $wget_agent "$source_url" -O "$destination"; then fail "download failed: $source_url" download_failed fi else diff --git a/public/install.ps1 b/public/install.ps1 index 82e607c..77bfb17 100644 --- a/public/install.ps1 +++ b/public/install.ps1 @@ -10,9 +10,17 @@ $ErrorActionPreference = "Stop" # went: one request carrying a single word and the binary name, nothing else, # so that a platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +$web = @{ UseBasicParsing = $true } +if ($env:SHELL_ONLINE_INSTALL_CHECK -eq "1") { $web.UserAgent = "shell.online-install-check" } + $script:reported = $false function Report([string]$Outcome) { if ($env:SHELL_ONLINE_INSTALL_REPORT -eq "0") { return } + if ($web.ContainsKey("UserAgent")) { return } if ($BaseUrl -notmatch '^https?://') { return } $script:reported = $true $platform = if ($script:artifact) { $script:artifact } else { "unknown" } @@ -48,8 +56,8 @@ $manifestPath = Join-Path $temporaryDirectory "SHA256SUMS" try { New-Item -ItemType Directory -Path $temporaryDirectory | Out-Null - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath $manifest = Get-Content -Raw $manifestPath $match = [regex]::Match($manifest, "(?m)^([a-f0-9]{64}) " + [regex]::Escape($artifact) + "$") if (-not $match.Success) { Fail "release manifest has no valid checksum for $artifact" "manifest_missing" } diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 1d7d9d8..84696fa 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -163,4 +163,19 @@ if grep -q "install/report" "$report_log"; then exit 1 fi +# A check run says so on every request and reports nothing, even when +# reporting is otherwise on. +: > "$report_log" +if output=$(SHELL_ONLINE_INSTALL_REPORT=1 SHELL_ONLINE_INSTALL_CHECK=1 SHELL_ONLINE_TEST_REPORT_LOG=$report_log \ + SHELL_ONLINE_BASE_URL=https://installer.invalid SHELL_ONLINE_INSTALL_DIR=$test_root/report-install \ + PATH=$report_bin:$PATH sh "$installer" 2>&1); then + printf 'Installer with a failing download unexpectedly succeeded.\n' >&2 + exit 1 +fi +assert_contains "$(cat "$report_log")" "-A shell.online-install-check https://installer.invalid/downloads/shell-" +if grep -q "install/report" "$report_log"; then + printf 'A check run reported its outcome.\n' >&2 + exit 1 +fi + printf 'Installer integration scenarios passed.\n' diff --git a/shared/stats-copy.ts b/shared/stats-copy.ts index 86384a4..7843f41 100644 --- a/shared/stats-copy.ts +++ b/shared/stats-copy.ts @@ -1,4 +1,4 @@ -import type { StatsSnapshot } from "./stats"; +import type { StatsAccountStats, StatsSnapshot } from "./stats"; import { DAY_MS } from "./stats-snapshot"; /* @@ -52,6 +52,10 @@ export function humanize(value: string): string { invite_created: "Sent an invite", invite_accepted: "Accepted an invite", feedback_sent: "Sent feedback", + one_day: "One day only", + two_days: "Two days", + three_to_six_days: "Three to six days", + seven_or_more_days: "Seven days or more", ok: "Installed", failed: "Failed, unspecified", unsupported_os: "Unsupported OS", @@ -225,6 +229,35 @@ export function sessionsInsight(snapshot: StatsSnapshot): string { return parts.join(" "); } +/** + * The accounts panel in a sentence: how many there are, how the range moved + * it, how many of them came back, and how many of our own were left out. + * Written so a quiet range reads as quiet rather than as a broken figure. + */ +export function accountsInsight(accounts: StatsAccountStats, rangeLabel: string): string { + const ours = accounts.excluded === 0 + ? "" + : ` ${integerFormatter.format(accounts.excluded)} of our own account${accounts.excluded === 1 ? " is" : "s are"} left out of every figure here.`; + if (accounts.total === 0) { + return `Nobody has signed up yet.${ours}`; + } + /* Over all time every account is new and nobody can have come back, so neither is said. */ + const allTime = accounts.newInRange === accounts.total && accounts.previous === null; + const parts = [ + allTime + ? `${integerFormatter.format(accounts.total)} account${accounts.total === 1 ? "" : "s"} in all.` + : `${integerFormatter.format(accounts.total)} account${accounts.total === 1 ? "" : "s"}, ${accounts.newInRange === 0 ? `none of them new in ${rangeLabel}` : `${integerFormatter.format(accounts.newInRange)} of them from ${rangeLabel}`}.`, + ]; + parts.push(accounts.activeInRange === 0 + ? `None of them opened the app in ${rangeLabel}.` + : `${integerFormatter.format(accounts.activeInRange)} used the app (${formatPercent(ratio(accounts.activeInRange, accounts.total))} of all of them)${allTime ? "" : accounts.returningInRange === 0 ? ", all of them for the first time" : `, ${integerFormatter.format(accounts.returningInRange)} of which had signed up earlier`}.`); + const previous = accounts.previous; + if (previous !== null && previous.newAccounts > 0 && accounts.newInRange === 0) { + parts.push(`The period before brought ${integerFormatter.format(previous.newAccounts)}; this one brought none.`); + } + return `${parts.join(" ")}${ours}`; +} + export interface DeltaChip { tone: "up" | "down" | "flat"; text: string; diff --git a/shared/stats-snapshot.ts b/shared/stats-snapshot.ts index e699e90..a8f10c2 100644 --- a/shared/stats-snapshot.ts +++ b/shared/stats-snapshot.ts @@ -90,6 +90,12 @@ export interface AudienceRow extends Record { count: number; } +/** The earliest day people were counted on one surface. */ +export interface SurfaceSinceRow extends Record { + surface: string; + minimum: number | null; +} + /** Distinct visitor hashes seen on one surface in a period. */ export interface PeriodUniqueRow extends Record { surface: string; @@ -125,6 +131,8 @@ export interface StatsSnapshotRows { uniquesConfigured: boolean; /** Midnight UTC of the earliest visitor day still kept, or null when there is none. */ uniquesSince: number | null; + /** The same, per surface. */ + uniquesSinceBySurface: SurfaceSinceRow[]; } /** Midnight UTC of the day that contains `at`. */ @@ -507,13 +515,18 @@ export function buildRetentionCohorts( function buildUniques(rows: StatsSnapshotRows): StatsUniques { const surfaces = Object.fromEntries( - UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0 }]), + UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0, since: null }]), ) as Record; for (const row of rows.uniques) { if (!isUniqueSurface(row.surface)) continue; const unique = Number(row.unique_count); const fresh = Math.min(unique, Number(row.new_count)); - surfaces[row.surface] = { unique, new: fresh, returning: unique - fresh }; + surfaces[row.surface] = { ...surfaces[row.surface], unique, new: fresh, returning: unique - fresh }; + } + if (rows.uniquesConfigured) { + for (const row of rows.uniquesSinceBySurface) { + if (isUniqueSurface(row.surface) && row.minimum !== null) surfaces[row.surface].since = Number(row.minimum); + } } const days = new Map(); for (const row of rows.uniqueDays) { @@ -541,11 +554,14 @@ function buildUniques(rows: StatsSnapshotRows): StatsUniques { * range has to say so, or 29,333 views next to 84 people reads as nonsense. */ export function peopleCountedSince( - uniques: Pick, + uniques: Pick & { surfaces?: Record> }, rangeStart: number, + surface?: UniqueSurface, ): number | null { - if (!uniques.configured || uniques.since === null) return null; - return uniques.since > dayStart(rangeStart) ? uniques.since : null; + if (!uniques.configured) return null; + const since = surface === undefined ? uniques.since : uniques.surfaces?.[surface]?.since ?? null; + if (since === null) return null; + return since > dayStart(rangeStart) ? since : null; } export function statsRangeStart( @@ -578,7 +594,7 @@ function buildTrend( const end = Math.floor(now / stepMs) * stepMs; const points = new Map(); for (let at = start; at <= end; at += stepMs) { - points.set(at, { at, sessions: 0, shares: 0, collaborations: 0, pageViews: 0 }); + points.set(at, { at, sessions: 0, started: 0, shares: 0, collaborations: 0, pageViews: 0, installs: 0 }); } for (const row of rows) { const at = Math.floor(Number(row.bucket) / stepMs) * stepMs; @@ -586,9 +602,11 @@ function buildTrend( if (!point) continue; const count = Number(row.count); if (row.event === "session_created") point.sessions += count; + if (row.event === "session_started") point.started += count; if (row.event === "share_opened") point.shares += count; if (row.event === "collaboration_started") point.collaborations += count; if (row.event === "page_view") point.pageViews += count; + if (row.event === "binary_download") point.installs += count; } return Array.from(points.values()).slice(-180); } diff --git a/shared/stats.ts b/shared/stats.ts index 37507ae..02a0daf 100644 --- a/shared/stats.ts +++ b/shared/stats.ts @@ -33,10 +33,16 @@ export interface StatsInstallConversion { export interface StatsSeriesPoint { at: number; + /** Sessions created. */ sessions: number; + /** Sessions whose host connected. */ + started: number; shares: number; collaborations: number; + /** Landing and documentation views, crawlers left out. */ pageViews: number; + /** Release binaries served, crawlers left out. */ + installs: number; } export interface StatsBreakdownItem { @@ -61,6 +67,13 @@ export interface StatsUniqueCount { new: number; /** Of those, seen before the range began. */ returning: number; + /** + * Midnight UTC of the earliest day this surface's people were counted, or + * null. Surfaces start on different days: machines were re-keyed after + * visitors were first counted, so a "new" machine and a "new" visitor are + * measured from different starts. + */ + since: number | null; } export interface StatsUniqueDay { @@ -192,12 +205,44 @@ export interface StatsRetention { cli: StatsRetentionCohort[]; } -/** Aggregates the accounts app answers with, when the dashboard is linked to it. */ +/** The account counts over one period, so a range can be read against the one before it. */ +export interface StatsAccountPeriod { + total: number; + newAccounts: number; + active: number; + returning: number; +} + +/** How many accounts have used the app on how many separate days. */ +export interface StatsAccountEngagement { + label: string; + value: number; +} + +/** + * Aggregates the accounts app answers with, when the dashboard is linked to + * it. Our own accounts are left out of every figure here before it is sent; + * `excluded` says how many, so the number can be checked rather than trusted. + */ export interface StatsAccountStats { total: number; newInRange: number; activeInRange: number; + /** Of the active, that had signed up before the range began: the ones that came back. */ + returningInRange: number; + /** The same four over the period of equal length before the range, or null over all time. */ + previous: StatsAccountPeriod | null; newByDay: { day: number; count: number }[]; + /** Accounts that used the app on each day, over the same days as newByDay. */ + activeByDay: { day: number; count: number }[]; + /** Accounts by how many separate days they have used the app. */ + engagement: StatsAccountEngagement[]; + /** How many accounts the engagement figures are over: those that signed up since activeSince. */ + engagementBase: number; + /** Midnight UTC of the first day an account was recorded as active, or null. */ + activeSince: number | null; + /** Accounts left out of every figure here because they are ours. */ + excluded: number; cohorts: StatsRetentionCohort[]; /** Things done in the app in the range, by kind: machines linked, commands sent. Counts of things, not of accounts. */ events: Record; diff --git a/tests/account-stats.test.ts b/tests/account-stats.test.ts new file mode 100644 index 0000000..2f83b4c --- /dev/null +++ b/tests/account-stats.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { completeAccountStats, fetchAccountStats } from "../worker/account-stats"; + +/* What the accounts app answers today. */ +const complete = { + total: 17, + newInRange: 3, + activeInRange: 4, + returningInRange: 1, + previous: { total: 14, newAccounts: 5, active: 2, returning: 0 }, + newByDay: [{ day: 1, count: 2 }], + activeByDay: [{ day: 1, count: 3 }], + engagement: [{ label: "one_day", value: 9 }], + engagementBase: 12, + activeSince: 86_400_000, + excluded: 21, + cohorts: [{ weekStart: 1, size: 2, active: [1] }], + events: { machine_linked: 4 }, +}; + +function answering(body: unknown, status = 200): typeof fetch { + return (async () => new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + })) as unknown as typeof fetch; +} + +describe("fetchAccountStats", () => { + it("is nothing at all when the dashboard is not linked to an app", async () => { + expect(await fetchAccountStats(undefined, "token", "7d", answering(complete))).toBeNull(); + expect(await fetchAccountStats("https://app.example", " ", "7d", answering(complete))).toBeNull(); + }); + + it("passes a complete answer through", async () => { + expect(await fetchAccountStats("https://app.example/", "token", "7d", answering(complete))).toEqual(complete); + }); + + it("names the failure rather than showing a quiet week", async () => { + expect(await fetchAccountStats("https://app.example", "token", "7d", answering({}, 401))) + .toEqual({ error: "accounts app answered 401" }); + expect(await fetchAccountStats("https://app.example", "token", "7d", answering({ total: 1 }))) + .toEqual({ error: "accounts app answered in an unexpected shape" }); + const throwing = (() => Promise.reject(new Error("no route to host"))) as unknown as typeof fetch; + expect(await fetchAccountStats("https://app.example", "token", "7d", throwing)) + .toEqual({ error: "accounts app did not answer" }); + }); + + it("asks the app for the range the dashboard is showing, with the token", async () => { + let asked = ""; + let authorization = ""; + const capture = (async (url: string, init: RequestInit) => { + asked = url; + authorization = new Headers(init.headers).get("authorization") ?? ""; + return new Response(JSON.stringify(complete)); + }) as unknown as typeof fetch; + await fetchAccountStats("https://app.example//", "token", "30d", capture); + expect(asked).toBe("https://app.example/api/stats/accounts?range=30d"); + expect(authorization).toBe("Bearer token"); + }); +}); + +describe("completeAccountStats", () => { + it("fills in what an app deployed before the Worker does not send yet", () => { + const old = { + total: 17, + newInRange: 3, + activeInRange: 4, + newByDay: [{ day: 1, count: 2 }], + cohorts: [{ weekStart: 1, size: 2, active: [1] }], + }; + expect(completeAccountStats(old)).toEqual({ + ...old, + returningInRange: 0, + previous: null, + activeByDay: [], + engagement: [], + engagementBase: 0, + activeSince: null, + excluded: 0, + events: {}, + }); + }); + + it("drops a malformed row rather than drawing a chart from it", () => { + const filled = completeAccountStats({ + ...complete, + newByDay: [{ day: 1, count: 2 }, { day: "one", count: 2 }, null], + previous: { total: 1 }, + engagement: [{ label: "one_day", value: 1 }, { label: 3, value: 1 }], + events: { machine_linked: "many" }, + }); + expect(filled.newByDay).toEqual([{ day: 1, count: 2 }]); + expect(filled.previous).toBeNull(); + expect(filled.engagement).toEqual([{ label: "one_day", value: 1 }]); + expect(filled.events).toEqual({}); + }); +}); diff --git a/tests/analytics.test.ts b/tests/analytics.test.ts index ae70c4a..1f9faca 100644 --- a/tests/analytics.test.ts +++ b/tests/analytics.test.ts @@ -61,6 +61,39 @@ describe("analytics", () => { }); }); + it("counts monitors as crawlers and HTTP libraries as tools, never as people", () => { + for (const agent of ["shell.online-downloads-check", "shell.online-install-check", "Better Uptime Bot", "Pingdom.com_bot", "Checkly/1.0", "Site24x7", "Datadog Synthetics"]) { + expect(classifyDevice(agent)).toBe("bot"); + expect(classifyClient(agent)).toBe("bot"); + } + for (const agent of [ + "node", + "node-fetch/1.0", + "undici", + "Go-http-client/1.1", + "python-requests/2.31.0", + "Java/17.0.2", + "okhttp/4.12.0", + "axios/1.6.0", + "Mozilla/5.0 (Windows NT 10.0; Microsoft Windows 10.0.19045; en-US) PowerShell/7.4.1", + "Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.1", + ]) { + expect(classifyDevice(agent)).toBe("cli"); + expect(classifyClient(agent)).toBe("tool"); + } + expect(classifyDevice("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/129.0 Safari/537.36")).toBe("desktop"); + expect(classifyDevice("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 Version/17.6 Safari/605.1.15")).toBe("desktop"); + expect(classifyDevice("")).toBe("unknown"); + }); + + it("does not count a page a browser fetched ahead of time", () => { + const headers = { Accept: "text/html", "Sec-Fetch-Dest": "document" }; + expect(isDocumentNavigation(new Request("https://shell.online/", { headers }))).toBe(true); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch;prerender" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, Purpose: "prefetch" } }))).toBe(false); + }); + it("reduces user agents and referrers to coarse categories", () => { expect(classifyDevice("shell/0.3.4")).toBe("cli"); expect(classifyClient("shell/0.3.4")).toBe("shell/0.3.4"); diff --git a/tests/stats-copy.test.ts b/tests/stats-copy.test.ts index e530483..87fd831 100644 --- a/tests/stats-copy.test.ts +++ b/tests/stats-copy.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import { buildStatsSnapshot, DAY_MS, dayStart, type StatsSnapshotRows } from "../shared/stats-snapshot"; +import type { StatsAccountStats } from "../shared/stats"; import { + accountsInsight, deltaChip, formatDuration, formatPercent, @@ -55,6 +57,7 @@ function rows(overrides: Partial = {}): StatsSnapshotRows { installConversion: null, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + uniquesSinceBySurface: [], ...overrides, }; } @@ -129,3 +132,53 @@ describe("the dashboard's copy", () => { expect(humanize("some_new_thing")).toBe("Some New Thing"); }); }); + +describe("accountsInsight", () => { + const accounts = (overrides: Partial = {}): StatsAccountStats => ({ + total: 20, + newInRange: 4, + activeInRange: 5, + returningInRange: 2, + previous: { total: 16, newAccounts: 6, active: 3, returning: 1 }, + newByDay: [], + activeByDay: [], + engagement: [], + engagementBase: 0, + activeSince: null, + excluded: 0, + cohorts: [], + events: {}, + ...overrides, + }); + + it("says how many there are, how many are new, and how many came back", () => { + expect(accountsInsight(accounts(), "last 7d")).toBe( + "20 accounts, 4 of them from last 7d. 5 used the app (25% of all of them), 2 of which had signed up earlier.", + ); + }); + + it("reads a quiet range as quiet rather than as a broken figure", () => { + expect(accountsInsight(accounts({ newInRange: 0, activeInRange: 0, returningInRange: 0 }), "last 7d")).toBe( + "20 accounts, none of them new in last 7d. None of them opened the app in last 7d. The period before brought 6; this one brought none.", + ); + expect(accountsInsight(accounts({ total: 0, newInRange: 0, activeInRange: 0, returningInRange: 0 }), "last 7d")) + .toBe("Nobody has signed up yet."); + }); + + it("says how many of our own accounts were left out, whatever else it says", () => { + expect(accountsInsight(accounts({ excluded: 21 }), "last 7d")).toContain( + "21 of our own accounts are left out of every figure here.", + ); + expect(accountsInsight(accounts({ total: 0, newInRange: 0, activeInRange: 0, returningInRange: 0, excluded: 1 }), "all time")) + .toBe("Nobody has signed up yet. 1 of our own account is left out of every figure here."); + }); + + it("does not claim a first-time account came back", () => { + expect(accountsInsight(accounts({ returningInRange: 0 }), "last 7d")).toContain("all of them for the first time"); + }); + + it("does not say new or returning over all time, where every account is both or neither", () => { + const all = accounts({ total: 20, newInRange: 20, activeInRange: 12, returningInRange: 0, previous: null }); + expect(accountsInsight(all, "all time")).toBe("20 accounts in all. 12 used the app (60% of all of them)."); + }); +}); diff --git a/tests/stats-database.test.ts b/tests/stats-database.test.ts index 5ee76fd..94d3591 100644 --- a/tests/stats-database.test.ts +++ b/tests/stats-database.test.ts @@ -8,6 +8,8 @@ import { collectStatsRows, HOUR_MS, initializeStatsSchema, + MACHINE_KEY_DAY, + migrateStatsData, parseStatsPresence, parseStatsRecord, recordStatsEvent, @@ -101,6 +103,19 @@ describe("the dashboard's database", () => { const day = collectStatsRows(sql, "24h", false, now); expect(day.rows.summary).toEqual([]); expect(day.rows.previous).toMatchObject({ rangeStart: now - 2 * DAY_MS, summary: [{ count: 2 }] }); + + /* Crawlers stay out of the charts, and installs and started sessions join them. */ + recordStatsEvent(sql, event(daysAgo(1), "page_view", "landing", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "cli", client: "curl" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "session_started", "cli", { device: "cli", client: "shell/0.16.0" })); + const charted = collectStatsRows(sql, "7d", false, now).rows.trend; + const hour = Math.floor(daysAgo(1) / HOUR_MS) * HOUR_MS; + expect(charted).toEqual([ + { bucket: hour, event: "binary_download", count: 1 }, + { bucket: hour, event: "page_view", count: 2 }, + { bucket: hour, event: "session_started", count: 1 }, + ]); }); it("splits the counted events by device class and ranks dimensions by count", () => { @@ -140,6 +155,10 @@ describe("the dashboard's database", () => { const { rows } = collectStatsRows(sql, "7d", true, now); expect(rows.uniquesSince).toBe(dayStart(daysAgo(10))); + expect(rows.uniquesSinceBySurface).toEqual([ + { surface: "cli", minimum: dayStart(daysAgo(1)) }, + { surface: "site", minimum: dayStart(daysAgo(10)) }, + ]); expect(bySurface(rows.uniques)).toEqual([ { surface: "cli", unique_count: 1, new_count: 1 }, { surface: "site", unique_count: 2, new_count: 1 }, @@ -192,6 +211,38 @@ describe("the dashboard's database", () => { expect(collectStatsRows(sql, "all", true, now).rows.installConversion).toMatchObject({ installers: 6 }); }); + it("drops the machine rows keyed the old way once, and never again", () => { + const sql = freshDatabase(); + /* A database from before the mark existed. */ + sql.exec("DELETE FROM dashboard_marks"); + const cutover = MACHINE_KEY_DAY + 6 * HOUR_MS; + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("o") })); + recordStatsEvent(sql, event(cutover, "binary_download", "darwin-arm64", { device: "cli", visitor: hash("p") })); + recordStatsEvent(sql, event(cutover - DAY_MS, "page_view", "landing", { visitor: hash("v") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("n") })); + /* Seen before and after the cut-over: the row survives, its first day with it. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + + migrateStatsData(sql); + const left = sql.exec<{ surface: string; visitor: string; first_day: number }>( + "SELECT surface, visitor, first_day FROM visitors ORDER BY surface, visitor", + ).toArray(); + expect(left).toEqual([ + { surface: "cli", visitor: hash("k"), first_day: dayStart(cutover - DAY_MS) }, + { surface: "cli", visitor: hash("n"), first_day: dayStart(cutover + DAY_MS) }, + { surface: "site", visitor: hash("v"), first_day: dayStart(cutover - DAY_MS) }, + ]); + expect(sql.exec<{ day: number }>("SELECT day FROM visitor_days WHERE visitor = ? ORDER BY day", hash("k")).toArray()) + .toEqual([{ day: dayStart(cutover + DAY_MS) }]); + + /* Done once: a later old-looking row is left alone. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("z") })); + initializeStatsSchema(sql); + migrateStatsData(sql); + expect(sql.exec<{ visitor: string }>("SELECT visitor FROM visitors WHERE visitor = ?", hash("z")).toArray()).toHaveLength(1); + }); + it("keeps a live presence lease until it ends, and drops it when told", () => { const sql = freshDatabase(); const key = (letter: string): string => letter.repeat(22); diff --git a/web/stats.css b/web/stats.css index a43a6fa..8916d41 100644 --- a/web/stats.css +++ b/web/stats.css @@ -1949,40 +1949,6 @@ html.stats-document #app { line-height: 1.5; } -.accounts-panel .kpi-row { - display: grid; - margin-bottom: 14px; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 10px; -} - -.accounts-panel .kpi-row span { - display: grid; - padding: 12px; - border: 1px solid rgb(150 170 207 / 9%); - border-radius: 12px; - background: rgb(12 17 26 / 45%); - gap: 4px; -} - -.accounts-panel .kpi-row small { - color: var(--stats-muted); - font-size: 10.5px; - letter-spacing: 0.04em; - text-transform: uppercase; -} - -.accounts-panel .kpi-row b { - color: var(--stats-ink); - font-size: 22px; - font-weight: 620; - font-variant-numeric: tabular-nums; -} - -.accounts-panel .kpi-spark { - height: 34px; -} - .traffic-split.is-wide { flex-wrap: wrap; } @@ -1997,19 +1963,6 @@ html.stats-document #app { } } -.accounts-panel .kpi-spark svg { - display: block; - width: 100%; - height: 34px; -} - -.accounts-panel .kpi-spark polyline { - fill: none; - stroke: #8eafff; - stroke-width: 1.5; - vector-effect: non-scaling-stroke; -} - .stats-content > .funnel-panel, .stats-content > .traffic-panel { padding: 20px 22px 18px; @@ -2115,3 +2068,57 @@ html.stats-document #app { font-size: 9px; font-weight: 500; } + +/* ---- Accounts ---- */ + +/* + * The accounts block sits between the funnel and the traffic it came from, + * and is the one part of the page counting people exactly rather than by a + * keyed hash. It gets a heading of its own so the boundary is visible: + * everything under it is accounts until the next panel's header. + */ +.stats-section-head { + display: flex; + margin: 10px 2px -4px; + align-items: baseline; + justify-content: space-between; + gap: 16px; +} + +.stats-section-head h2 { + margin: 2px 0 0; + color: var(--stats-ink); + font-size: 16px; + font-weight: 600; + letter-spacing: -0.01em; +} + +/* Four headline figures, where the page's other row has six. */ +.stats-kpis.accounts-kpis { + grid-template-columns: repeat(4, minmax(0, 1fr)); +} + +@media (max-width: 1180px) { + .stats-kpis.accounts-kpis { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media (max-width: 340px) { + .stats-kpis.accounts-kpis { + grid-template-columns: 1fr; + } +} + +.accounts-events-panel { + padding: 20px 22px 18px; +} + +/* Now the whole of its own panel, so it no longer hangs off what was above it. */ +.accounts-events-panel .accounts-events { + margin-top: 0; + padding-top: 0; + border-top: 0; +} + +.kind-band .breakdown-list > div > i b { background: linear-gradient(90deg, #4a6a86, #8ec2dc); } diff --git a/web/stats.ts b/web/stats.ts index bbdfe99..2874d3c 100644 --- a/web/stats.ts +++ b/web/stats.ts @@ -1,14 +1,18 @@ import { INSTALL_CONVERSION_DAYS, STATS_RANGES, + type StatsAccounts, + type StatsAccountStats, type StatsBreakdownItem, type StatsRange, type StatsRetentionCohort, type StatsSeriesPoint, type StatsSnapshot, + type UniqueSurface, } from "../shared/stats"; -import { peopleCountedSince } from "../shared/stats-snapshot"; +import { DAY_MS, peopleCountedSince } from "../shared/stats-snapshot"; import { + accountsInsight, deltaChip, formatDuration, formatPercent, @@ -23,21 +27,47 @@ import { import { RELEASE_CHECKSUMS_PATH, RELEASE_VERSION } from "../shared/release"; import "./stats.css"; -type SeriesKey = "sessions" | "shares" | "collaborations" | "pageViews"; +type SeriesKey = "sessions" | "started" | "shares" | "collaborations" | "pageViews" | "installs"; interface ChartSeries { - key: SeriesKey; + key: string; label: string; color: string; } +/** + * One moment on a chart, and the value of each series at it. + * + * The trend the Worker sends is one shape; the accounts app's days are + * another, and both are drawn by the same code, so both are turned into this + * first rather than the chart learning about either of them. + */ +interface ChartPoint { + at: number; + values: Record; +} + +function trendPoints(trend: StatsSeriesPoint[]): ChartPoint[] { + return trend.map((point) => ({ + at: point.at, + values: { + sessions: point.sessions, + started: point.started, + shares: point.shares, + collaborations: point.collaborations, + pageViews: point.pageViews, + installs: point.installs, + }, + })); +} + const ACTIVITY_SERIES: ChartSeries[] = [ { key: "sessions", label: "Created", color: "#8eafff" }, { key: "shares", label: "Shared", color: "#75dac2" }, { key: "collaborations", label: "Collaborated", color: "#d7a6ff" }, ]; const TRAFFIC_SERIES: ChartSeries[] = [ - { key: "pageViews", label: "Page views", color: "#9ab7e8" }, + { key: "pageViews", label: "Page views by people", color: "#9ab7e8" }, ]; let activeDashboardCleanup: (() => void) | null = null; let statsRenderId = 0; @@ -345,6 +375,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { const previous = snapshot.previous; const rangeLabel = snapshot.range === "all" ? "all time" : `last ${snapshot.range}`; const priorLabel = snapshot.range === "all" ? "" : `the ${snapshot.range} before`; + const trend = trendPoints(snapshot.trend); const outcomes = snapshot.breakdowns.outcomes; const endedSessions = outcomes.reduce((sum, item) => sum + item.value, 0); const people = snapshot.uniques; @@ -357,7 +388,8 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { */ const peopleSince = peopleCountedSince(people, snapshot.rangeStart); /* Plain text: renderKpi escapes its detail. */ - const sinceNote = peopleSince === null ? "" : ` · counted since ${formatDay(peopleSince)}`; + const visitorsNote = peopleNote(snapshot, "site"); + const machinesNote = peopleNote(snapshot, "cli"); const totalViews = metrics.landingViews + metrics.docsViews; const otherViews = audiences.views.tools + audiences.views.unknown; const ctaByLink = snapshot.targets @@ -390,9 +422,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${renderKpi( "Visitors", people.configured ? site.unique : "—", - people.configured - ? `${integerFormatter.format(site.new)} new · ${integerFormatter.format(site.returning)} returning${sinceNote}` - : "people are not counted yet", + people.configured ? visitorsNote : "people are not counted yet", snapshot.trend, "pageViews", "silver", @@ -416,7 +446,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ? "no installer runs in this range" : `${formatPercent(ratio(figures.installs, figures.installerRuns))} of ${integerFormatter.format(figures.installerRuns)} installer run${figures.installerRuns === 1 ? "" : "s"}`, snapshot.trend, - "sessions", + "installs", "amber", delta(figures.installs, previous?.figures.installs), )} @@ -424,10 +454,10 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { "Sessions started", figures.sessionsStarted, people.configured - ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"}, ${integerFormatter.format(cli.new)} new${sinceNote}` + ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"} · ${machinesNote}` : `${integerFormatter.format(metrics.sessionsCreated)} created`, snapshot.trend, - "sessions", + "started", "violet", delta(figures.sessionsStarted, previous?.figures.sessionsStarted), )} @@ -461,12 +491,14 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${escapeHtml(rangeLabel)}

${escapeHtml(funnelInsight(snapshot))}

- ${renderFunnel(snapshot, peopleSince)} + ${renderFunnel(snapshot)} ${peopleSince === null ? "" : `

People have been counted since ${escapeHtml(formatDay(peopleSince))}; event counts run from the start of the range. Until the range begins after that day, a people figure covers fewer days than the count beside it.

`} ${renderInstallConversion(snapshot)} ${renderUniquesStrip(snapshot)} + ${renderAccounts(snapshot)} +
@@ -474,7 +506,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${integerFormatter.format(totalViews)} views

${escapeHtml(trafficInsight(snapshot))}

- ${renderTimeChart("traffic", snapshot.trend, TRAFFIC_SERIES, snapshot.range, true)} + ${renderTimeChart("traffic", trend, TRAFFIC_SERIES, snapshot.trendStepMs, true)}
People ${integerFormatter.format(audiences.views.browsers)} Crawlers ${integerFormatter.format(audiences.views.crawlers)} @@ -516,7 +548,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void {

${escapeHtml(sessionsInsight(snapshot))}

- ${renderTimeChart("activity", snapshot.trend, ACTIVITY_SERIES, snapshot.range)} + ${renderTimeChart("activity", trend, ACTIVITY_SERIES, snapshot.trendStepMs)}
@@ -607,8 +639,6 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { )}
- ${renderAccounts(snapshot)} -
Complete event ledgerEvery tracked aggregate, every audience
@@ -634,12 +664,14 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${snapshot.collectingSince ? `Collecting exact dashboard metrics since ${formatDate(snapshot.collectingSince)}.` : "Waiting for the first event."} ${people.configured && people.since !== null ? `Counting people since ${escapeHtml(formatDay(people.since, "long"))}.` : ""} Crawlers are requests whose user agent says so; they stay in the ledger and out of every figure about people. + ${accountsExcludedNote(snapshot.accounts)} People are keyed hashes of address and browser family, forgotten ${people.memoryDays} days after they were last seen; a person seen again after that counts as new.
`; - bindChartInteraction(container.querySelector("#activity-chart"), snapshot.trend, ACTIVITY_SERIES); - bindChartInteraction(container.querySelector("#traffic-chart"), snapshot.trend, TRAFFIC_SERIES); + bindChartInteraction(container.querySelector("#activity-chart"), trend, ACTIVITY_SERIES, snapshot.trendStepMs); + bindChartInteraction(container.querySelector("#traffic-chart"), trend, TRAFFIC_SERIES, snapshot.trendStepMs); + bindChartInteraction(container.querySelector("#accounts-chart"), accountPoints(snapshot.accounts), ACCOUNT_SERIES, DAY_MS); } /** What is happening this minute, under the page title, so it is not mistaken for a range figure. */ @@ -690,11 +722,17 @@ function renderSparkline(values: number[]): string { return ``; } +/* + * `stepMs` is how much time one point covers, not which range is selected: + * the trend's step changes with the range, and the accounts chart is always a + * day whatever the range is. It decides whether an axis label and a tooltip + * name an hour or a day. + */ function renderTimeChart( id: string, - points: StatsSeriesPoint[], + points: ChartPoint[], series: ChartSeries[], - range: StatsRange, + stepMs: number, compact = false, ): string { const width = 760; @@ -702,12 +740,12 @@ function renderTimeChart( const top = 16; const bottom = compact ? 24 : 32; const usableHeight = height - top - bottom; - const maximum = Math.max(1, ...points.flatMap((point) => series.map((item) => point[item.key]))); + const maximum = Math.max(1, ...points.flatMap((point) => series.map((item) => point.values[item.key] ?? 0))); const roundedMaximum = niceMaximum(maximum); const paths = series.map((item, index) => { const coordinates = points.map((point, pointIndex) => ({ x: points.length <= 1 ? width / 2 : pointIndex * width / (points.length - 1), - y: top + usableHeight - point[item.key] / roundedMaximum * usableHeight, + y: top + usableHeight - (point.values[item.key] ?? 0) / roundedMaximum * usableHeight, })); const line = smoothPath(coordinates); const area = coordinates.length === 0 @@ -726,7 +764,7 @@ function renderTimeChart( const xLabels = axisLabelIndexes(points.length).map((index) => { const point = points[index]; const x = points.length <= 1 ? width / 2 : index * width / (points.length - 1); - return `${point ? escapeHtml(formatChartTime(point.at, range)) : ""}`; + return `${point ? escapeHtml(formatChartTime(point.at, stepMs)) : ""}`; }).join(""); return ` @@ -748,9 +786,16 @@ function renderTimeChart( `; } -function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): string { +/** Which surface a funnel step's people are counted on, for the day their count starts. */ +const STEP_SURFACES: Record = { visited: "site", installer: "install", session: "cli", opened: "viewer" }; + +function renderFunnel(snapshot: StatsSnapshot): string { const steps = snapshot.funnel; - const since = peopleSince === null ? "" : ` since ${escapeHtml(formatDay(peopleSince))}`; + const sinceFor = (key: string): string => { + const surface = STEP_SURFACES[key]; + const since = surface === undefined ? null : peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + return since === null ? "" : ` since ${escapeHtml(formatDay(since))}`; + }; const colors = ["#9ab7e8", "#8eafff", "#819de5", "#f4bd78", "#8eafff", "#75dac2", "#d7a6ff"]; const maximum = Math.max(1, ...steps.map((step) => step.count)); return ` @@ -771,7 +816,7 @@ function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): stri return `
${escapeHtml(step.label)} - ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${since}`} + ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${sinceFor(step.key)}`}
${escapeHtml(share)} ${excluded} @@ -799,15 +844,30 @@ function renderInstallConversion(snapshot: StatsSnapshot): string { return `

${escapeHtml(verdict)} Machines are followed by address from the binary download to the first session.

`; } +/* + * What to say beside a people figure. New means first seen since the range + * began, so until a whole range has passed since this surface's people were + * first counted, everyone is new and the split says nothing: name the day it + * starts to. The 24h range counts people by UTC day, which is two of them. + */ +function peopleNote(snapshot: StatsSnapshot, surface: UniqueSurface): string { + const count = snapshot.uniques.surfaces[surface]; + const days = snapshot.range === "24h" ? " · by UTC day, so two days" : ""; + const since = peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + if (since === null) return `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} returning${days}`; + const knownFrom = formatDay(since + (snapshot.generatedAt - snapshot.rangeStart)); + return `counted since ${formatDay(since)}; new or returning cannot be told until ${knownFrom}${days}`; +} + function renderUniquesStrip(snapshot: StatsSnapshot): string { const people = snapshot.uniques; - const cell = (label: string, surface: keyof typeof people.surfaces): string => { + const cell = (label: string, surface: UniqueSurface): string => { const count = people.surfaces[surface]; return ` ${escapeHtml(label)} ${people.configured ? integerFormatter.format(count.unique) : "—"} - ${people.configured ? `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} back` : "not counted"} + ${people.configured ? escapeHtml(peopleNote(snapshot, surface)) : "not counted"} `; }; @@ -865,6 +925,16 @@ function renderCohorts( `; } +/** + * Accounts: the one exact count of people on this page. + * + * Everything else here is inferred from requests -- a keyed hash is the best + * guess at a person that a page with no sign-in can make. An account is a + * person who gave us an address and came back to it, so this panel is read + * first and laid out to be read in one pass: how many there are and how that + * moved, then how many of them are actually using it, then a day-by-day line + * of both, then what they did and whether they stayed. + */ function renderAccounts(snapshot: StatsSnapshot): string { const accounts = snapshot.accounts; if (accounts === null) return ""; @@ -877,22 +947,90 @@ function renderAccounts(snapshot: StatsSnapshot): string { `; } const rangeLabel = snapshot.range === "all" ? "all time" : `last ${snapshot.range}`; + const priorLabel = snapshot.range === "all" ? "" : `the ${snapshot.range} before`; + const previous = accounts.previous; + const points = accountPoints(accounts); + const signups = points.map((point) => point.values.signups); + const active = points.map((point) => point.values.active); + const stale = accounts.total - accounts.activeInRange; return ` -
+
+
+ The only exact count of people on this page +

Accounts

+
+ ${escapeHtml(rangeLabel)} +
+ +
+ ${renderAccountKpi( + "Accounts", + accounts.total, + accounts.newInRange === 0 + ? `no new account in ${rangeLabel}` + : `${integerFormatter.format(accounts.newInRange)} signed up in ${rangeLabel}`, + signups, + "blue", + renderDelta(accounts.total, previous?.total ?? null, priorLabel), + )} + ${renderAccountKpi( + "Signed up", + accounts.newInRange, + snapshot.range === "all" ? "every account there is" : `in ${rangeLabel}`, + signups, + "green", + renderDelta(accounts.newInRange, previous?.newAccounts ?? null, priorLabel), + )} + ${renderAccountKpi( + "Used the app", + accounts.activeInRange, + accounts.total === 0 + ? "no accounts yet" + : `${formatPercent(ratio(accounts.activeInRange, accounts.total))} of all accounts · ${integerFormatter.format(stale)} did not`, + active, + "violet", + renderDelta(accounts.activeInRange, previous?.active ?? null, priorLabel), + )} + ${renderAccountKpi( + "Came back", + accounts.returningInRange, + snapshot.range === "all" + ? "nothing comes before all time, so nothing here has come back to it" + : accounts.activeInRange === 0 + ? "nobody used the app in this range" + : `${formatPercent(ratio(accounts.returningInRange, accounts.activeInRange))} of the accounts that used it had signed up earlier`, + active, + "pink", + renderDelta(accounts.returningInRange, previous?.returning ?? null, priorLabel), + )} +
+ +
-
Exact, from the accounts app

Accounts

+
Sign-ups, and the accounts that opened it

Day by day

+
+ ${ACCOUNT_SERIES.map((series) => `${series.label}`).join("")} +
+
+

${escapeHtml(accountsInsight(accounts, rangeLabel))}

+ ${points.length === 0 + ? '

No account has signed up yet.

' + : renderTimeChart("accounts", points, ACCOUNT_SERIES, DAY_MS)} +

Every day since the first account, whatever range is chosen above: accounts arrive a few a day, and a day is the smallest step that says anything. A day counts an account as active if it signed up or made a request that day.${accountsSinceNote(accounts)}

+
+ +
+
+
What they did

In the app

${escapeHtml(rangeLabel)}
-
- Accounts${integerFormatter.format(accounts.total)} - New${integerFormatter.format(accounts.newInRange)} - Active${integerFormatter.format(accounts.activeInRange)} -
-
${renderSparkline(accounts.newByDay.map((point) => point.count))}
-

New accounts per day. Active means the account used the app in the range.

${renderAccountEvents(accounts.events)}
+
+ +
+ ${renderEngagement(accounts)} ${renderCohorts( "Accounts that came back", "Accounts by the week they signed up", @@ -905,6 +1043,91 @@ function renderAccounts(snapshot: StatsSnapshot): string { `; } +/** The footer line that makes the exclusion visible wherever the reader stops. */ +function accountsExcludedNote(accounts: StatsAccounts): string { + if (accounts === null || "error" in accounts || accounts.excluded === 0) return ""; + return `${integerFormatter.format(accounts.excluded)} of our own account${accounts.excluded === 1 ? " is" : "s are"} left out of every account figure, along with everything ${accounts.excluded === 1 ? "it" : "they"} did in the app. Set by STATS_EXCLUDE on the accounts app.`; +} + +/* + * Used first, signed up second. An account is active on the day it signs up, + * so the first line is never below the second: drawn this way the filled area + * belongs to the larger of the two and neither line hides the other. + */ +const ACCOUNT_SERIES: ChartSeries[] = [ + { key: "active", label: "Used the app", color: "#8eafff" }, + { key: "signups", label: "Signed up", color: "#75dac2" }, +]; + +/** + * The account days as chart points, trimmed to start at the first day + * anything happened. The app sends a fixed window of days whatever the range, + * and a product ten days old would otherwise be drawn as eighty days of zero. + */ +function accountPoints(accounts: StatsAccounts): ChartPoint[] { + if (accounts === null || "error" in accounts) return []; + const active = new Map(accounts.activeByDay.map((point) => [point.day, point.count])); + const points = accounts.newByDay.map((point) => ({ + at: point.day, + values: { signups: point.count, active: active.get(point.day) ?? 0 }, + })); + const first = points.findIndex((point) => point.values.signups > 0 || point.values.active > 0); + return first === -1 ? [] : points.slice(first); +} + +/** Says when the app started recording days, where that is younger than the accounts. */ +function accountsSinceNote(accounts: StatsAccountStats): string { + if (accounts.activeSince === null) return ""; + return ` Days have been recorded since ${formatDay(accounts.activeSince, "long")}; before that an account is only counted active on the day it signed up.`; +} + +/** + * How deeply accounts use the app: not how many came, but how many days each + * of them has been in it. The bands are always all four, zeros included -- + * the shape of the distribution is the point, and hiding the empty end of it + * would make one busy band look like the whole picture. + */ +function renderEngagement(accounts: StatsAccountStats): string { + const maximum = Math.max(1, ...accounts.engagement.map((bucket) => bucket.value)); + return ` +
+
+
Accounts by the days they have been in the app

How much they use it

+ ${integerFormatter.format(accounts.engagementBase)} +
+
+ ${accounts.engagementBase === 0 + ? "No account has signed up since days started being recorded." + : accounts.engagement.map((bucket) => ` +
+ ${escapeHtml(humanize(bucket.label))} + + ${integerFormatter.format(bucket.value)} +
+ `).join("")} +
+

Over the accounts that signed up since days started being recorded. An older account is missing the days before that and would read here as one that never came back, so it is left out rather than counted against the product.

+
+ `; +} + +function renderAccountKpi( + label: string, + value: number, + detail: string, + values: number[], + tone: string, + delta: string, +): string { + return ` +
+
${escapeHtml(label)}${integerFormatter.format(value)}${delta}
+
${renderSparkline(values)}
+ ${escapeHtml(detail)} +
+ `; +} + /** A dashboard day is a UTC day, so it is named in UTC wherever the reader is. */ function formatDay(at: number, month: "short" | "long" = "short"): string { return new Date(at).toLocaleDateString([], month === "long" @@ -944,13 +1167,14 @@ function renderDonut(items: StatsBreakdownItem[]): string { /** * What accounts did, in the order the product hopes for: link a machine, * register a session, send it a command. Counts of things done, not of - * accounts, since the app sends no identifiers with them. + * accounts, since the app sends no identifiers with them -- and only what + * customers did: the app counts our own separately and never sends it. */ function renderAccountEvents(events: Record): string { const order = ["machine_linked", "session_registered", "command_sent", "vault_created", "invite_created", "invite_accepted", "feedback_sent"]; const items = order.filter((key) => (events[key] ?? 0) > 0).map((key) => ({ label: key, value: events[key] })); if (items.length === 0) { - return '

Nothing done in the app in this range yet: no machine linked, session registered, command sent, vault created, invite, or feedback.

'; + return '

Nothing done in the app in this range: no machine linked, session registered, command sent, vault created, invite, or feedback. What we did ourselves is counted apart and never shown here.

'; } const maximum = Math.max(1, ...items.map((item) => item.value)); return ` @@ -963,7 +1187,7 @@ function renderAccountEvents(events: Record): string {
`).join("")} -

Things done, not distinct accounts: one account linking three machines counts three times.

+

Things done, not distinct accounts: one account linking three machines counts three times. What we did ourselves is counted apart and never shown here.

`; } @@ -1024,8 +1248,9 @@ function renderBreakdown( function bindChartInteraction( element: Element | null, - points: StatsSeriesPoint[], + points: ChartPoint[], series: ChartSeries[], + stepMs: number, ): void { if (!(element instanceof HTMLElement) || points.length === 0) return; const tooltip = element.querySelector(".chart-tooltip"); @@ -1040,8 +1265,8 @@ function bindChartInteraction( guide.style.left = `${left}%`; tooltip.style.left = `${left}%`; tooltip.innerHTML = ` - - ${series.map((item) => `${item.label}${integerFormatter.format(point[item.key])}`).join("")} + + ${series.map((item) => `${item.label}${integerFormatter.format(point.values[item.key] ?? 0)}`).join("")} `; element.classList.add("hovering"); }; @@ -1089,13 +1314,19 @@ function renderSkeleton(): string { `; } -function formatChartTime(at: number, range: StatsRange): string { +function formatChartTime(at: number, stepMs: number): string { const date = new Date(at); - if (range === "24h") return date.toLocaleTimeString([], { hour: "numeric" }); + if (stepMs < DAY_MS) return date.toLocaleTimeString([], { hour: "numeric" }); return date.toLocaleDateString([], { month: "short", day: "numeric" }); } -function formatTooltipTime(at: number): string { +/* + * The hour is only meaningful where a point covers less than a day. A point + * that covers a whole UTC day is named as that day: a local time on it would + * put it in the wrong one for most of the world. + */ +function formatTooltipTime(at: number, stepMs: number): string { + if (stepMs >= DAY_MS) return `${formatDay(at, "long")} UTC`; return new Date(at).toLocaleString([], { month: "short", day: "numeric", diff --git a/worker/account-stats.ts b/worker/account-stats.ts new file mode 100644 index 0000000..c58c67b --- /dev/null +++ b/worker/account-stats.ts @@ -0,0 +1,97 @@ +import type { StatsAccountPeriod, StatsAccounts, StatsAccountStats, StatsRange } from "../shared/stats"; + +/* + * The accounts app keeps the only exact count of people: accounts. It answers + * aggregates -- how many, how many new, how many active, how many back each + * week -- to a bearer token, and nothing per person. Our own accounts are + * already out of every figure by the time it answers; see the app's + * internal-accounts.ts for why that is done there and not here. + * + * Left out when the dashboard is not linked to an app, and reported as + * unavailable rather than left out when it is linked and does not answer, so + * a broken link shows on the dashboard instead of looking like a quiet week. + */ +export async function fetchAccountStats( + base: string | undefined, + token: string | undefined, + range: StatsRange, + fetchImplementation: typeof fetch = fetch, +): Promise { + const origin = base?.trim(); + const secret = token?.trim(); + if (!origin || !secret) return null; + try { + const response = await fetchImplementation( + `${origin.replace(/\/+$/, "")}/api/stats/accounts?range=${range}`, + { + headers: { Authorization: `Bearer ${secret}`, Accept: "application/json" }, + signal: AbortSignal.timeout(4_000), + }, + ); + if (!response.ok) return { error: `accounts app answered ${response.status}` }; + const body = await response.json(); + if (!isAccountStats(body)) return { error: "accounts app answered in an unexpected shape" }; + return completeAccountStats(body); + } catch { + return { error: "accounts app did not answer" }; + } +} + +/* + * The counts the dashboard cannot do without. Everything else it asks for is + * filled in below when an older app has not got it yet, so a Worker deployed + * ahead of the app shows the figures it can rather than an error. + */ +function isAccountStats(value: unknown): value is Record { + if (typeof value !== "object" || value === null) return false; + const candidate = value as Record; + return typeof candidate.total === "number" && + typeof candidate.newInRange === "number" && + typeof candidate.activeInRange === "number" && + Array.isArray(candidate.newByDay) && + Array.isArray(candidate.cohorts); +} + +export function completeAccountStats(body: Record): StatsAccountStats { + return { + total: body.total as number, + newInRange: body.newInRange as number, + activeInRange: body.activeInRange as number, + returningInRange: typeof body.returningInRange === "number" ? body.returningInRange : 0, + previous: isAccountPeriod(body.previous) ? body.previous : null, + newByDay: dayCounts(body.newByDay), + activeByDay: dayCounts(body.activeByDay), + engagement: Array.isArray(body.engagement) + ? body.engagement.filter((bucket): bucket is { label: string; value: number } => + isObject(bucket) && typeof bucket.label === "string" && typeof bucket.value === "number") + : [], + engagementBase: typeof body.engagementBase === "number" ? body.engagementBase : 0, + activeSince: typeof body.activeSince === "number" ? body.activeSince : null, + excluded: typeof body.excluded === "number" ? body.excluded : 0, + cohorts: (body.cohorts as StatsAccountStats["cohorts"]).filter((cohort) => + isObject(cohort) && typeof cohort.weekStart === "number" && typeof cohort.size === "number" && + Array.isArray(cohort.active)), + /* An older app answers without events; the dashboard then shows none rather than nothing. */ + events: isCountRecord(body.events) ? body.events : {}, + }; +} + +function dayCounts(value: unknown): { day: number; count: number }[] { + if (!Array.isArray(value)) return []; + return value.filter((point): point is { day: number; count: number } => + isObject(point) && typeof point.day === "number" && typeof point.count === "number"); +} + +function isAccountPeriod(value: unknown): value is StatsAccountPeriod { + return isObject(value) && typeof value.total === "number" && typeof value.newAccounts === "number" && + typeof value.active === "number" && typeof value.returning === "number"; +} + +function isCountRecord(value: unknown): value is Record { + return isObject(value) && !Array.isArray(value) && + Object.values(value).every((count) => typeof count === "number"); +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/worker/analytics.ts b/worker/analytics.ts index 08e7617..7ddca32 100644 --- a/worker/analytics.ts +++ b/worker/analytics.ts @@ -303,6 +303,8 @@ export function documentTarget(pathname: string, status: number, currentVersion: export function isDocumentNavigation(request: Request): boolean { if (request.method !== "GET") return false; + /* A browser fetching a page it guesses will be wanted has not shown it to anyone. */ + if (request.headers.get("Sec-Purpose")?.includes("prefetch") || request.headers.get("Purpose") === "prefetch") return false; if (request.headers.get("Sec-Fetch-Dest") === "document") return true; return request.headers.get("Accept")?.toLowerCase().includes("text/html") ?? false; } @@ -312,11 +314,26 @@ export function binaryDownloadTarget(pathname: string): string | null { return match ? `${match[1]}-${match[2]}` : null; } +/** + * Crawlers, and monitors that say so, the site's own download checks among + * them: requests to count, never people. A monitor that says nothing looks + * like whatever library it used, which the tool pattern catches next. + */ +const CRAWLER_AGENT = /bot|crawler|spider|slurp|headless|preview|monitor|uptime|pingdom|statuscake|checkly|site24x7|synthetic|shell\.online-(?:downloads|install)-check/; + +/** + * Tools a person or a script runs: the CLI, curl and wget, PowerShell's web + * cmdlets, and the HTTP libraries scripts are written with. Not browsers, + * whatever else the agent string says; the default used to be "desktop", + * which made every Node script a person. + */ +const TOOL_AGENT = /^shell\/|curl|wget|powershell|\bnode\b|node-fetch|undici|go-http-client|python|\bjava\b|\bjava\/|okhttp|axios|libwww|httpie/; + export function classifyDevice(userAgent: string, mobileHint: string | null = null): DeviceClass { const normalized = userAgent.toLowerCase(); if (!normalized) return "unknown"; - if (/bot|crawler|spider|slurp|headless|preview/.test(normalized)) return "bot"; - if (/^shell\//.test(normalized) || /curl|wget/.test(normalized)) return "cli"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "cli"; if (mobileHint === "?1" || /iphone|ipod|android.+mobile|mobile.+android/.test(normalized)) { return "mobile"; } @@ -327,9 +344,11 @@ export function classifyDevice(userAgent: string, mobileHint: string | null = nu export function classifyClient(userAgent: string): string { const shellVersion = userAgent.match(/\bshell\/(\d{1,3}\.\d{1,3}\.\d{1,3})\b/i)?.[1]; if (shellVersion) return `shell/${shellVersion}`; - if (/curl/i.test(userAgent)) return "curl"; - if (/wget/i.test(userAgent)) return "wget"; - if (/bot|crawler|spider|slurp|headless|preview/i.test(userAgent)) return "bot"; + const normalized = userAgent.toLowerCase(); + if (/curl/.test(normalized)) return "curl"; + if (/wget/.test(normalized)) return "wget"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "tool"; return userAgent ? "web" : "unknown"; } diff --git a/worker/index.ts b/worker/index.ts index f0de980..feb0e5f 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -16,7 +16,8 @@ import { type AnalyticsEvent, type DeviceClass, } from "./analytics"; -import { isStatsRange, type StatsAccountStats, type StatsAccounts, type StatsRange } from "../shared/stats"; +import { isStatsRange, type StatsRange } from "../shared/stats"; +import { fetchAccountStats } from "./account-stats"; import { RELEASE_VERSION } from "../shared/release"; import { downloadAssetIsSpaFallback } from "../shared/download-assets"; import { viewerFrameAction } from "../shared/session-access"; @@ -508,7 +509,7 @@ async function handleStatsRequest(request: Request, env: Env, url: URL): Promise const range: StatsRange = isStatsRange(requestedRange) ? requestedRange : "7d"; const [snapshotResponse, accounts] = await Promise.all([ fetchStatsSnapshot(env.STATS, range, hasVisitorSalt(env.STATS_VISITOR_SALT)), - fetchAccountStats(env, range), + fetchAccountStats(env.APP_STATS_URL, env.APP_STATS_TOKEN, range), ]); if (!snapshotResponse.ok) return secureStatsResponse(snapshotResponse); const snapshot = await snapshotResponse.json>(); @@ -518,47 +519,6 @@ async function handleStatsRequest(request: Request, env: Env, url: URL): Promise return secureStatsResponse(json({ error: "not found" }, 404)); } -/* - * The accounts app keeps the only exact count of people: accounts. It answers - * aggregates -- how many, how many new, how many back each week -- to a bearer - * token, and nothing per person. Left out when it is not linked, and reported - * as unavailable rather than left out when it is linked and does not answer, - * so a broken link is visible on the dashboard rather than silent. - */ -async function fetchAccountStats(env: Env, range: StatsRange): Promise { - const base = env.APP_STATS_URL?.trim(); - const token = env.APP_STATS_TOKEN?.trim(); - if (!base || !token) return null; - try { - const response = await fetch(`${base.replace(/\/+$/, "")}/api/stats/accounts?range=${range}`, { - headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, - signal: AbortSignal.timeout(4_000), - }); - if (!response.ok) return { error: `accounts app answered ${response.status}` }; - const body = await response.json(); - if (!isAccountStats(body)) return { error: "accounts app answered in an unexpected shape" }; - /* An older app answers without events; the dashboard then shows none rather than nothing. */ - return { ...body, events: isCountRecord(body.events) ? body.events : {} }; - } catch { - return { error: "accounts app did not answer" }; - } -} - -function isAccountStats(value: unknown): value is Omit & { events?: unknown } { - if (typeof value !== "object" || value === null) return false; - const candidate = value as Record; - return typeof candidate.total === "number" && - typeof candidate.newInRange === "number" && - typeof candidate.activeInRange === "number" && - Array.isArray(candidate.newByDay) && - Array.isArray(candidate.cohorts); -} - -function isCountRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) && - Object.values(value as Record).every((count) => typeof count === "number"); -} - function statsPassword(env: Env): string | null { return typeof env.STATS_PASSWORD === "string" && env.STATS_PASSWORD.length >= 12 ? env.STATS_PASSWORD diff --git a/worker/stats-database.ts b/worker/stats-database.ts index 2611e80..669db83 100644 --- a/worker/stats-database.ts +++ b/worker/stats-database.ts @@ -21,6 +21,7 @@ import { type PreviousPeriodRows, type RetentionRow, type StatsSnapshotRows, + type SurfaceSinceRow, type UniqueDayRow, type UniqueSummaryRow, } from "../shared/stats-snapshot"; @@ -136,6 +137,35 @@ export function initializeStatsSchema(sql: StatsSql): void { ) `); sql.exec("CREATE INDEX IF NOT EXISTS visitors_last_day ON visitors(last_day)"); + sql.exec(` + CREATE TABLE IF NOT EXISTS dashboard_marks ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + migrateStatsData(sql); +} + +/** + * The last day machines were keyed by address and user agent; since the day + * after, by address alone. A machine row from before cannot match the same + * machine seen after, so it would sit in the cohorts as a ghost that never + * came back. + */ +export const MACHINE_KEY_DAY = Date.UTC(2026, 8, 15); + +/** + * One-time repairs to what is stored, each done once and marked as done. + * The first drops the machine rows keyed the old way; a machine seen on the + * cut-over day itself is lost with them, a few hours of history against 120 + * days of ghosts. + */ +export function migrateStatsData(sql: StatsSql): void { + const done = sql.exec<{ value: number }>("SELECT value FROM dashboard_marks WHERE name = 'machine_key'").toArray(); + if (done.length > 0) return; + sql.exec("DELETE FROM visitor_days WHERE surface IN ('cli', 'install') AND day <= ?", MACHINE_KEY_DAY); + sql.exec("DELETE FROM visitors WHERE surface IN ('cli', 'install') AND last_day <= ?", MACHINE_KEY_DAY); + sql.exec("INSERT INTO dashboard_marks (name, value) VALUES ('machine_key', 1)", ); } /** One event into its hour bucket, and its person into the day's visitors when it has one. */ @@ -228,6 +258,9 @@ export function collectStatsRows( const collectingSince = sql.exec("SELECT MIN(bucket) AS minimum FROM metric_hourly").one().minimum; /* Taken after the purge, so it is the earliest day people can still be counted from. */ const uniquesSince = sql.exec("SELECT MIN(day) AS minimum FROM visitor_days").one().minimum; + const uniquesSinceBySurface = sql.exec( + "SELECT surface, MIN(day) AS minimum FROM visitor_days GROUP BY surface ORDER BY surface", + ).toArray(); const rangeStart = statsRangeStart(range, now, collectingSince); /* Buckets are hours and events at most ten minutes ahead of this clock, so nothing sits past tomorrow. */ const rangeEnd = now + DAY_MS; @@ -255,13 +288,15 @@ export function collectStatsRows( ).toArray(), }; } + /* The charts are about people, so crawlers stay out of them; the ledger keeps every request. */ const trend = sql.exec( `SELECT bucket, event, SUM(count) AS count FROM metric_hourly WHERE bucket >= ? - AND event IN ('session_created', 'share_opened', 'collaboration_started', 'page_view') + AND event IN ('session_created', 'session_started', 'share_opened', 'collaboration_started', 'page_view', 'binary_download') + AND device != 'bot' GROUP BY bucket, event - ORDER BY bucket`, + ORDER BY bucket, event`, rangeStart, ).toArray(); const live = sql.exec( @@ -346,6 +381,7 @@ export function collectStatsRows( installConversion, uniquesConfigured, uniquesSince, + uniquesSinceBySurface, }, }; } diff --git a/worker/stats-store.test.ts b/worker/stats-store.test.ts index 3c84b6a..77abd21 100644 --- a/worker/stats-store.test.ts +++ b/worker/stats-store.test.ts @@ -40,6 +40,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "all", now, collectingSince); expect(snapshot.metrics).toMatchObject({ @@ -73,6 +74,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "24h", now, now - 24 * 60 * 60 * 1_000); expect(snapshot.metrics.activeSessions).toBe(0); @@ -151,6 +153,12 @@ describe("people and the funnel", () => { installConversion: { installers: 12, matured: 9, started: 4 }, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + /* Machines were re-keyed after visitors were first counted, so their start is later. */ + uniquesSinceBySurface: [ + { surface: "site", minimum: dayStart(now - 20 * DAY_MS) }, + { surface: "cli", minimum: dayStart(now - 2 * DAY_MS) }, + { surface: "install", minimum: dayStart(now - 2 * DAY_MS) }, + ], }; it("keeps docs, unknown paths and 404s apart and counts people beside events", () => { @@ -164,8 +172,8 @@ describe("people and the funnel", () => { ctaClicks: 12, binaryDownloads: 4, }); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50 }); - expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50, since: dayStart(now - 20 * DAY_MS) }); + expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7, since: dayStart(now - 2 * DAY_MS) }); expect(snapshot.uniques.daily.map((day) => [day.site, day.cli])).toEqual([[120, 4], [140, 0]]); expect(snapshot.rates.signup).toBeCloseTo(12 / 969); expect(snapshot.breakdowns.pages.map((page) => page.label)).toEqual(["landing", "unknown_path", "docs_app", "docs", "not_found"]); @@ -310,6 +318,22 @@ describe("people and the funnel", () => { expect(buildStatsSnapshot({ ...rows, previous: null }, "all", now, now - 30 * DAY_MS).previous).toBeNull(); }); + it("draws installs and started sessions as their own lines", () => { + const hour = 60 * 60 * 1_000; + const bucket = Math.floor((now - DAY_MS) / hour) * hour; + const snapshot = buildStatsSnapshot({ + ...rows, + trend: [ + { bucket, event: "session_created", count: 4 }, + { bucket, event: "session_started", count: 3 }, + { bucket, event: "binary_download", count: 2 }, + { bucket, event: "page_view", count: 9 }, + ], + }, "7d", now, rangeStart); + const point = snapshot.trend.find((candidate) => candidate.at === Math.floor(bucket / (6 * hour)) * 6 * hour); + expect(point).toMatchObject({ sessions: 4, started: 3, installs: 2, pageViews: 9, shares: 0, collaborations: 0 }); + }); + it("follows machines from the installer to a first session", () => { const snapshot = buildStatsSnapshot(rows, "7d", now, rangeStart); expect(snapshot.installConversion).toEqual({ installers: 12, matured: 9, started: 4 }); @@ -323,7 +347,7 @@ describe("people and the funnel", () => { expect(snapshot.uniques.since).toBeNull(); expect(snapshot.installConversion).toBeNull(); expect(snapshot.funnel[0].unique).toBeNull(); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0, since: null }); }); /* @@ -342,6 +366,16 @@ describe("people and the funnel", () => { expect(partial.uniques.since).toBe(yesterday); expect(peopleCountedSince(partial.uniques, partial.rangeStart)).toBe(yesterday); + /* Each surface starts on its own day: visitors cover the week, machines do not. */ + expect(covered.uniques.surfaces.site.since).toBe(dayStart(now - 20 * DAY_MS)); + expect(covered.uniques.surfaces.cli.since).toBe(dayStart(now - 2 * DAY_MS)); + expect(covered.uniques.surfaces.viewer.since).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "site")).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "cli")).toBe(dayStart(now - 2 * DAY_MS)); + expect(peopleCountedSince(covered.uniques, rangeStart, "viewer")).toBeNull(); + const unconfigured = buildStatsSnapshot({ ...rows, uniquesConfigured: false }, "7d", now, rangeStart); + expect(unconfigured.uniques.surfaces.cli.since).toBeNull(); + /* People are kept by day, so a range that starts inside their first day is covered. */ expect(peopleCountedSince({ configured: true, since: dayStart(rangeStart) }, rangeStart + 60_000)).toBeNull(); expect(peopleCountedSince({ configured: false, since: yesterday }, now - 30 * DAY_MS)).toBeNull();