Fix the statistics funnel's measurement, lay the dashboard out as a story, and count what it could not see

CHANGELOG.md

@@ -37,6 +37,36 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve
   app can add exact account counts and sign-up retention when the two are
   linked.
 - Clicks on the landing page's Sign up free and Web app links are counted.
+- The funnel gains "Copied an install command" from the landing page's copy
+  buttons, and lists sessions created but never connected beside "Started a
+  session". A named `utm_source` or `ref` on a landing link counts as the
+  source when the browser hid the referrer, from a fixed list of names;
+  visits from the web app are their own source.
+- A session's first open records whether typing is allowed and how long the
+  link waited; the first keystroke, how long after the open it came; a
+  viewer's disconnect, how long they stayed. Browsers turned away by a full,
+  expired or unknown session are counted by reason, and a viewer refused
+  input in a read-only session once. The dashboard shows typed rate by
+  device, who was turned away, and the typed share over sessions that allow
+  typing.
+- Machines running the installer or the CLI are keyed by address alone, so
+  the dashboard can say how many machines that installed at least a week ago
+  started a session within seven days. The privacy policy says so.
+- Both install scripts send one word at their end, the outcome, and the
+  binary name, so a platform that keeps failing gets noticed; nothing else
+  goes with it, and `SHELL_ONLINE_INSTALL_REPORT=0` skips it. The dashboard
+  shows how installs ended by the scripts' own account.
+- The accounts app counts what accounts do, by day and by kind and nothing
+  else: machines linked, sessions registered, commands sent, vaults created,
+  invites sent and accepted, feedback sent. The dashboard shows them under
+  Accounts as things done, not as distinct accounts.
+- The statistics dashboard reads top to bottom as a story: what is live now,
+  six headline figures each with its change against the period before, the
+  funnel, then traffic, sessions, retention and accounts, each section opening
+  with the finding in a sentence. Every figure about people leaves crawlers
+  out and says so beside the step: page views by people, the installer run by
+  curl or wget rather than read or crawled, installs completed on a person's
+  machine. The raw totals stay in the ledger.
 
 ### Fixed
 
@@ -47,6 +77,17 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve
   as "Not found". Every documentation route, current or versioned, now has its
   own page-view target; unknown paths the site answers with the landing page
   are counted apart from real 404s, and real 404s are counted at all.
+- The statistics dashboard says since when people have been counted. Event
+  counts run from the first event and people from the day the visitor salt was
+  set, so a 30-day range could show thirty days of views beside one day of
+  people. A people figure over fewer days than the count beside it now names
+  that day, in the funnel, the headline tiles and the footer.
+- Crawlers that identify themselves are no longer counted as people. The
+  funnel said they were not, and they were.
+- Deploying to production refuses a Wrangler config that serves a documentation
+  page from the assets binding instead of the Worker, since such a page is
+  never counted. Production served the web app, CLI, Refstream and platforms
+  pages that way.
 
 ### Changed
 

README.md

@@ -28,6 +28,10 @@ Windows PowerShell:
 irm https://shell.online/install.ps1 | iex
 ```
 
+At the end, and on failure, the installer sends one word back, its outcome,
+so a platform that keeps failing gets noticed and fixed. Nothing else goes
+with it. Set `SHELL_ONLINE_INSTALL_REPORT=0` to skip that.
+
 Homebrew (the tap lives in this repository):
 
 ```sh

app/server/app.test.ts

@@ -236,6 +236,8 @@ describe("session registry", () => {
     const tokens = await login();
     const created = await call("POST", "/api/sessions", { auth: tokens.access_token, body: session });
     expect(created.status).toBe(201);
+    /* Counted for the dashboard: the login linked a machine, the registration a session. */
+    expect(await store.appEvents(0)).toEqual([{ event: "machine_linked", count: 1 }, { event: "session_registered", count: 1 }]);
 
     const listed = await call("GET", "/api/sessions", { auth: await idToken() });
     expect(listed.status).toBe(200);
@@ -556,6 +558,7 @@ describe("driving a machine from the browser", () => {
       body: { device_id: deviceId, kind: "start", command: "top" },
     });
     expect(queued.status).toBe(202);
+    expect((await store.appEvents(0)).find((entry) => entry.event === "command_sent")?.count).toBe(1);
 
     const claimed = await call("GET", "/api/agent/commands", { auth: tokens.access_token });
     expect(claimed.body.commands).toHaveLength(1);
@@ -1041,6 +1044,8 @@ describe("organizations", () => {
     expect(joined.body.joined).toBe(true);
     expect(joined.body.you.role).toBe("member");
     expect(joined.body.members).toHaveLength(2);
+    /* Both halves counted for the dashboard, with nothing about who. */
+    expect(await store.appEvents(0)).toEqual([{ event: "invite_accepted", count: 1 }, { event: "invite_created", count: 1 }]);
   });
 
   it("shows colleagues each other's sessions", async () => {
@@ -2031,6 +2036,7 @@ describe("session vault", () => {
     const { body } = await vaultBody();
     const created = await call("POST", "/api/vault", { auth: await idToken(), body });
     expect(created.status).toBe(201);
+    expect((await store.appEvents(0)).find((entry) => entry.event === "vault_created")?.count).toBe(1);
     const fetched = await call("GET", "/api/vault", { auth: await idToken() });
     expect(fetched.body.vault).toMatchObject({
       publicKey: body.public_key,
@@ -2542,6 +2548,12 @@ describe("feedback", () => {
     const posted = await call("POST", "/api/feedback", { auth: await idToken(), body: message });
     expect(posted.status).toBe(201);
     expect(posted.body.feedback.id).toMatch(/^fbk_/);
+    /* Counted for the dashboard as a thing done, with nothing about who did it. */
+    expect(await store.appEvents(0)).toEqual([{ event: "feedback_sent", count: 1 }]);
+    /* Counting must never change an answer: a store that cannot count still answers 201. */
+    vi.spyOn(store, "recordAppEvent").mockRejectedValueOnce(new Error("db down"));
+    const again = await call("POST", "/api/feedback", { auth: await idToken(), body: message });
+    expect(again.status).toBe(201);
     const [kept] = await store.feedback();
     expect(kept).toMatchObject({
       uid: "uid-1",
@@ -2599,9 +2611,18 @@ describe("account figures for the statistics dashboard", () => {
     expect((await call("GET", "/api/stats/accounts?range=7d", { auth: "stats-token-with-thirty-two-characters?" })).status).toBe(401);
     expect((await call("GET", "/api/stats/accounts?range=7d", { auth: await idToken() })).status).toBe(401);
 
+    await store.recordAppEvent("machine_linked");
+    await store.recordAppEvent("machine_linked");
+    await store.recordAppEvent("command_sent");
     const answer = await call("GET", "/api/stats/accounts?range=7d", { auth: TOKEN });
     expect(answer.status).toBe(200);
-    expect(answer.body).toMatchObject({ total: 1, newInRange: 1, activeInRange: 1 });
+    expect(answer.body).toMatchObject({ total: 1, newInRange: 1, activeInRange: 1, events: { machine_linked: 2, command_sent: 1 } });
+    /* Only the range's days count; the all-time range counts every day kept. */
+    await store.recordAppEvent("vault_created", Date.now() - 40 * 24 * 60 * 60_000);
+    const week = await call("GET", "/api/stats/accounts?range=7d", { auth: TOKEN });
+    expect(week.body.events).toEqual({ machine_linked: 2, command_sent: 1 });
+    const all = await call("GET", "/api/stats/accounts?range=all", { auth: TOKEN });
+    expect(all.body.events).toEqual({ machine_linked: 2, command_sent: 1, vault_created: 1 });
     expect(answer.body.newByDay).toHaveLength(90);
     expect(answer.body.cohorts).toHaveLength(1);
     expect(answer.body.cohorts[0].size).toBe(1);

app/server/app.ts

@@ -1,7 +1,9 @@
 import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
 import type { Store } from "./lib/store";
 import type { Invite, Membership } from "./lib/orgs";
-import type { AuditEvent, SessionRecord } from "./lib/types";
+import type { AuditEvent, SessionRecord,
+  AppEvent,
+} from "./lib/types";
 import type { VerifyResult } from "./lib/firebase-token";
 import type { SessionLiveness, SessionLivenessSource } from "./lib/session-liveness";
 import { exchangeCode, issueCode } from "./lib/codes";
@@ -45,7 +47,7 @@ import { recordAudit, assignSession, auditCsv, SEALED_KINDS } from "./routes/aud
 import { addComment, inbox, notifyAssigned, notifySessionStarted } from "./routes/social";
 import { deleteAccount } from "./routes/account";
 import { submitFeedback } from "./routes/feedback";
-import { accountStats, isStatsRange } from "./routes/stats";
+import { accountStats, dayStart, isStatsRange, rangeStart } from "./routes/stats";
 import { timingSafeEqual } from "node:crypto";
 import { callerAddress, rateLimiter } from "./lib/rate-limit";
 import { logMailer, type Mailer } from "./lib/mail";
@@ -378,6 +380,15 @@ export function createApp(options: AppOptions) {
     return membership;
   }
 
+  /*
+   * One count per thing done, never who did it, for the statistics
+   * dashboard. Counting must never change an answer, so a store that cannot
+   * count is nobody's problem here.
+   */
+  const track = (event: AppEvent): void => {
+    void store.recordAppEvent(event).catch(() => undefined);
+  };
+
   /* The CLI authenticates with an opaque access token issued by this service. */
   async function requireCli(request: IncomingMessage) {
     const check = await checkAccessToken(store, bearer(request));
@@ -483,6 +494,7 @@ export function createApp(options: AppOptions) {
           label: String(body.label ?? "shell cli").slice(0, 80),
           machineId,
         });
+        track("machine_linked");
         return send(response, 200, {
           access_token: tokens.accessToken,
           refresh_token: tokens.refreshToken,
@@ -536,6 +548,7 @@ export function createApp(options: AppOptions) {
           return send(response, 400, { error: "invalid browser public key" });
         }
         if (publicKey) await store.setMemberKey(identity.uid, publicKey);
+        if (resolved.joined && invite) track("invite_accepted");
         const described = await describeOrganization(store, resolved.membership);
         return send(response, described.status, {
           ...(described.body as Record<string, unknown>),
@@ -576,6 +589,7 @@ export function createApp(options: AppOptions) {
           const invite = (result.body as { invite?: Invite }).invite;
           if (invite) await notifyInvited(store, mailer, webOrigin, membership, invite, log);
         }
+        if (result.status < 300) track("invite_created");
         return send(response, result.status, result.body);
       }
 
@@ -916,6 +930,7 @@ export function createApp(options: AppOptions) {
           });
           if (!created) return send(response, 409, { error: "this account already has a vault" });
           const stored = await store.accountKey(identity.uid);
+          track("vault_created");
           return send(response, 201, { vault: stored ? vaultForApi(stored) : null });
         }
 
@@ -1068,6 +1083,7 @@ export function createApp(options: AppOptions) {
             result.session.name || result.session.command,
           );
         }
+        track("session_registered");
         return send(response, 201, { session: sessionForApi(result.session) });
       }
 
@@ -1282,6 +1298,7 @@ export function createApp(options: AppOptions) {
             createdAt: Date.now(),
           };
           await store.putCommand(queued);
+          track("command_sent");
           return send(response, 202, { command: queued });
         }
 
@@ -1468,6 +1485,7 @@ export function createApp(options: AppOptions) {
           log,
         );
         if (!result.ok) return send(response, result.status, { error: result.error });
+        track("feedback_sent");
         return send(response, 201, { feedback: { id: result.value.id, at: result.value.at } });
       }
 
@@ -1480,8 +1498,15 @@ export function createApp(options: AppOptions) {
         const matches = presented.length === expected.length &&
           timingSafeEqual(Buffer.from(presented), Buffer.from(expected));
         if (!matches) return send(response, 401, { error: "sign in first" });
-        const range = url.searchParams.get("range");
-        return send(response, 200, accountStats(await store.accountActivity(), isStatsRange(range) ? range : "7d"));
+        const requested = url.searchParams.get("range");
+        const range = isStatsRange(requested) ? requested : "7d";
+        const now = Date.now();
+        return send(response, 200, accountStats(
+          await store.accountActivity(),
+          range,
+          now,
+          await store.appEvents(dayStart(rangeStart(range, now))),
+        ));
       }
 
       /* ---- Inbox ---- */

app/server/lib/migrations/013_app_events.sql

@@ -0,0 +1,13 @@
+-- What accounts did in the app, counted by day and by kind and nothing else:
+-- no account, no address, no session. The statistics dashboard reads totals
+-- over a range from it, so it can say how many machines were linked or
+-- commands sent next to how many accounts there are. Rows are dropped after
+-- the same 400 days as activity days.
+CREATE TABLE IF NOT EXISTS app_events (
+  event TEXT    NOT NULL,
+  day   BIGINT  NOT NULL,
+  count INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (event, day)
+);
+
+CREATE INDEX IF NOT EXISTS app_events_day ON app_events (day);

app/server/lib/store-conformance.test.ts

@@ -149,6 +149,7 @@ function feedback(overrides: Partial<Feedback> = {}): Feedback {
 const TABLES = [
   "feedback",
   "account_activity",
+  "app_events",
   "deleted_accounts",
   "account_keys",
   "session_key_shares",
@@ -943,6 +944,29 @@ for (const implementation of implementations) {
       });
     });
 
+    describe("app events", () => {
+      const day = 24 * 60 * 60_000;
+      const noon = 10 * day + 12 * 60 * 60_000;
+
+      it("counts by kind and day, sums from a day on, and says nothing about who", async () => {
+        await store.recordAppEvent("machine_linked", noon);
+        await store.recordAppEvent("machine_linked", noon + 60_000);
+        await store.recordAppEvent("command_sent", noon + day);
+        expect(await store.appEvents(0)).toEqual([
+          { event: "command_sent", count: 1 },
+          { event: "machine_linked", count: 2 },
+        ]);
+        expect(await store.appEvents(11 * day)).toEqual([{ event: "command_sent", count: 1 }]);
+        expect(await store.appEvents(12 * day)).toEqual([]);
+      });
+
+      it("forgets counts older than the memory window when purging", async () => {
+        await store.recordAppEvent("vault_created", noon);
+        await store.purgeExpired(noon + 401 * day);
+        expect(await store.appEvents(0)).toEqual([]);
+      });
+    });
+
     describe("claiming an organization on first sight", () => {
       /*
        * Signing in fires several requests at once. On a new account none of

app/server/lib/store-memory.ts

@@ -14,6 +14,8 @@ import {
 } from "./store";
 import type {
   AccountActivity,
+  AppEvent,
+  AppEventCount,
   AccountKey,
   AgentCommand,
   AuditEvent,
@@ -73,6 +75,7 @@ interface Shape {
   accountKeys: AccountKey[];
   deletedAccounts: { uid: string; deletedAt: number }[];
   accountActivity: { uid: string; day: number }[];
+  appEvents: { event: AppEvent; day: number; count: number }[];
   teamKeys: TeamKey[];
   teamKeyShares: TeamKeyShare[];
 }
@@ -81,7 +84,7 @@ const EMPTY: Shape = {
   codes: [], tokens: [], sessions: [], commands: [],
   organizations: [], memberships: [], invites: [], audit: [],
   comments: [], notifications: [], feedback: [], accountKeys: [], deletedAccounts: [],
-  accountActivity: [], teamKeys: [], teamKeyShares: [],
+  accountActivity: [], appEvents: [], teamKeys: [], teamKeyShares: [],
 };
 
 /**
@@ -148,6 +151,7 @@ export class MemoryStore implements Store {
         accountKeys: parsed.accountKeys ?? [],
         deletedAccounts: parsed.deletedAccounts ?? [],
         accountActivity: parsed.accountActivity ?? [],
+        appEvents: parsed.appEvents ?? [],
         teamKeys: parsed.teamKeys ?? [],
         teamKeyShares: parsed.teamKeyShares ?? [],
       };
@@ -823,6 +827,9 @@ export class MemoryStore implements Store {
     this.data.accountActivity = this.data.accountActivity.filter(
       (entry) => entry.day >= now - ACCOUNT_ACTIVITY_MEMORY_MS,
     );
+    this.data.appEvents = this.data.appEvents.filter(
+      (entry) => entry.day >= now - ACCOUNT_ACTIVITY_MEMORY_MS,
+    );
     const before = this.data.codes.length;
     this.data.codes = this.data.codes.filter((entry) => entry.expiresAt > now);
     /* Finished commands are only kept long enough to be reported back. */
@@ -893,6 +900,26 @@ export class MemoryStore implements Store {
     }));
   }
 
+  /* ---- App events ---- */
+
+  async recordAppEvent(event: AppEvent, now = Date.now()): Promise<void> {
+    const day = Math.floor(now / DAY_MS) * DAY_MS;
+    const row = this.data.appEvents.find((entry) => entry.event === event && entry.day === day);
+    if (row) row.count += 1;
+    else this.data.appEvents.push({ event, day, count: 1 });
+    this.flush();
+  }
+
+  async appEvents(sinceDay: number): Promise<AppEventCount[]> {
+    const totals = new Map<AppEvent, number>();
+    for (const entry of this.data.appEvents) {
+      if (entry.day >= sinceDay) totals.set(entry.event, (totals.get(entry.event) ?? 0) + entry.count);
+    }
+    return [...totals.entries()]
+      .map(([event, count]) => ({ event, count }))
+      .sort((left, right) => left.event.localeCompare(right.event));
+  }
+
   async tokensForImport(): Promise<CliToken[]> {
     return this.data.tokens;
   }