From 28c75cfaf24da600bed1be42e359861b219fa165 Mon Sep 17 00:00:00 2001 From: Philip Stayetski Date: Wed, 16 Sep 2026 14:07:04 -0700 Subject: [PATCH] Keep the site's own monitor out of the statistics, and draw each tile its own line The scheduled download check installed on three fresh GitHub runners every half hour, and the dashboard counted it: six installs, five installer fetches, three "ok" outcomes and three or four new machines per run, which is a day of ninety installs and fifty-six machines that never came back. Both install scripts now take SHELL_ONLINE_INSTALL_CHECK=1, which puts a check user agent on every request and reports nothing; the workflow sets it, and the Worker counts that agent, and monitors in general, as crawlers. HTTP libraries and PowerShell are tools rather than desktop browsers, so a Node script is not a person and a Windows install is a run. The installs tile drew the sessions line, since the trend had no installs series; it now has installs and started sessions, crawlers left out of every line. "New" is measured per surface from the day its people were first counted, and until a whole range has passed since then the split is replaced by the day it becomes meaningful. Machine rows keyed the old way are dropped once. Prefetched pages are not views, and the 24h range says people are counted by UTC day. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/downloads.yml | 8 +++-- CHANGELOG.md | 22 ++++++++++++++ public/install | 19 ++++++++++-- public/install.ps1 | 12 ++++++-- scripts/test-install.sh | 15 ++++++++++ shared/stats-snapshot.ts | 30 +++++++++++++++---- shared/stats.ts | 13 +++++++++ tests/analytics.test.ts | 33 +++++++++++++++++++++ tests/stats-copy.test.ts | 1 + tests/stats-database.test.ts | 51 ++++++++++++++++++++++++++++++++ web/stats.ts | 52 +++++++++++++++++++++++---------- worker/analytics.ts | 29 ++++++++++++++---- worker/stats-database.ts | 40 +++++++++++++++++++++++-- worker/stats-store.test.ts | 40 +++++++++++++++++++++++-- 14 files changed, 328 insertions(+), 37 deletions(-) diff --git a/.github/workflows/downloads.yml b/.github/workflows/downloads.yml index a919ec6..58af418 100644 --- a/.github/workflows/downloads.yml +++ b/.github/workflows/downloads.yml @@ -58,21 +58,25 @@ jobs: script: powershell runs-on: ${{ matrix.os }} timeout-minutes: 10 + # Forty-eight installs a day from fresh runner addresses would read as + # people on the statistics dashboard. The check agent on every request, + # the scripts' own included, tells the site this is a monitor. env: ORIGIN: ${{ inputs.origin || 'https://shell.online' }} + SHELL_ONLINE_INSTALL_CHECK: "1" steps: - name: Install with the documented command if: matrix.script == 'posix' run: | export SHELL_ONLINE_INSTALL_DIR="$RUNNER_TEMP/shell-online" - curl -fsSL "$ORIGIN/install" | sh + curl -fsSL -A shell.online-install-check "$ORIGIN/install" | sh "$SHELL_ONLINE_INSTALL_DIR/shell" --version - name: Install with the documented command if: matrix.script == 'powershell' shell: pwsh run: | $env:SHELL_ONLINE_INSTALL_DIR = Join-Path $env:RUNNER_TEMP "shell-online" - Invoke-RestMethod "$env:ORIGIN/install.ps1" | Invoke-Expression + Invoke-RestMethod -UserAgent shell.online-install-check "$env:ORIGIN/install.ps1" | Invoke-Expression & (Join-Path $env:SHELL_ONLINE_INSTALL_DIR "shell.exe") --version report: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad69e4..04f6726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,28 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve ## Unreleased +### Fixed + +- The scheduled download check installed on three fresh GitHub runners every + half hour and was counted as installs, installer runs, installer outcomes + and new machines, which is where a dashboard day of ninety installs and + fifty-six new machines came from. Both install scripts now take + `SHELL_ONLINE_INSTALL_CHECK=1`, which puts a check user agent on every + request and reports nothing; the workflow sets it, and the site counts + that agent, and monitors in general, as crawlers. +- HTTP libraries and PowerShell's web cmdlets were classified as desktop + browsers, so a Node script or a Windows install read as a person reading + the installer. They are tools now, and a Windows install counts as a run. +- The installs tile drew the sessions line. The trend now carries installs + and started sessions as their own series, crawlers left out of every line. +- "New" people are measured from the day a surface's people were first + counted, per surface, and while a whole range has not yet passed since + that day the split is replaced by the day it becomes meaningful. Machine + rows keyed the old way are dropped once, so they do not sit in the cohorts + as machines that never came back. +- Pages a browser fetched ahead of time (prefetch, prerender) are not views. +- The 24h range says that people are counted by UTC day, so over two days. + ## [0.16.0] — 2026-09-15 ### Changed diff --git a/public/install b/public/install index 0f94ffb..6ff8378 100755 --- a/public/install +++ b/public/install @@ -6,6 +6,17 @@ set -eu # checksum_mismatch, ...) and the binary name, nothing else, so that a # platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +curl_agent= +wget_agent= +if [ "${SHELL_ONLINE_INSTALL_CHECK:-0}" = 1 ]; then + curl_agent="-A shell.online-install-check" + wget_agent="-U shell.online-install-check" +fi fail() { printf 'shell.online: %s\n' "$1" >&2 @@ -15,6 +26,7 @@ fail() { report() { [ "${SHELL_ONLINE_INSTALL_REPORT:-1}" != 0 ] || return 0 + [ -z "$curl_agent" ] || return 0 case "${base_url:-}" in http://*|https://*) ;; *) return 0 ;; @@ -101,12 +113,15 @@ binary_url="$base_url/downloads/$binary_name" download() { source_url=$1 destination=$2 + # The agent flags are two words or none, on purpose unquoted. if command -v curl >/dev/null 2>&1; then - if ! curl -fsSL "$source_url" -o "$destination"; then + # shellcheck disable=SC2086 + if ! curl -fsSL $curl_agent "$source_url" -o "$destination"; then fail "download failed: $source_url" download_failed fi elif command -v wget >/dev/null 2>&1; then - if ! wget -q "$source_url" -O "$destination"; then + # shellcheck disable=SC2086 + if ! wget -q $wget_agent "$source_url" -O "$destination"; then fail "download failed: $source_url" download_failed fi else diff --git a/public/install.ps1 b/public/install.ps1 index 82e607c..77bfb17 100644 --- a/public/install.ps1 +++ b/public/install.ps1 @@ -10,9 +10,17 @@ $ErrorActionPreference = "Stop" # went: one request carrying a single word and the binary name, nothing else, # so that a platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +$web = @{ UseBasicParsing = $true } +if ($env:SHELL_ONLINE_INSTALL_CHECK -eq "1") { $web.UserAgent = "shell.online-install-check" } + $script:reported = $false function Report([string]$Outcome) { if ($env:SHELL_ONLINE_INSTALL_REPORT -eq "0") { return } + if ($web.ContainsKey("UserAgent")) { return } if ($BaseUrl -notmatch '^https?://') { return } $script:reported = $true $platform = if ($script:artifact) { $script:artifact } else { "unknown" } @@ -48,8 +56,8 @@ $manifestPath = Join-Path $temporaryDirectory "SHA256SUMS" try { New-Item -ItemType Directory -Path $temporaryDirectory | Out-Null - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath $manifest = Get-Content -Raw $manifestPath $match = [regex]::Match($manifest, "(?m)^([a-f0-9]{64}) " + [regex]::Escape($artifact) + "$") if (-not $match.Success) { Fail "release manifest has no valid checksum for $artifact" "manifest_missing" } diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 1d7d9d8..84696fa 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -163,4 +163,19 @@ if grep -q "install/report" "$report_log"; then exit 1 fi +# A check run says so on every request and reports nothing, even when +# reporting is otherwise on. +: > "$report_log" +if output=$(SHELL_ONLINE_INSTALL_REPORT=1 SHELL_ONLINE_INSTALL_CHECK=1 SHELL_ONLINE_TEST_REPORT_LOG=$report_log \ + SHELL_ONLINE_BASE_URL=https://installer.invalid SHELL_ONLINE_INSTALL_DIR=$test_root/report-install \ + PATH=$report_bin:$PATH sh "$installer" 2>&1); then + printf 'Installer with a failing download unexpectedly succeeded.\n' >&2 + exit 1 +fi +assert_contains "$(cat "$report_log")" "-A shell.online-install-check https://installer.invalid/downloads/shell-" +if grep -q "install/report" "$report_log"; then + printf 'A check run reported its outcome.\n' >&2 + exit 1 +fi + printf 'Installer integration scenarios passed.\n' diff --git a/shared/stats-snapshot.ts b/shared/stats-snapshot.ts index e699e90..a8f10c2 100644 --- a/shared/stats-snapshot.ts +++ b/shared/stats-snapshot.ts @@ -90,6 +90,12 @@ export interface AudienceRow extends Record { count: number; } +/** The earliest day people were counted on one surface. */ +export interface SurfaceSinceRow extends Record { + surface: string; + minimum: number | null; +} + /** Distinct visitor hashes seen on one surface in a period. */ export interface PeriodUniqueRow extends Record { surface: string; @@ -125,6 +131,8 @@ export interface StatsSnapshotRows { uniquesConfigured: boolean; /** Midnight UTC of the earliest visitor day still kept, or null when there is none. */ uniquesSince: number | null; + /** The same, per surface. */ + uniquesSinceBySurface: SurfaceSinceRow[]; } /** Midnight UTC of the day that contains `at`. */ @@ -507,13 +515,18 @@ export function buildRetentionCohorts( function buildUniques(rows: StatsSnapshotRows): StatsUniques { const surfaces = Object.fromEntries( - UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0 }]), + UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0, since: null }]), ) as Record; for (const row of rows.uniques) { if (!isUniqueSurface(row.surface)) continue; const unique = Number(row.unique_count); const fresh = Math.min(unique, Number(row.new_count)); - surfaces[row.surface] = { unique, new: fresh, returning: unique - fresh }; + surfaces[row.surface] = { ...surfaces[row.surface], unique, new: fresh, returning: unique - fresh }; + } + if (rows.uniquesConfigured) { + for (const row of rows.uniquesSinceBySurface) { + if (isUniqueSurface(row.surface) && row.minimum !== null) surfaces[row.surface].since = Number(row.minimum); + } } const days = new Map(); for (const row of rows.uniqueDays) { @@ -541,11 +554,14 @@ function buildUniques(rows: StatsSnapshotRows): StatsUniques { * range has to say so, or 29,333 views next to 84 people reads as nonsense. */ export function peopleCountedSince( - uniques: Pick, + uniques: Pick & { surfaces?: Record> }, rangeStart: number, + surface?: UniqueSurface, ): number | null { - if (!uniques.configured || uniques.since === null) return null; - return uniques.since > dayStart(rangeStart) ? uniques.since : null; + if (!uniques.configured) return null; + const since = surface === undefined ? uniques.since : uniques.surfaces?.[surface]?.since ?? null; + if (since === null) return null; + return since > dayStart(rangeStart) ? since : null; } export function statsRangeStart( @@ -578,7 +594,7 @@ function buildTrend( const end = Math.floor(now / stepMs) * stepMs; const points = new Map(); for (let at = start; at <= end; at += stepMs) { - points.set(at, { at, sessions: 0, shares: 0, collaborations: 0, pageViews: 0 }); + points.set(at, { at, sessions: 0, started: 0, shares: 0, collaborations: 0, pageViews: 0, installs: 0 }); } for (const row of rows) { const at = Math.floor(Number(row.bucket) / stepMs) * stepMs; @@ -586,9 +602,11 @@ function buildTrend( if (!point) continue; const count = Number(row.count); if (row.event === "session_created") point.sessions += count; + if (row.event === "session_started") point.started += count; if (row.event === "share_opened") point.shares += count; if (row.event === "collaboration_started") point.collaborations += count; if (row.event === "page_view") point.pageViews += count; + if (row.event === "binary_download") point.installs += count; } return Array.from(points.values()).slice(-180); } diff --git a/shared/stats.ts b/shared/stats.ts index 37507ae..1109ff1 100644 --- a/shared/stats.ts +++ b/shared/stats.ts @@ -33,10 +33,16 @@ export interface StatsInstallConversion { export interface StatsSeriesPoint { at: number; + /** Sessions created. */ sessions: number; + /** Sessions whose host connected. */ + started: number; shares: number; collaborations: number; + /** Landing and documentation views, crawlers left out. */ pageViews: number; + /** Release binaries served, crawlers left out. */ + installs: number; } export interface StatsBreakdownItem { @@ -61,6 +67,13 @@ export interface StatsUniqueCount { new: number; /** Of those, seen before the range began. */ returning: number; + /** + * Midnight UTC of the earliest day this surface's people were counted, or + * null. Surfaces start on different days: machines were re-keyed after + * visitors were first counted, so a "new" machine and a "new" visitor are + * measured from different starts. + */ + since: number | null; } export interface StatsUniqueDay { diff --git a/tests/analytics.test.ts b/tests/analytics.test.ts index ae70c4a..1f9faca 100644 --- a/tests/analytics.test.ts +++ b/tests/analytics.test.ts @@ -61,6 +61,39 @@ describe("analytics", () => { }); }); + it("counts monitors as crawlers and HTTP libraries as tools, never as people", () => { + for (const agent of ["shell.online-downloads-check", "shell.online-install-check", "Better Uptime Bot", "Pingdom.com_bot", "Checkly/1.0", "Site24x7", "Datadog Synthetics"]) { + expect(classifyDevice(agent)).toBe("bot"); + expect(classifyClient(agent)).toBe("bot"); + } + for (const agent of [ + "node", + "node-fetch/1.0", + "undici", + "Go-http-client/1.1", + "python-requests/2.31.0", + "Java/17.0.2", + "okhttp/4.12.0", + "axios/1.6.0", + "Mozilla/5.0 (Windows NT 10.0; Microsoft Windows 10.0.19045; en-US) PowerShell/7.4.1", + "Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.1", + ]) { + expect(classifyDevice(agent)).toBe("cli"); + expect(classifyClient(agent)).toBe("tool"); + } + expect(classifyDevice("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/129.0 Safari/537.36")).toBe("desktop"); + expect(classifyDevice("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 Version/17.6 Safari/605.1.15")).toBe("desktop"); + expect(classifyDevice("")).toBe("unknown"); + }); + + it("does not count a page a browser fetched ahead of time", () => { + const headers = { Accept: "text/html", "Sec-Fetch-Dest": "document" }; + expect(isDocumentNavigation(new Request("https://shell.online/", { headers }))).toBe(true); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch;prerender" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, Purpose: "prefetch" } }))).toBe(false); + }); + it("reduces user agents and referrers to coarse categories", () => { expect(classifyDevice("shell/0.3.4")).toBe("cli"); expect(classifyClient("shell/0.3.4")).toBe("shell/0.3.4"); diff --git a/tests/stats-copy.test.ts b/tests/stats-copy.test.ts index e530483..1388a32 100644 --- a/tests/stats-copy.test.ts +++ b/tests/stats-copy.test.ts @@ -55,6 +55,7 @@ function rows(overrides: Partial = {}): StatsSnapshotRows { installConversion: null, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + uniquesSinceBySurface: [], ...overrides, }; } diff --git a/tests/stats-database.test.ts b/tests/stats-database.test.ts index 5ee76fd..94d3591 100644 --- a/tests/stats-database.test.ts +++ b/tests/stats-database.test.ts @@ -8,6 +8,8 @@ import { collectStatsRows, HOUR_MS, initializeStatsSchema, + MACHINE_KEY_DAY, + migrateStatsData, parseStatsPresence, parseStatsRecord, recordStatsEvent, @@ -101,6 +103,19 @@ describe("the dashboard's database", () => { const day = collectStatsRows(sql, "24h", false, now); expect(day.rows.summary).toEqual([]); expect(day.rows.previous).toMatchObject({ rangeStart: now - 2 * DAY_MS, summary: [{ count: 2 }] }); + + /* Crawlers stay out of the charts, and installs and started sessions join them. */ + recordStatsEvent(sql, event(daysAgo(1), "page_view", "landing", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "cli", client: "curl" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "session_started", "cli", { device: "cli", client: "shell/0.16.0" })); + const charted = collectStatsRows(sql, "7d", false, now).rows.trend; + const hour = Math.floor(daysAgo(1) / HOUR_MS) * HOUR_MS; + expect(charted).toEqual([ + { bucket: hour, event: "binary_download", count: 1 }, + { bucket: hour, event: "page_view", count: 2 }, + { bucket: hour, event: "session_started", count: 1 }, + ]); }); it("splits the counted events by device class and ranks dimensions by count", () => { @@ -140,6 +155,10 @@ describe("the dashboard's database", () => { const { rows } = collectStatsRows(sql, "7d", true, now); expect(rows.uniquesSince).toBe(dayStart(daysAgo(10))); + expect(rows.uniquesSinceBySurface).toEqual([ + { surface: "cli", minimum: dayStart(daysAgo(1)) }, + { surface: "site", minimum: dayStart(daysAgo(10)) }, + ]); expect(bySurface(rows.uniques)).toEqual([ { surface: "cli", unique_count: 1, new_count: 1 }, { surface: "site", unique_count: 2, new_count: 1 }, @@ -192,6 +211,38 @@ describe("the dashboard's database", () => { expect(collectStatsRows(sql, "all", true, now).rows.installConversion).toMatchObject({ installers: 6 }); }); + it("drops the machine rows keyed the old way once, and never again", () => { + const sql = freshDatabase(); + /* A database from before the mark existed. */ + sql.exec("DELETE FROM dashboard_marks"); + const cutover = MACHINE_KEY_DAY + 6 * HOUR_MS; + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("o") })); + recordStatsEvent(sql, event(cutover, "binary_download", "darwin-arm64", { device: "cli", visitor: hash("p") })); + recordStatsEvent(sql, event(cutover - DAY_MS, "page_view", "landing", { visitor: hash("v") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("n") })); + /* Seen before and after the cut-over: the row survives, its first day with it. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + + migrateStatsData(sql); + const left = sql.exec<{ surface: string; visitor: string; first_day: number }>( + "SELECT surface, visitor, first_day FROM visitors ORDER BY surface, visitor", + ).toArray(); + expect(left).toEqual([ + { surface: "cli", visitor: hash("k"), first_day: dayStart(cutover - DAY_MS) }, + { surface: "cli", visitor: hash("n"), first_day: dayStart(cutover + DAY_MS) }, + { surface: "site", visitor: hash("v"), first_day: dayStart(cutover - DAY_MS) }, + ]); + expect(sql.exec<{ day: number }>("SELECT day FROM visitor_days WHERE visitor = ? ORDER BY day", hash("k")).toArray()) + .toEqual([{ day: dayStart(cutover + DAY_MS) }]); + + /* Done once: a later old-looking row is left alone. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("z") })); + initializeStatsSchema(sql); + migrateStatsData(sql); + expect(sql.exec<{ visitor: string }>("SELECT visitor FROM visitors WHERE visitor = ?", hash("z")).toArray()).toHaveLength(1); + }); + it("keeps a live presence lease until it ends, and drops it when told", () => { const sql = freshDatabase(); const key = (letter: string): string => letter.repeat(22); diff --git a/web/stats.ts b/web/stats.ts index bbdfe99..4e61401 100644 --- a/web/stats.ts +++ b/web/stats.ts @@ -6,6 +6,7 @@ import { type StatsRetentionCohort, type StatsSeriesPoint, type StatsSnapshot, + type UniqueSurface, } from "../shared/stats"; import { peopleCountedSince } from "../shared/stats-snapshot"; import { @@ -23,7 +24,7 @@ import { import { RELEASE_CHECKSUMS_PATH, RELEASE_VERSION } from "../shared/release"; import "./stats.css"; -type SeriesKey = "sessions" | "shares" | "collaborations" | "pageViews"; +type SeriesKey = "sessions" | "started" | "shares" | "collaborations" | "pageViews" | "installs"; interface ChartSeries { key: SeriesKey; @@ -37,7 +38,7 @@ const ACTIVITY_SERIES: ChartSeries[] = [ { key: "collaborations", label: "Collaborated", color: "#d7a6ff" }, ]; const TRAFFIC_SERIES: ChartSeries[] = [ - { key: "pageViews", label: "Page views", color: "#9ab7e8" }, + { key: "pageViews", label: "Page views by people", color: "#9ab7e8" }, ]; let activeDashboardCleanup: (() => void) | null = null; let statsRenderId = 0; @@ -357,7 +358,8 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { */ const peopleSince = peopleCountedSince(people, snapshot.rangeStart); /* Plain text: renderKpi escapes its detail. */ - const sinceNote = peopleSince === null ? "" : ` · counted since ${formatDay(peopleSince)}`; + const visitorsNote = peopleNote(snapshot, "site"); + const machinesNote = peopleNote(snapshot, "cli"); const totalViews = metrics.landingViews + metrics.docsViews; const otherViews = audiences.views.tools + audiences.views.unknown; const ctaByLink = snapshot.targets @@ -390,9 +392,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${renderKpi( "Visitors", people.configured ? site.unique : "—", - people.configured - ? `${integerFormatter.format(site.new)} new · ${integerFormatter.format(site.returning)} returning${sinceNote}` - : "people are not counted yet", + people.configured ? visitorsNote : "people are not counted yet", snapshot.trend, "pageViews", "silver", @@ -416,7 +416,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ? "no installer runs in this range" : `${formatPercent(ratio(figures.installs, figures.installerRuns))} of ${integerFormatter.format(figures.installerRuns)} installer run${figures.installerRuns === 1 ? "" : "s"}`, snapshot.trend, - "sessions", + "installs", "amber", delta(figures.installs, previous?.figures.installs), )} @@ -424,10 +424,10 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { "Sessions started", figures.sessionsStarted, people.configured - ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"}, ${integerFormatter.format(cli.new)} new${sinceNote}` + ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"} · ${machinesNote}` : `${integerFormatter.format(metrics.sessionsCreated)} created`, snapshot.trend, - "sessions", + "started", "violet", delta(figures.sessionsStarted, previous?.figures.sessionsStarted), )} @@ -461,7 +461,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${escapeHtml(rangeLabel)}

${escapeHtml(funnelInsight(snapshot))}

- ${renderFunnel(snapshot, peopleSince)} + ${renderFunnel(snapshot)} ${peopleSince === null ? "" : `

People have been counted since ${escapeHtml(formatDay(peopleSince))}; event counts run from the start of the range. Until the range begins after that day, a people figure covers fewer days than the count beside it.

`} ${renderInstallConversion(snapshot)} ${renderUniquesStrip(snapshot)} @@ -748,9 +748,16 @@ function renderTimeChart( `; } -function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): string { +/** Which surface a funnel step's people are counted on, for the day their count starts. */ +const STEP_SURFACES: Record = { visited: "site", installer: "install", session: "cli", opened: "viewer" }; + +function renderFunnel(snapshot: StatsSnapshot): string { const steps = snapshot.funnel; - const since = peopleSince === null ? "" : ` since ${escapeHtml(formatDay(peopleSince))}`; + const sinceFor = (key: string): string => { + const surface = STEP_SURFACES[key]; + const since = surface === undefined ? null : peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + return since === null ? "" : ` since ${escapeHtml(formatDay(since))}`; + }; const colors = ["#9ab7e8", "#8eafff", "#819de5", "#f4bd78", "#8eafff", "#75dac2", "#d7a6ff"]; const maximum = Math.max(1, ...steps.map((step) => step.count)); return ` @@ -771,7 +778,7 @@ function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): stri return `
${escapeHtml(step.label)} - ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${since}`} + ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${sinceFor(step.key)}`}
${escapeHtml(share)} ${excluded} @@ -799,15 +806,30 @@ function renderInstallConversion(snapshot: StatsSnapshot): string { return `

${escapeHtml(verdict)} Machines are followed by address from the binary download to the first session.

`; } +/* + * What to say beside a people figure. New means first seen since the range + * began, so until a whole range has passed since this surface's people were + * first counted, everyone is new and the split says nothing: name the day it + * starts to. The 24h range counts people by UTC day, which is two of them. + */ +function peopleNote(snapshot: StatsSnapshot, surface: UniqueSurface): string { + const count = snapshot.uniques.surfaces[surface]; + const days = snapshot.range === "24h" ? " · by UTC day, so two days" : ""; + const since = peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + if (since === null) return `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} returning${days}`; + const knownFrom = formatDay(since + (snapshot.generatedAt - snapshot.rangeStart)); + return `counted since ${formatDay(since)}; new or returning cannot be told until ${knownFrom}${days}`; +} + function renderUniquesStrip(snapshot: StatsSnapshot): string { const people = snapshot.uniques; - const cell = (label: string, surface: keyof typeof people.surfaces): string => { + const cell = (label: string, surface: UniqueSurface): string => { const count = people.surfaces[surface]; return ` ${escapeHtml(label)} ${people.configured ? integerFormatter.format(count.unique) : "—"} - ${people.configured ? `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} back` : "not counted"} + ${people.configured ? escapeHtml(peopleNote(snapshot, surface)) : "not counted"} `; }; diff --git a/worker/analytics.ts b/worker/analytics.ts index 08e7617..7ddca32 100644 --- a/worker/analytics.ts +++ b/worker/analytics.ts @@ -303,6 +303,8 @@ export function documentTarget(pathname: string, status: number, currentVersion: export function isDocumentNavigation(request: Request): boolean { if (request.method !== "GET") return false; + /* A browser fetching a page it guesses will be wanted has not shown it to anyone. */ + if (request.headers.get("Sec-Purpose")?.includes("prefetch") || request.headers.get("Purpose") === "prefetch") return false; if (request.headers.get("Sec-Fetch-Dest") === "document") return true; return request.headers.get("Accept")?.toLowerCase().includes("text/html") ?? false; } @@ -312,11 +314,26 @@ export function binaryDownloadTarget(pathname: string): string | null { return match ? `${match[1]}-${match[2]}` : null; } +/** + * Crawlers, and monitors that say so, the site's own download checks among + * them: requests to count, never people. A monitor that says nothing looks + * like whatever library it used, which the tool pattern catches next. + */ +const CRAWLER_AGENT = /bot|crawler|spider|slurp|headless|preview|monitor|uptime|pingdom|statuscake|checkly|site24x7|synthetic|shell\.online-(?:downloads|install)-check/; + +/** + * Tools a person or a script runs: the CLI, curl and wget, PowerShell's web + * cmdlets, and the HTTP libraries scripts are written with. Not browsers, + * whatever else the agent string says; the default used to be "desktop", + * which made every Node script a person. + */ +const TOOL_AGENT = /^shell\/|curl|wget|powershell|\bnode\b|node-fetch|undici|go-http-client|python|\bjava\b|\bjava\/|okhttp|axios|libwww|httpie/; + export function classifyDevice(userAgent: string, mobileHint: string | null = null): DeviceClass { const normalized = userAgent.toLowerCase(); if (!normalized) return "unknown"; - if (/bot|crawler|spider|slurp|headless|preview/.test(normalized)) return "bot"; - if (/^shell\//.test(normalized) || /curl|wget/.test(normalized)) return "cli"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "cli"; if (mobileHint === "?1" || /iphone|ipod|android.+mobile|mobile.+android/.test(normalized)) { return "mobile"; } @@ -327,9 +344,11 @@ export function classifyDevice(userAgent: string, mobileHint: string | null = nu export function classifyClient(userAgent: string): string { const shellVersion = userAgent.match(/\bshell\/(\d{1,3}\.\d{1,3}\.\d{1,3})\b/i)?.[1]; if (shellVersion) return `shell/${shellVersion}`; - if (/curl/i.test(userAgent)) return "curl"; - if (/wget/i.test(userAgent)) return "wget"; - if (/bot|crawler|spider|slurp|headless|preview/i.test(userAgent)) return "bot"; + const normalized = userAgent.toLowerCase(); + if (/curl/.test(normalized)) return "curl"; + if (/wget/.test(normalized)) return "wget"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "tool"; return userAgent ? "web" : "unknown"; } diff --git a/worker/stats-database.ts b/worker/stats-database.ts index 2611e80..669db83 100644 --- a/worker/stats-database.ts +++ b/worker/stats-database.ts @@ -21,6 +21,7 @@ import { type PreviousPeriodRows, type RetentionRow, type StatsSnapshotRows, + type SurfaceSinceRow, type UniqueDayRow, type UniqueSummaryRow, } from "../shared/stats-snapshot"; @@ -136,6 +137,35 @@ export function initializeStatsSchema(sql: StatsSql): void { ) `); sql.exec("CREATE INDEX IF NOT EXISTS visitors_last_day ON visitors(last_day)"); + sql.exec(` + CREATE TABLE IF NOT EXISTS dashboard_marks ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + migrateStatsData(sql); +} + +/** + * The last day machines were keyed by address and user agent; since the day + * after, by address alone. A machine row from before cannot match the same + * machine seen after, so it would sit in the cohorts as a ghost that never + * came back. + */ +export const MACHINE_KEY_DAY = Date.UTC(2026, 8, 15); + +/** + * One-time repairs to what is stored, each done once and marked as done. + * The first drops the machine rows keyed the old way; a machine seen on the + * cut-over day itself is lost with them, a few hours of history against 120 + * days of ghosts. + */ +export function migrateStatsData(sql: StatsSql): void { + const done = sql.exec<{ value: number }>("SELECT value FROM dashboard_marks WHERE name = 'machine_key'").toArray(); + if (done.length > 0) return; + sql.exec("DELETE FROM visitor_days WHERE surface IN ('cli', 'install') AND day <= ?", MACHINE_KEY_DAY); + sql.exec("DELETE FROM visitors WHERE surface IN ('cli', 'install') AND last_day <= ?", MACHINE_KEY_DAY); + sql.exec("INSERT INTO dashboard_marks (name, value) VALUES ('machine_key', 1)", ); } /** One event into its hour bucket, and its person into the day's visitors when it has one. */ @@ -228,6 +258,9 @@ export function collectStatsRows( const collectingSince = sql.exec("SELECT MIN(bucket) AS minimum FROM metric_hourly").one().minimum; /* Taken after the purge, so it is the earliest day people can still be counted from. */ const uniquesSince = sql.exec("SELECT MIN(day) AS minimum FROM visitor_days").one().minimum; + const uniquesSinceBySurface = sql.exec( + "SELECT surface, MIN(day) AS minimum FROM visitor_days GROUP BY surface ORDER BY surface", + ).toArray(); const rangeStart = statsRangeStart(range, now, collectingSince); /* Buckets are hours and events at most ten minutes ahead of this clock, so nothing sits past tomorrow. */ const rangeEnd = now + DAY_MS; @@ -255,13 +288,15 @@ export function collectStatsRows( ).toArray(), }; } + /* The charts are about people, so crawlers stay out of them; the ledger keeps every request. */ const trend = sql.exec( `SELECT bucket, event, SUM(count) AS count FROM metric_hourly WHERE bucket >= ? - AND event IN ('session_created', 'share_opened', 'collaboration_started', 'page_view') + AND event IN ('session_created', 'session_started', 'share_opened', 'collaboration_started', 'page_view', 'binary_download') + AND device != 'bot' GROUP BY bucket, event - ORDER BY bucket`, + ORDER BY bucket, event`, rangeStart, ).toArray(); const live = sql.exec( @@ -346,6 +381,7 @@ export function collectStatsRows( installConversion, uniquesConfigured, uniquesSince, + uniquesSinceBySurface, }, }; } diff --git a/worker/stats-store.test.ts b/worker/stats-store.test.ts index 3c84b6a..77abd21 100644 --- a/worker/stats-store.test.ts +++ b/worker/stats-store.test.ts @@ -40,6 +40,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "all", now, collectingSince); expect(snapshot.metrics).toMatchObject({ @@ -73,6 +74,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "24h", now, now - 24 * 60 * 60 * 1_000); expect(snapshot.metrics.activeSessions).toBe(0); @@ -151,6 +153,12 @@ describe("people and the funnel", () => { installConversion: { installers: 12, matured: 9, started: 4 }, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + /* Machines were re-keyed after visitors were first counted, so their start is later. */ + uniquesSinceBySurface: [ + { surface: "site", minimum: dayStart(now - 20 * DAY_MS) }, + { surface: "cli", minimum: dayStart(now - 2 * DAY_MS) }, + { surface: "install", minimum: dayStart(now - 2 * DAY_MS) }, + ], }; it("keeps docs, unknown paths and 404s apart and counts people beside events", () => { @@ -164,8 +172,8 @@ describe("people and the funnel", () => { ctaClicks: 12, binaryDownloads: 4, }); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50 }); - expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50, since: dayStart(now - 20 * DAY_MS) }); + expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7, since: dayStart(now - 2 * DAY_MS) }); expect(snapshot.uniques.daily.map((day) => [day.site, day.cli])).toEqual([[120, 4], [140, 0]]); expect(snapshot.rates.signup).toBeCloseTo(12 / 969); expect(snapshot.breakdowns.pages.map((page) => page.label)).toEqual(["landing", "unknown_path", "docs_app", "docs", "not_found"]); @@ -310,6 +318,22 @@ describe("people and the funnel", () => { expect(buildStatsSnapshot({ ...rows, previous: null }, "all", now, now - 30 * DAY_MS).previous).toBeNull(); }); + it("draws installs and started sessions as their own lines", () => { + const hour = 60 * 60 * 1_000; + const bucket = Math.floor((now - DAY_MS) / hour) * hour; + const snapshot = buildStatsSnapshot({ + ...rows, + trend: [ + { bucket, event: "session_created", count: 4 }, + { bucket, event: "session_started", count: 3 }, + { bucket, event: "binary_download", count: 2 }, + { bucket, event: "page_view", count: 9 }, + ], + }, "7d", now, rangeStart); + const point = snapshot.trend.find((candidate) => candidate.at === Math.floor(bucket / (6 * hour)) * 6 * hour); + expect(point).toMatchObject({ sessions: 4, started: 3, installs: 2, pageViews: 9, shares: 0, collaborations: 0 }); + }); + it("follows machines from the installer to a first session", () => { const snapshot = buildStatsSnapshot(rows, "7d", now, rangeStart); expect(snapshot.installConversion).toEqual({ installers: 12, matured: 9, started: 4 }); @@ -323,7 +347,7 @@ describe("people and the funnel", () => { expect(snapshot.uniques.since).toBeNull(); expect(snapshot.installConversion).toBeNull(); expect(snapshot.funnel[0].unique).toBeNull(); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0, since: null }); }); /* @@ -342,6 +366,16 @@ describe("people and the funnel", () => { expect(partial.uniques.since).toBe(yesterday); expect(peopleCountedSince(partial.uniques, partial.rangeStart)).toBe(yesterday); + /* Each surface starts on its own day: visitors cover the week, machines do not. */ + expect(covered.uniques.surfaces.site.since).toBe(dayStart(now - 20 * DAY_MS)); + expect(covered.uniques.surfaces.cli.since).toBe(dayStart(now - 2 * DAY_MS)); + expect(covered.uniques.surfaces.viewer.since).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "site")).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "cli")).toBe(dayStart(now - 2 * DAY_MS)); + expect(peopleCountedSince(covered.uniques, rangeStart, "viewer")).toBeNull(); + const unconfigured = buildStatsSnapshot({ ...rows, uniquesConfigured: false }, "7d", now, rangeStart); + expect(unconfigured.uniques.surfaces.cli.since).toBeNull(); + /* People are kept by day, so a range that starts inside their first day is covered. */ expect(peopleCountedSince({ configured: true, since: dayStart(rangeStart) }, rangeStart + 60_000)).toBeNull(); expect(peopleCountedSince({ configured: false, since: yesterday }, now - 30 * DAY_MS)).toBeNull();