diff --git a/.github/workflows/downloads.yml b/.github/workflows/downloads.yml index a919ec6..58af418 100644 --- a/.github/workflows/downloads.yml +++ b/.github/workflows/downloads.yml @@ -58,21 +58,25 @@ jobs: script: powershell runs-on: ${{ matrix.os }} timeout-minutes: 10 + # Forty-eight installs a day from fresh runner addresses would read as + # people on the statistics dashboard. The check agent on every request, + # the scripts' own included, tells the site this is a monitor. env: ORIGIN: ${{ inputs.origin || 'https://shell.online' }} + SHELL_ONLINE_INSTALL_CHECK: "1" steps: - name: Install with the documented command if: matrix.script == 'posix' run: | export SHELL_ONLINE_INSTALL_DIR="$RUNNER_TEMP/shell-online" - curl -fsSL "$ORIGIN/install" | sh + curl -fsSL -A shell.online-install-check "$ORIGIN/install" | sh "$SHELL_ONLINE_INSTALL_DIR/shell" --version - name: Install with the documented command if: matrix.script == 'powershell' shell: pwsh run: | $env:SHELL_ONLINE_INSTALL_DIR = Join-Path $env:RUNNER_TEMP "shell-online" - Invoke-RestMethod "$env:ORIGIN/install.ps1" | Invoke-Expression + Invoke-RestMethod -UserAgent shell.online-install-check "$env:ORIGIN/install.ps1" | Invoke-Expression & (Join-Path $env:SHELL_ONLINE_INSTALL_DIR "shell.exe") --version report: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad69e4..04f6726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,28 @@ All notable user-visible changes are recorded here. Versions follow [Semantic Ve ## Unreleased +### Fixed + +- The scheduled download check installed on three fresh GitHub runners every + half hour and was counted as installs, installer runs, installer outcomes + and new machines, which is where a dashboard day of ninety installs and + fifty-six new machines came from. Both install scripts now take + `SHELL_ONLINE_INSTALL_CHECK=1`, which puts a check user agent on every + request and reports nothing; the workflow sets it, and the site counts + that agent, and monitors in general, as crawlers. +- HTTP libraries and PowerShell's web cmdlets were classified as desktop + browsers, so a Node script or a Windows install read as a person reading + the installer. They are tools now, and a Windows install counts as a run. +- The installs tile drew the sessions line. The trend now carries installs + and started sessions as their own series, crawlers left out of every line. +- "New" people are measured from the day a surface's people were first + counted, per surface, and while a whole range has not yet passed since + that day the split is replaced by the day it becomes meaningful. Machine + rows keyed the old way are dropped once, so they do not sit in the cohorts + as machines that never came back. +- Pages a browser fetched ahead of time (prefetch, prerender) are not views. +- The 24h range says that people are counted by UTC day, so over two days. + ## [0.16.0] — 2026-09-15 ### Changed diff --git a/public/install b/public/install index 0f94ffb..6ff8378 100755 --- a/public/install +++ b/public/install @@ -6,6 +6,17 @@ set -eu # checksum_mismatch, ...) and the binary name, nothing else, so that a # platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +curl_agent= +wget_agent= +if [ "${SHELL_ONLINE_INSTALL_CHECK:-0}" = 1 ]; then + curl_agent="-A shell.online-install-check" + wget_agent="-U shell.online-install-check" +fi fail() { printf 'shell.online: %s\n' "$1" >&2 @@ -15,6 +26,7 @@ fail() { report() { [ "${SHELL_ONLINE_INSTALL_REPORT:-1}" != 0 ] || return 0 + [ -z "$curl_agent" ] || return 0 case "${base_url:-}" in http://*|https://*) ;; *) return 0 ;; @@ -101,12 +113,15 @@ binary_url="$base_url/downloads/$binary_name" download() { source_url=$1 destination=$2 + # The agent flags are two words or none, on purpose unquoted. if command -v curl >/dev/null 2>&1; then - if ! curl -fsSL "$source_url" -o "$destination"; then + # shellcheck disable=SC2086 + if ! curl -fsSL $curl_agent "$source_url" -o "$destination"; then fail "download failed: $source_url" download_failed fi elif command -v wget >/dev/null 2>&1; then - if ! wget -q "$source_url" -O "$destination"; then + # shellcheck disable=SC2086 + if ! wget -q $wget_agent "$source_url" -O "$destination"; then fail "download failed: $source_url" download_failed fi else diff --git a/public/install.ps1 b/public/install.ps1 index 82e607c..77bfb17 100644 --- a/public/install.ps1 +++ b/public/install.ps1 @@ -10,9 +10,17 @@ $ErrorActionPreference = "Stop" # went: one request carrying a single word and the binary name, nothing else, # so that a platform that keeps failing gets noticed and fixed. Set # SHELL_ONLINE_INSTALL_REPORT=0 to skip it. +# A run that is only a check, a monitor or a CI job, should set +# SHELL_ONLINE_INSTALL_CHECK=1: every request then carries the user agent +# shell.online-install-check, which the site counts as a monitor and not as +# a person, and nothing is reported. +$web = @{ UseBasicParsing = $true } +if ($env:SHELL_ONLINE_INSTALL_CHECK -eq "1") { $web.UserAgent = "shell.online-install-check" } + $script:reported = $false function Report([string]$Outcome) { if ($env:SHELL_ONLINE_INSTALL_REPORT -eq "0") { return } + if ($web.ContainsKey("UserAgent")) { return } if ($BaseUrl -notmatch '^https?://') { return } $script:reported = $true $platform = if ($script:artifact) { $script:artifact } else { "unknown" } @@ -48,8 +56,8 @@ $manifestPath = Join-Path $temporaryDirectory "SHA256SUMS" try { New-Item -ItemType Directory -Path $temporaryDirectory | Out-Null - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath - Invoke-WebRequest -UseBasicParsing -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/$artifact" -OutFile $binaryPath + Invoke-WebRequest @web -Uri "$BaseUrl/downloads/SHA256SUMS" -OutFile $manifestPath $manifest = Get-Content -Raw $manifestPath $match = [regex]::Match($manifest, "(?m)^([a-f0-9]{64}) " + [regex]::Escape($artifact) + "$") if (-not $match.Success) { Fail "release manifest has no valid checksum for $artifact" "manifest_missing" } diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 1d7d9d8..84696fa 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -163,4 +163,19 @@ if grep -q "install/report" "$report_log"; then exit 1 fi +# A check run says so on every request and reports nothing, even when +# reporting is otherwise on. +: > "$report_log" +if output=$(SHELL_ONLINE_INSTALL_REPORT=1 SHELL_ONLINE_INSTALL_CHECK=1 SHELL_ONLINE_TEST_REPORT_LOG=$report_log \ + SHELL_ONLINE_BASE_URL=https://installer.invalid SHELL_ONLINE_INSTALL_DIR=$test_root/report-install \ + PATH=$report_bin:$PATH sh "$installer" 2>&1); then + printf 'Installer with a failing download unexpectedly succeeded.\n' >&2 + exit 1 +fi +assert_contains "$(cat "$report_log")" "-A shell.online-install-check https://installer.invalid/downloads/shell-" +if grep -q "install/report" "$report_log"; then + printf 'A check run reported its outcome.\n' >&2 + exit 1 +fi + printf 'Installer integration scenarios passed.\n' diff --git a/shared/stats-snapshot.ts b/shared/stats-snapshot.ts index e699e90..a8f10c2 100644 --- a/shared/stats-snapshot.ts +++ b/shared/stats-snapshot.ts @@ -90,6 +90,12 @@ export interface AudienceRow extends Record { count: number; } +/** The earliest day people were counted on one surface. */ +export interface SurfaceSinceRow extends Record { + surface: string; + minimum: number | null; +} + /** Distinct visitor hashes seen on one surface in a period. */ export interface PeriodUniqueRow extends Record { surface: string; @@ -125,6 +131,8 @@ export interface StatsSnapshotRows { uniquesConfigured: boolean; /** Midnight UTC of the earliest visitor day still kept, or null when there is none. */ uniquesSince: number | null; + /** The same, per surface. */ + uniquesSinceBySurface: SurfaceSinceRow[]; } /** Midnight UTC of the day that contains `at`. */ @@ -507,13 +515,18 @@ export function buildRetentionCohorts( function buildUniques(rows: StatsSnapshotRows): StatsUniques { const surfaces = Object.fromEntries( - UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0 }]), + UNIQUE_SURFACES.map((surface): [UniqueSurface, StatsUniqueCount] => [surface, { unique: 0, new: 0, returning: 0, since: null }]), ) as Record; for (const row of rows.uniques) { if (!isUniqueSurface(row.surface)) continue; const unique = Number(row.unique_count); const fresh = Math.min(unique, Number(row.new_count)); - surfaces[row.surface] = { unique, new: fresh, returning: unique - fresh }; + surfaces[row.surface] = { ...surfaces[row.surface], unique, new: fresh, returning: unique - fresh }; + } + if (rows.uniquesConfigured) { + for (const row of rows.uniquesSinceBySurface) { + if (isUniqueSurface(row.surface) && row.minimum !== null) surfaces[row.surface].since = Number(row.minimum); + } } const days = new Map(); for (const row of rows.uniqueDays) { @@ -541,11 +554,14 @@ function buildUniques(rows: StatsSnapshotRows): StatsUniques { * range has to say so, or 29,333 views next to 84 people reads as nonsense. */ export function peopleCountedSince( - uniques: Pick, + uniques: Pick & { surfaces?: Record> }, rangeStart: number, + surface?: UniqueSurface, ): number | null { - if (!uniques.configured || uniques.since === null) return null; - return uniques.since > dayStart(rangeStart) ? uniques.since : null; + if (!uniques.configured) return null; + const since = surface === undefined ? uniques.since : uniques.surfaces?.[surface]?.since ?? null; + if (since === null) return null; + return since > dayStart(rangeStart) ? since : null; } export function statsRangeStart( @@ -578,7 +594,7 @@ function buildTrend( const end = Math.floor(now / stepMs) * stepMs; const points = new Map(); for (let at = start; at <= end; at += stepMs) { - points.set(at, { at, sessions: 0, shares: 0, collaborations: 0, pageViews: 0 }); + points.set(at, { at, sessions: 0, started: 0, shares: 0, collaborations: 0, pageViews: 0, installs: 0 }); } for (const row of rows) { const at = Math.floor(Number(row.bucket) / stepMs) * stepMs; @@ -586,9 +602,11 @@ function buildTrend( if (!point) continue; const count = Number(row.count); if (row.event === "session_created") point.sessions += count; + if (row.event === "session_started") point.started += count; if (row.event === "share_opened") point.shares += count; if (row.event === "collaboration_started") point.collaborations += count; if (row.event === "page_view") point.pageViews += count; + if (row.event === "binary_download") point.installs += count; } return Array.from(points.values()).slice(-180); } diff --git a/shared/stats.ts b/shared/stats.ts index 37507ae..1109ff1 100644 --- a/shared/stats.ts +++ b/shared/stats.ts @@ -33,10 +33,16 @@ export interface StatsInstallConversion { export interface StatsSeriesPoint { at: number; + /** Sessions created. */ sessions: number; + /** Sessions whose host connected. */ + started: number; shares: number; collaborations: number; + /** Landing and documentation views, crawlers left out. */ pageViews: number; + /** Release binaries served, crawlers left out. */ + installs: number; } export interface StatsBreakdownItem { @@ -61,6 +67,13 @@ export interface StatsUniqueCount { new: number; /** Of those, seen before the range began. */ returning: number; + /** + * Midnight UTC of the earliest day this surface's people were counted, or + * null. Surfaces start on different days: machines were re-keyed after + * visitors were first counted, so a "new" machine and a "new" visitor are + * measured from different starts. + */ + since: number | null; } export interface StatsUniqueDay { diff --git a/tests/analytics.test.ts b/tests/analytics.test.ts index ae70c4a..1f9faca 100644 --- a/tests/analytics.test.ts +++ b/tests/analytics.test.ts @@ -61,6 +61,39 @@ describe("analytics", () => { }); }); + it("counts monitors as crawlers and HTTP libraries as tools, never as people", () => { + for (const agent of ["shell.online-downloads-check", "shell.online-install-check", "Better Uptime Bot", "Pingdom.com_bot", "Checkly/1.0", "Site24x7", "Datadog Synthetics"]) { + expect(classifyDevice(agent)).toBe("bot"); + expect(classifyClient(agent)).toBe("bot"); + } + for (const agent of [ + "node", + "node-fetch/1.0", + "undici", + "Go-http-client/1.1", + "python-requests/2.31.0", + "Java/17.0.2", + "okhttp/4.12.0", + "axios/1.6.0", + "Mozilla/5.0 (Windows NT 10.0; Microsoft Windows 10.0.19045; en-US) PowerShell/7.4.1", + "Mozilla/5.0 (Windows NT; Windows NT 10.0; en-US) WindowsPowerShell/5.1.19041.1", + ]) { + expect(classifyDevice(agent)).toBe("cli"); + expect(classifyClient(agent)).toBe("tool"); + } + expect(classifyDevice("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/129.0 Safari/537.36")).toBe("desktop"); + expect(classifyDevice("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 Version/17.6 Safari/605.1.15")).toBe("desktop"); + expect(classifyDevice("")).toBe("unknown"); + }); + + it("does not count a page a browser fetched ahead of time", () => { + const headers = { Accept: "text/html", "Sec-Fetch-Dest": "document" }; + expect(isDocumentNavigation(new Request("https://shell.online/", { headers }))).toBe(true); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, "Sec-Purpose": "prefetch;prerender" } }))).toBe(false); + expect(isDocumentNavigation(new Request("https://shell.online/", { headers: { ...headers, Purpose: "prefetch" } }))).toBe(false); + }); + it("reduces user agents and referrers to coarse categories", () => { expect(classifyDevice("shell/0.3.4")).toBe("cli"); expect(classifyClient("shell/0.3.4")).toBe("shell/0.3.4"); diff --git a/tests/stats-copy.test.ts b/tests/stats-copy.test.ts index e530483..1388a32 100644 --- a/tests/stats-copy.test.ts +++ b/tests/stats-copy.test.ts @@ -55,6 +55,7 @@ function rows(overrides: Partial = {}): StatsSnapshotRows { installConversion: null, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + uniquesSinceBySurface: [], ...overrides, }; } diff --git a/tests/stats-database.test.ts b/tests/stats-database.test.ts index 5ee76fd..94d3591 100644 --- a/tests/stats-database.test.ts +++ b/tests/stats-database.test.ts @@ -8,6 +8,8 @@ import { collectStatsRows, HOUR_MS, initializeStatsSchema, + MACHINE_KEY_DAY, + migrateStatsData, parseStatsPresence, parseStatsRecord, recordStatsEvent, @@ -101,6 +103,19 @@ describe("the dashboard's database", () => { const day = collectStatsRows(sql, "24h", false, now); expect(day.rows.summary).toEqual([]); expect(day.rows.previous).toMatchObject({ rangeStart: now - 2 * DAY_MS, summary: [{ count: 2 }] }); + + /* Crawlers stay out of the charts, and installs and started sessions join them. */ + recordStatsEvent(sql, event(daysAgo(1), "page_view", "landing", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "cli", client: "curl" })); + recordStatsEvent(sql, event(daysAgo(1), "binary_download", "linux-amd64", { device: "bot", client: "bot" })); + recordStatsEvent(sql, event(daysAgo(1), "session_started", "cli", { device: "cli", client: "shell/0.16.0" })); + const charted = collectStatsRows(sql, "7d", false, now).rows.trend; + const hour = Math.floor(daysAgo(1) / HOUR_MS) * HOUR_MS; + expect(charted).toEqual([ + { bucket: hour, event: "binary_download", count: 1 }, + { bucket: hour, event: "page_view", count: 2 }, + { bucket: hour, event: "session_started", count: 1 }, + ]); }); it("splits the counted events by device class and ranks dimensions by count", () => { @@ -140,6 +155,10 @@ describe("the dashboard's database", () => { const { rows } = collectStatsRows(sql, "7d", true, now); expect(rows.uniquesSince).toBe(dayStart(daysAgo(10))); + expect(rows.uniquesSinceBySurface).toEqual([ + { surface: "cli", minimum: dayStart(daysAgo(1)) }, + { surface: "site", minimum: dayStart(daysAgo(10)) }, + ]); expect(bySurface(rows.uniques)).toEqual([ { surface: "cli", unique_count: 1, new_count: 1 }, { surface: "site", unique_count: 2, new_count: 1 }, @@ -192,6 +211,38 @@ describe("the dashboard's database", () => { expect(collectStatsRows(sql, "all", true, now).rows.installConversion).toMatchObject({ installers: 6 }); }); + it("drops the machine rows keyed the old way once, and never again", () => { + const sql = freshDatabase(); + /* A database from before the mark existed. */ + sql.exec("DELETE FROM dashboard_marks"); + const cutover = MACHINE_KEY_DAY + 6 * HOUR_MS; + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("o") })); + recordStatsEvent(sql, event(cutover, "binary_download", "darwin-arm64", { device: "cli", visitor: hash("p") })); + recordStatsEvent(sql, event(cutover - DAY_MS, "page_view", "landing", { visitor: hash("v") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("n") })); + /* Seen before and after the cut-over: the row survives, its first day with it. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + recordStatsEvent(sql, event(cutover + DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("k") })); + + migrateStatsData(sql); + const left = sql.exec<{ surface: string; visitor: string; first_day: number }>( + "SELECT surface, visitor, first_day FROM visitors ORDER BY surface, visitor", + ).toArray(); + expect(left).toEqual([ + { surface: "cli", visitor: hash("k"), first_day: dayStart(cutover - DAY_MS) }, + { surface: "cli", visitor: hash("n"), first_day: dayStart(cutover + DAY_MS) }, + { surface: "site", visitor: hash("v"), first_day: dayStart(cutover - DAY_MS) }, + ]); + expect(sql.exec<{ day: number }>("SELECT day FROM visitor_days WHERE visitor = ? ORDER BY day", hash("k")).toArray()) + .toEqual([{ day: dayStart(cutover + DAY_MS) }]); + + /* Done once: a later old-looking row is left alone. */ + recordStatsEvent(sql, event(cutover - DAY_MS, "session_created", "cli", { device: "cli", visitor: hash("z") })); + initializeStatsSchema(sql); + migrateStatsData(sql); + expect(sql.exec<{ visitor: string }>("SELECT visitor FROM visitors WHERE visitor = ?", hash("z")).toArray()).toHaveLength(1); + }); + it("keeps a live presence lease until it ends, and drops it when told", () => { const sql = freshDatabase(); const key = (letter: string): string => letter.repeat(22); diff --git a/web/stats.ts b/web/stats.ts index bbdfe99..4e61401 100644 --- a/web/stats.ts +++ b/web/stats.ts @@ -6,6 +6,7 @@ import { type StatsRetentionCohort, type StatsSeriesPoint, type StatsSnapshot, + type UniqueSurface, } from "../shared/stats"; import { peopleCountedSince } from "../shared/stats-snapshot"; import { @@ -23,7 +24,7 @@ import { import { RELEASE_CHECKSUMS_PATH, RELEASE_VERSION } from "../shared/release"; import "./stats.css"; -type SeriesKey = "sessions" | "shares" | "collaborations" | "pageViews"; +type SeriesKey = "sessions" | "started" | "shares" | "collaborations" | "pageViews" | "installs"; interface ChartSeries { key: SeriesKey; @@ -37,7 +38,7 @@ const ACTIVITY_SERIES: ChartSeries[] = [ { key: "collaborations", label: "Collaborated", color: "#d7a6ff" }, ]; const TRAFFIC_SERIES: ChartSeries[] = [ - { key: "pageViews", label: "Page views", color: "#9ab7e8" }, + { key: "pageViews", label: "Page views by people", color: "#9ab7e8" }, ]; let activeDashboardCleanup: (() => void) | null = null; let statsRenderId = 0; @@ -357,7 +358,8 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { */ const peopleSince = peopleCountedSince(people, snapshot.rangeStart); /* Plain text: renderKpi escapes its detail. */ - const sinceNote = peopleSince === null ? "" : ` · counted since ${formatDay(peopleSince)}`; + const visitorsNote = peopleNote(snapshot, "site"); + const machinesNote = peopleNote(snapshot, "cli"); const totalViews = metrics.landingViews + metrics.docsViews; const otherViews = audiences.views.tools + audiences.views.unknown; const ctaByLink = snapshot.targets @@ -390,9 +392,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${renderKpi( "Visitors", people.configured ? site.unique : "—", - people.configured - ? `${integerFormatter.format(site.new)} new · ${integerFormatter.format(site.returning)} returning${sinceNote}` - : "people are not counted yet", + people.configured ? visitorsNote : "people are not counted yet", snapshot.trend, "pageViews", "silver", @@ -416,7 +416,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ? "no installer runs in this range" : `${formatPercent(ratio(figures.installs, figures.installerRuns))} of ${integerFormatter.format(figures.installerRuns)} installer run${figures.installerRuns === 1 ? "" : "s"}`, snapshot.trend, - "sessions", + "installs", "amber", delta(figures.installs, previous?.figures.installs), )} @@ -424,10 +424,10 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { "Sessions started", figures.sessionsStarted, people.configured - ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"}, ${integerFormatter.format(cli.new)} new${sinceNote}` + ? `on ${integerFormatter.format(cli.unique)} machine${cli.unique === 1 ? "" : "s"} · ${machinesNote}` : `${integerFormatter.format(metrics.sessionsCreated)} created`, snapshot.trend, - "sessions", + "started", "violet", delta(figures.sessionsStarted, previous?.figures.sessionsStarted), )} @@ -461,7 +461,7 @@ function renderSnapshot(container: HTMLElement, snapshot: StatsSnapshot): void { ${escapeHtml(rangeLabel)}

${escapeHtml(funnelInsight(snapshot))}

- ${renderFunnel(snapshot, peopleSince)} + ${renderFunnel(snapshot)} ${peopleSince === null ? "" : `

People have been counted since ${escapeHtml(formatDay(peopleSince))}; event counts run from the start of the range. Until the range begins after that day, a people figure covers fewer days than the count beside it.

`} ${renderInstallConversion(snapshot)} ${renderUniquesStrip(snapshot)} @@ -748,9 +748,16 @@ function renderTimeChart( `; } -function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): string { +/** Which surface a funnel step's people are counted on, for the day their count starts. */ +const STEP_SURFACES: Record = { visited: "site", installer: "install", session: "cli", opened: "viewer" }; + +function renderFunnel(snapshot: StatsSnapshot): string { const steps = snapshot.funnel; - const since = peopleSince === null ? "" : ` since ${escapeHtml(formatDay(peopleSince))}`; + const sinceFor = (key: string): string => { + const surface = STEP_SURFACES[key]; + const since = surface === undefined ? null : peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + return since === null ? "" : ` since ${escapeHtml(formatDay(since))}`; + }; const colors = ["#9ab7e8", "#8eafff", "#819de5", "#f4bd78", "#8eafff", "#75dac2", "#d7a6ff"]; const maximum = Math.max(1, ...steps.map((step) => step.count)); return ` @@ -771,7 +778,7 @@ function renderFunnel(snapshot: StatsSnapshot, peopleSince: number | null): stri return `
${escapeHtml(step.label)} - ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${since}`} + ${integerFormatter.format(step.count)}${step.unique === null ? "" : `${integerFormatter.format(step.unique)} ${step.unique === 1 ? "person" : "people"}${sinceFor(step.key)}`}
${escapeHtml(share)} ${excluded} @@ -799,15 +806,30 @@ function renderInstallConversion(snapshot: StatsSnapshot): string { return `

${escapeHtml(verdict)} Machines are followed by address from the binary download to the first session.

`; } +/* + * What to say beside a people figure. New means first seen since the range + * began, so until a whole range has passed since this surface's people were + * first counted, everyone is new and the split says nothing: name the day it + * starts to. The 24h range counts people by UTC day, which is two of them. + */ +function peopleNote(snapshot: StatsSnapshot, surface: UniqueSurface): string { + const count = snapshot.uniques.surfaces[surface]; + const days = snapshot.range === "24h" ? " · by UTC day, so two days" : ""; + const since = peopleCountedSince(snapshot.uniques, snapshot.rangeStart, surface); + if (since === null) return `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} returning${days}`; + const knownFrom = formatDay(since + (snapshot.generatedAt - snapshot.rangeStart)); + return `counted since ${formatDay(since)}; new or returning cannot be told until ${knownFrom}${days}`; +} + function renderUniquesStrip(snapshot: StatsSnapshot): string { const people = snapshot.uniques; - const cell = (label: string, surface: keyof typeof people.surfaces): string => { + const cell = (label: string, surface: UniqueSurface): string => { const count = people.surfaces[surface]; return ` ${escapeHtml(label)} ${people.configured ? integerFormatter.format(count.unique) : "—"} - ${people.configured ? `${integerFormatter.format(count.new)} new · ${integerFormatter.format(count.returning)} back` : "not counted"} + ${people.configured ? escapeHtml(peopleNote(snapshot, surface)) : "not counted"} `; }; diff --git a/worker/analytics.ts b/worker/analytics.ts index 08e7617..7ddca32 100644 --- a/worker/analytics.ts +++ b/worker/analytics.ts @@ -303,6 +303,8 @@ export function documentTarget(pathname: string, status: number, currentVersion: export function isDocumentNavigation(request: Request): boolean { if (request.method !== "GET") return false; + /* A browser fetching a page it guesses will be wanted has not shown it to anyone. */ + if (request.headers.get("Sec-Purpose")?.includes("prefetch") || request.headers.get("Purpose") === "prefetch") return false; if (request.headers.get("Sec-Fetch-Dest") === "document") return true; return request.headers.get("Accept")?.toLowerCase().includes("text/html") ?? false; } @@ -312,11 +314,26 @@ export function binaryDownloadTarget(pathname: string): string | null { return match ? `${match[1]}-${match[2]}` : null; } +/** + * Crawlers, and monitors that say so, the site's own download checks among + * them: requests to count, never people. A monitor that says nothing looks + * like whatever library it used, which the tool pattern catches next. + */ +const CRAWLER_AGENT = /bot|crawler|spider|slurp|headless|preview|monitor|uptime|pingdom|statuscake|checkly|site24x7|synthetic|shell\.online-(?:downloads|install)-check/; + +/** + * Tools a person or a script runs: the CLI, curl and wget, PowerShell's web + * cmdlets, and the HTTP libraries scripts are written with. Not browsers, + * whatever else the agent string says; the default used to be "desktop", + * which made every Node script a person. + */ +const TOOL_AGENT = /^shell\/|curl|wget|powershell|\bnode\b|node-fetch|undici|go-http-client|python|\bjava\b|\bjava\/|okhttp|axios|libwww|httpie/; + export function classifyDevice(userAgent: string, mobileHint: string | null = null): DeviceClass { const normalized = userAgent.toLowerCase(); if (!normalized) return "unknown"; - if (/bot|crawler|spider|slurp|headless|preview/.test(normalized)) return "bot"; - if (/^shell\//.test(normalized) || /curl|wget/.test(normalized)) return "cli"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "cli"; if (mobileHint === "?1" || /iphone|ipod|android.+mobile|mobile.+android/.test(normalized)) { return "mobile"; } @@ -327,9 +344,11 @@ export function classifyDevice(userAgent: string, mobileHint: string | null = nu export function classifyClient(userAgent: string): string { const shellVersion = userAgent.match(/\bshell\/(\d{1,3}\.\d{1,3}\.\d{1,3})\b/i)?.[1]; if (shellVersion) return `shell/${shellVersion}`; - if (/curl/i.test(userAgent)) return "curl"; - if (/wget/i.test(userAgent)) return "wget"; - if (/bot|crawler|spider|slurp|headless|preview/i.test(userAgent)) return "bot"; + const normalized = userAgent.toLowerCase(); + if (/curl/.test(normalized)) return "curl"; + if (/wget/.test(normalized)) return "wget"; + if (CRAWLER_AGENT.test(normalized)) return "bot"; + if (TOOL_AGENT.test(normalized)) return "tool"; return userAgent ? "web" : "unknown"; } diff --git a/worker/stats-database.ts b/worker/stats-database.ts index 2611e80..669db83 100644 --- a/worker/stats-database.ts +++ b/worker/stats-database.ts @@ -21,6 +21,7 @@ import { type PreviousPeriodRows, type RetentionRow, type StatsSnapshotRows, + type SurfaceSinceRow, type UniqueDayRow, type UniqueSummaryRow, } from "../shared/stats-snapshot"; @@ -136,6 +137,35 @@ export function initializeStatsSchema(sql: StatsSql): void { ) `); sql.exec("CREATE INDEX IF NOT EXISTS visitors_last_day ON visitors(last_day)"); + sql.exec(` + CREATE TABLE IF NOT EXISTS dashboard_marks ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) + `); + migrateStatsData(sql); +} + +/** + * The last day machines were keyed by address and user agent; since the day + * after, by address alone. A machine row from before cannot match the same + * machine seen after, so it would sit in the cohorts as a ghost that never + * came back. + */ +export const MACHINE_KEY_DAY = Date.UTC(2026, 8, 15); + +/** + * One-time repairs to what is stored, each done once and marked as done. + * The first drops the machine rows keyed the old way; a machine seen on the + * cut-over day itself is lost with them, a few hours of history against 120 + * days of ghosts. + */ +export function migrateStatsData(sql: StatsSql): void { + const done = sql.exec<{ value: number }>("SELECT value FROM dashboard_marks WHERE name = 'machine_key'").toArray(); + if (done.length > 0) return; + sql.exec("DELETE FROM visitor_days WHERE surface IN ('cli', 'install') AND day <= ?", MACHINE_KEY_DAY); + sql.exec("DELETE FROM visitors WHERE surface IN ('cli', 'install') AND last_day <= ?", MACHINE_KEY_DAY); + sql.exec("INSERT INTO dashboard_marks (name, value) VALUES ('machine_key', 1)", ); } /** One event into its hour bucket, and its person into the day's visitors when it has one. */ @@ -228,6 +258,9 @@ export function collectStatsRows( const collectingSince = sql.exec("SELECT MIN(bucket) AS minimum FROM metric_hourly").one().minimum; /* Taken after the purge, so it is the earliest day people can still be counted from. */ const uniquesSince = sql.exec("SELECT MIN(day) AS minimum FROM visitor_days").one().minimum; + const uniquesSinceBySurface = sql.exec( + "SELECT surface, MIN(day) AS minimum FROM visitor_days GROUP BY surface ORDER BY surface", + ).toArray(); const rangeStart = statsRangeStart(range, now, collectingSince); /* Buckets are hours and events at most ten minutes ahead of this clock, so nothing sits past tomorrow. */ const rangeEnd = now + DAY_MS; @@ -255,13 +288,15 @@ export function collectStatsRows( ).toArray(), }; } + /* The charts are about people, so crawlers stay out of them; the ledger keeps every request. */ const trend = sql.exec( `SELECT bucket, event, SUM(count) AS count FROM metric_hourly WHERE bucket >= ? - AND event IN ('session_created', 'share_opened', 'collaboration_started', 'page_view') + AND event IN ('session_created', 'session_started', 'share_opened', 'collaboration_started', 'page_view', 'binary_download') + AND device != 'bot' GROUP BY bucket, event - ORDER BY bucket`, + ORDER BY bucket, event`, rangeStart, ).toArray(); const live = sql.exec( @@ -346,6 +381,7 @@ export function collectStatsRows( installConversion, uniquesConfigured, uniquesSince, + uniquesSinceBySurface, }, }; } diff --git a/worker/stats-store.test.ts b/worker/stats-store.test.ts index 3c84b6a..77abd21 100644 --- a/worker/stats-store.test.ts +++ b/worker/stats-store.test.ts @@ -40,6 +40,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "all", now, collectingSince); expect(snapshot.metrics).toMatchObject({ @@ -73,6 +74,7 @@ describe("statistics live presence", () => { installConversion: null, uniquesConfigured: false, uniquesSince: null, + uniquesSinceBySurface: [], }, "24h", now, now - 24 * 60 * 60 * 1_000); expect(snapshot.metrics.activeSessions).toBe(0); @@ -151,6 +153,12 @@ describe("people and the funnel", () => { installConversion: { installers: 12, matured: 9, started: 4 }, uniquesConfigured: true, uniquesSince: dayStart(now - 20 * DAY_MS), + /* Machines were re-keyed after visitors were first counted, so their start is later. */ + uniquesSinceBySurface: [ + { surface: "site", minimum: dayStart(now - 20 * DAY_MS) }, + { surface: "cli", minimum: dayStart(now - 2 * DAY_MS) }, + { surface: "install", minimum: dayStart(now - 2 * DAY_MS) }, + ], }; it("keeps docs, unknown paths and 404s apart and counts people beside events", () => { @@ -164,8 +172,8 @@ describe("people and the funnel", () => { ctaClicks: 12, binaryDownloads: 4, }); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50 }); - expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 400, new: 350, returning: 50, since: dayStart(now - 20 * DAY_MS) }); + expect(snapshot.uniques.surfaces.cli).toEqual({ unique: 9, new: 2, returning: 7, since: dayStart(now - 2 * DAY_MS) }); expect(snapshot.uniques.daily.map((day) => [day.site, day.cli])).toEqual([[120, 4], [140, 0]]); expect(snapshot.rates.signup).toBeCloseTo(12 / 969); expect(snapshot.breakdowns.pages.map((page) => page.label)).toEqual(["landing", "unknown_path", "docs_app", "docs", "not_found"]); @@ -310,6 +318,22 @@ describe("people and the funnel", () => { expect(buildStatsSnapshot({ ...rows, previous: null }, "all", now, now - 30 * DAY_MS).previous).toBeNull(); }); + it("draws installs and started sessions as their own lines", () => { + const hour = 60 * 60 * 1_000; + const bucket = Math.floor((now - DAY_MS) / hour) * hour; + const snapshot = buildStatsSnapshot({ + ...rows, + trend: [ + { bucket, event: "session_created", count: 4 }, + { bucket, event: "session_started", count: 3 }, + { bucket, event: "binary_download", count: 2 }, + { bucket, event: "page_view", count: 9 }, + ], + }, "7d", now, rangeStart); + const point = snapshot.trend.find((candidate) => candidate.at === Math.floor(bucket / (6 * hour)) * 6 * hour); + expect(point).toMatchObject({ sessions: 4, started: 3, installs: 2, pageViews: 9, shares: 0, collaborations: 0 }); + }); + it("follows machines from the installer to a first session", () => { const snapshot = buildStatsSnapshot(rows, "7d", now, rangeStart); expect(snapshot.installConversion).toEqual({ installers: 12, matured: 9, started: 4 }); @@ -323,7 +347,7 @@ describe("people and the funnel", () => { expect(snapshot.uniques.since).toBeNull(); expect(snapshot.installConversion).toBeNull(); expect(snapshot.funnel[0].unique).toBeNull(); - expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0 }); + expect(snapshot.uniques.surfaces.site).toEqual({ unique: 0, new: 0, returning: 0, since: null }); }); /* @@ -342,6 +366,16 @@ describe("people and the funnel", () => { expect(partial.uniques.since).toBe(yesterday); expect(peopleCountedSince(partial.uniques, partial.rangeStart)).toBe(yesterday); + /* Each surface starts on its own day: visitors cover the week, machines do not. */ + expect(covered.uniques.surfaces.site.since).toBe(dayStart(now - 20 * DAY_MS)); + expect(covered.uniques.surfaces.cli.since).toBe(dayStart(now - 2 * DAY_MS)); + expect(covered.uniques.surfaces.viewer.since).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "site")).toBeNull(); + expect(peopleCountedSince(covered.uniques, rangeStart, "cli")).toBe(dayStart(now - 2 * DAY_MS)); + expect(peopleCountedSince(covered.uniques, rangeStart, "viewer")).toBeNull(); + const unconfigured = buildStatsSnapshot({ ...rows, uniquesConfigured: false }, "7d", now, rangeStart); + expect(unconfigured.uniques.surfaces.cli.since).toBeNull(); + /* People are kept by day, so a range that starts inside their first day is covered. */ expect(peopleCountedSince({ configured: true, since: dayStart(rangeStart) }, rangeStart + 60_000)).toBeNull(); expect(peopleCountedSince({ configured: false, since: yesterday }, now - 30 * DAY_MS)).toBeNull();