feat(telemetry): qualify usage diagnostics and installation return cohorts

apps/presentation/dashboard/src/data/chat.ts

@@ -2220,6 +2220,11 @@ const usageStatisticsSchema = z.object({
   notice: z.object({ version: z.number(), endpoint: z.string(), policy: z.string() }),
   automatic_notice_required: z.boolean(),
   next_payload: z.unknown(), aggregate_preview: z.unknown(), goal_preview: z.unknown(),
+  diagnostic_preview: z.unknown().optional(), diagnostic_dropped: z.number().optional(),
+  identity_scope: z.string().optional(), delivery_history: z.array(z.object({
+    day: z.string(), channel: z.enum(["heartbeat", "cli", "goal"]), rows: z.number(),
+    status: z.enum(["accepted", "rejected", "unavailable"]),
+  })).optional(),
 });
 export type UsageStatistics = z.infer<typeof usageStatisticsSchema>;
 export async function usageStatistics(enabled?: boolean): Promise<UsageStatistics> {

apps/presentation/dashboard/src/features/personal-workspace/usage-statistics-notice.tsx

@@ -73,11 +73,11 @@ export function UsageStatisticsNotice({ onDetails }: { onDetails: () => void })
           : state.automatic_notice_required ? (zh ? "基础使用统计 · 告知后自动开启" : "Basic usage statistics · enabled after this notice")
             : (zh ? "基础使用统计当前不发送，请查看详情" : "Basic usage statistics are not sending; see details")}</strong>
       <p>{zh
-        ? "用于改进平台支持与使用体验。发送随机安装标识和环境信息，以及另行汇总的 CLI 使用次数、结果、耗时和 Goal 时长区间；不采集对话、代码、路径或命令参数。可随时关闭。"
-        : "Helps improve platform support and usage. Sends a random installation ID and environment information, plus separate CLI usage, result, timing and Goal duration summaries. No conversations, code, paths or command arguments. You can turn it off at any time."}</p>
+        ? "用于改进平台支持与使用体验。发送随机安装标识和环境信息，另行汇总 CLI 子操作、版本/活动日期、结果/耗时与回执信号，以及 Goal 时长。环境类型自愿声明，默认未知；不采集对话、代码、路径或参数值。可随时关闭。"
+        : "Helps improve platform support and usage. Sends a random installation ID and environment information, plus separate CLI sub-operation, release/activity day, result/timing, receipt signals and Goal duration summaries. Deployment context is voluntary, unknown by default. No conversations, code, paths or argument values. You can turn it off at any time."}</p>
       <p>{zh
-        ? "首个已测量的 CLI 结果立即上报，后续由使用活动触发，至少间隔 15 分钟发送一批。CLI 汇总不含安装标识；更频繁的请求仍可能让网络服务通过 IP 和请求时间关联活动。"
-        : "The first measured CLI result is sent immediately; later activity sends buffered counts at most once every 15 minutes. CLI summaries contain no installation ID; more frequent requests may still let network services correlate activity using IP addresses and request timing."}</p>
+        ? "首个已测量的 CLI 结果立即上报，后续由使用活动触发，至少间隔 15 分钟发送一批。CLI 汇总不含安装标识。"
+        : "The first measured CLI result is sent immediately; later activity sends buffered counts at most once every 15 minutes. CLI summaries contain no installation ID."}</p>
       <p className="personal-usage-recipient">{zh ? "接收方：" : "Recipient: "}{state.endpoint}</p>
       {error ? <p role="alert">{zh ? "设置未能保存，请打开详情重试。" : "Could not save this setting. Open details to retry."}</p> : null}
     </div>

apps/presentation/dashboard/src/features/personal-workspace/usage-statistics-settings.tsx

@@ -25,7 +25,7 @@ export function UsageStatisticsSettings() {
     <p>{zh
       ? "用于决定平台支持和改进命令体验。每天向 LoopX 的 Cloudflare 收集服务发送随机安装标识、版本、系统、CPU 架构、Python 版本和安装渠道；固定的 CLI 功能、结果、耗时区间和错误类别在本机汇总，不带安装标识。首个可采集的命令结果立即尝试发送，之后有活动时每隔至少 15 分钟发送一批。"
       : "Helps prioritize platform support and CLI improvements. A daily heartbeat sends a random installation ID, version, OS, CPU architecture, Python version and install channel to the LoopX Cloudflare collector. Fixed CLI feature, result, duration and error counts are aggregated locally without the ID. The first measured result attempts a send immediately; later activity sends batches at least 15 minutes apart."}</p>
-    <p>{zh ? "不采集提示词、代码、路径、命令参数、Goal 内容或原始错误。当前功能计数仅覆盖 CLI；命令成功不等于 Goal 完成。" : "No prompts, code, paths, arguments, Goal contents or raw errors. Feature counts currently cover CLI only; command success is not Goal completion."}</p>
+    <p>{zh ? "新增固定子操作、版本、UTC 活动日期、阻塞/失败分类和已回读的生命周期信号。运行环境类型仅由 LOOPX_USAGE_CONTEXT 自愿声明，默认 unknown，不推断个人或企业。不会上传提示词、代码、路径、参数值、Goal 内容或原始错误。命令成功不等于 Goal 完成。" : "Adds fixed sub-operations, release version, UTC activity date, blocked/failure classes and receipt-backed lifecycle signals. Deployment context is voluntary via LOOPX_USAGE_CONTEXT, unknown by default, never inferred. No prompts, code, paths, argument values, Goal contents or raw errors. Command success is not Goal completion."}</p>
     <p>{zh ? "按天分别汇总所有 Host 的 quota→spend 推进周期、已绑定 Codex 任务的本地轮次时间、受管 Turn 与普通 Goal 对话的 Host 调用时间。上传固定 Host 类别及跨度/时长区间，不上传会话内容、Goal 或安装标识。三种口径重叠，不能相加；可能漏计，不代表完成、CPU 用时或计费。" : "Daily, separate span/duration buckets for all Hosts using quota→spend, local timing events from bound Codex tasks, and direct Host calls in managed Turns and regular owner Goal chat. Sends fixed Host categories, never session contents, Goal or installation IDs. The three overlapping populations cannot be added; partial observations are not completion, CPU time or billing."}</p>
     {state ? <>
       <label><input type="checkbox" checked={state.consent === "enabled" || (state.consent === "default" && !state.notice_required)} disabled={busy}
@@ -35,7 +35,7 @@ export function UsageStatisticsSettings() {
           : (zh ? `当前不发送：${({disabled:"已关闭",CI:"CI 环境",DO_NOT_TRACK:"请勿追踪开关",LOOPX_USAGE_PING:"环境变量已关闭",consent_required:"需要明确同意",invalid_policy:"策略配置无效",invalid_endpoint:"接收地址无效",notice_required:"需要重新告知"} as Record<string,string>)[state.blocked_by ?? ""] ?? "请检查配置"}` : `Not sending: ${state.blocked_by}`)}</p>
       {state.notice_required && state.consent !== "disabled" ? <button className="personal-secondary-action" type="button" disabled={busy} onClick={() => void update(true)}>{zh ? "已了解，启用统计" : "Understood, enable statistics"}</button> : null}
       <p>{zh ? "接收地址：" : "Recipient: "}{state.endpoint ?? (zh ? "未配置" : "Not configured")}</p>
-      <details><summary>{zh ? "查看待发送数据" : "Preview outgoing data"}</summary><pre>{JSON.stringify({ heartbeat: state.next_payload, aggregate: state.aggregate_preview, goals: state.goal_preview }, null, 2)}</pre></details>
+      <details><summary>{zh ? "查看待发送数据与本地发送摘要" : "Preview outgoing data and local delivery summaries"}</summary><pre>{JSON.stringify({ heartbeat: state.next_payload, aggregate: state.aggregate_preview, diagnostics: state.diagnostic_preview, goals: state.goal_preview, identity_scope: state.identity_scope, dropped: state.diagnostic_dropped, delivery_history: state.delivery_history }, null, 2)}</pre></details>
     </> : null}
     {error ? <p role="alert">{zh ? "无法读取或保存；请用终端检查：" : "Could not read or save; inspect in terminal: "}<code>loopx usage-ping status</code></p> : null}
     <p><code>loopx usage-ping disable</code> · <code>LOOPX_USAGE_PING=0</code></p>

apps/usage-collector/README.md

@@ -2,7 +2,8 @@
 
 Cloudflare Worker + D1 for [basic usage statistics](../../docs/reference/usage-ping.md).
 The TypeScript client/collector allowlist lives in
-`loopx/control_plane/runtime/usage_statistics_contract.ts`.
+`loopx/control_plane/runtime/usage_statistics_contract.ts` and
+`usage_statistics_diagnostics.ts`.
 
 | Endpoint | Contract |
 |---|---|
@@ -12,6 +13,8 @@ The TypeScript client/collector allowlist lives in
 | `GET /v1/goal-stats` | Independent 30-day duration histograms; cells below 5 omitted |
 | `GET /v0/stats` | Deduplicated active/new installations, including retained v0 clients; version/OS/CPU/channel breakdown |
 | `GET /v1/aggregate-stats` | Independent 30-day feature/result/duration/error totals; cells below 5 omitted |
+| `GET /v1/diagnostic-stats` | Independent versioned CLI/lifecycle marginals; cells below 5 omitted |
+| `GET /v1/adoption-stats` | Mature 1/7/30-day installation return cohorts and 30-day activity-day buckets |
 | `POST /v0/ping` | Retained six-field opt-in client contract; no new default-on clients use this route |
 
 Heartbeats are deduplicated by installation/day and retained 400 days.
@@ -24,6 +27,16 @@ adds each delta without requiring a schema migration. Delivery cadence does not
 add a version or installation join key. Counters are estimates, not people,
 accepted Goal outcomes or billing records.
 
+The aggregate endpoint also accepts `loopx_usage_diagnostics_v1`.
+`diagnostic_counts` stores fixed feature/sub-operation/result/reason/duration,
+numeric version, UTC activity date, receipt date, voluntary deployment context
+and receipt-backed lifecycle signal. It has no installation join key or raw
+request rows. Activity dates older than seven days or in the future are rejected.
+Keep 30 receipt days; legacy counters are never backfilled or reattributed.
+Public endpoints expose independent marginals, not multi-dimensional histories.
+Return cohorts count installation state, not people; `within_Nd` means any later
+heartbeat within a mature N-day window, not exact day-N retention.
+
 Neither handler reads/stores IP, user agent or Cloudflare request metadata.
 The template disables Worker observability; Cloudflare still handles network
 metadata. Do not describe the identified heartbeat as fully anonymous, or the
@@ -61,6 +74,12 @@ must never be summed. The unreleased Goal payload requires measurement/Host
 labels and uses `duration` instead of `execution`.
 A Worker rollback can leave that additive table intact.
 
+Before releasing notice-v5 clients, apply `0004-diagnostics.sql` with D1
+migrations and deploy the updated Worker. It preserves existing installs,
+pings and legacy counters. An older Worker rejects new diagnostics; loss is
+not retried. Roll back the Worker/client without dropping the additive table.
+Merging this code does not deploy the collector.
+
 Qualify `/v1/ping`, `/v1/aggregate`, `/v1/goals`, all stats endpoints, and invalid-field/size
 rejections on a separate database first. Deploy the collector before releasing
 the new client default: the v0-only Worker does not accept v1 requests. Server
@@ -83,4 +102,4 @@ node --no-warnings --experimental-strip-types --test tests/control_plane_ts/usag
 ```
 
 Run from repository root. The collector suite executes actual SQL in SQLite,
-including both additive migrations; no production telemetry is needed for these tests.
+including additive migrations and mature/suppressed cohorts; no production telemetry is needed for these tests.

apps/usage-collector/migrations/0004-diagnostics.sql

@@ -0,0 +1,8 @@
+-- Additive: retain v0/v1 history and rollback tables, never relabel old counts.
+CREATE TABLE IF NOT EXISTS diagnostic_counts (
+  receipt_day TEXT NOT NULL, activity_day TEXT NOT NULL, version TEXT NOT NULL,
+  context TEXT NOT NULL, feature TEXT NOT NULL, operation TEXT NOT NULL,
+  outcome TEXT NOT NULL, error TEXT NOT NULL, duration TEXT NOT NULL,
+  signal TEXT NOT NULL, count INTEGER NOT NULL,
+  PRIMARY KEY (receipt_day, activity_day, version, context, feature, operation, outcome, error, duration, signal)
+);

apps/usage-collector/schema.sql

@@ -39,3 +39,11 @@ CREATE TABLE IF NOT EXISTS goal_duration_counts (
   span TEXT NOT NULL, duration TEXT NOT NULL, count INTEGER NOT NULL,
   PRIMARY KEY (day, measurement, host, span, duration)
 );
+
+CREATE TABLE IF NOT EXISTS diagnostic_counts (
+  receipt_day TEXT NOT NULL, activity_day TEXT NOT NULL, version TEXT NOT NULL,
+  context TEXT NOT NULL, feature TEXT NOT NULL, operation TEXT NOT NULL,
+  outcome TEXT NOT NULL, error TEXT NOT NULL, duration TEXT NOT NULL,
+  signal TEXT NOT NULL, count INTEGER NOT NULL,
+  PRIMARY KEY (receipt_day, activity_day, version, context, feature, operation, outcome, error, duration, signal)
+);

apps/usage-collector/src/basic-usage.ts

@@ -1,10 +1,29 @@
 /** Aggregate storage has no foreign key or identifier linking it to installations. */
 import { validAggregate, validPing } from "../../../loopx/control_plane/runtime/usage_statistics_contract.ts";
 import type { Aggregate } from "../../../loopx/control_plane/runtime/usage_statistics_contract.ts";
+import { validDiagnostics } from "../../../loopx/control_plane/runtime/usage_statistics_diagnostics.ts";
+import type { DiagnosticAggregate } from "../../../loopx/control_plane/runtime/usage_statistics_diagnostics.ts";
 
 type Statement = { bind(...args: unknown[]): Statement; all(): Promise<{ results: Record<string, unknown>[] }> };
 type Database = { prepare(sql: string): Statement; batch(statements: Statement[]): Promise<unknown> };
 export { validAggregate, validPing };
+export { validDiagnostics };
+export async function recordDiagnostics(db: Database, value: DiagnosticAggregate, receiptDay: string) {
+  await db.batch(value.counters.map(row => db.prepare(
+    "INSERT INTO diagnostic_counts (receipt_day, activity_day, version, context, feature, operation, outcome, error, duration, signal, count) " +
+    "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) " +
+    "ON CONFLICT (receipt_day, activity_day, version, context, feature, operation, outcome, error, duration, signal) DO UPDATE SET count = count + excluded.count",
+  ).bind(receiptDay, row.activity_day, row.version, row.context, row.feature, row.operation, row.outcome, row.error, row.duration, row.signal, row.count)));
+}
+export async function diagnosticStats(db: Database, since: string) {
+  const totals: Record<string, Record<string, number>> = {};
+  for (const column of ["feature", "operation", "outcome", "error", "duration", "version", "context", "signal"]) {
+    const rows = await db.prepare(`SELECT ${column} AS key, SUM(count) AS n FROM diagnostic_counts WHERE receipt_day >= ?1 GROUP BY ${column}`).bind(since).all();
+    totals[column] = {};
+    for (const row of rows.results) if (Number(row.n) >= 5) totals[column][String(row.key)] = Number(row.n);
+  }
+  return { schema: "loopx_usage_diagnostic_stats_v1", definition: "Independent marginal totals from versioned, lossy observations in the last 30 receipt days. Signals describe observed transitions, not unique Goals, people, independent outcome quality or billing. Context is voluntary self-report, never inferred. Cells below 5 omitted.", totals };
+}
 export async function recordAggregate(db: Database, value: Aggregate, day: string) {
   await db.batch(value.counters.map(row => db.prepare(
     "INSERT INTO usage_counts (day, feature, outcome, duration, error, count) VALUES (?1, ?2, ?3, ?4, ?5, ?6) " +

apps/usage-collector/src/collector.js

@@ -1,4 +1,5 @@
 import { validAggregate, validPing, recordAggregate, aggregateStats, validGoalAggregate, recordGoals, goalStats } from "./basic-usage.ts";
+import { validDiagnostics, recordDiagnostics, diagnosticStats } from "./basic-usage.ts";
 // Pure request handling for the LoopX usage collector. worker.js binds it to
 // Cloudflare; tests bind it to an in-memory database.
 
@@ -138,13 +139,45 @@ export async function purge(db, day) {
     db.prepare("DELETE FROM goal_duration_counts WHERE day < ?1").bind(shiftDays(day, -30)),
     db.prepare("DELETE FROM goal_usage_counts WHERE day < ?1").bind(shiftDays(day, -30)),
     db.prepare("DELETE FROM usage_counts WHERE day < ?1").bind(shiftDays(day, -30)),
+    db.prepare("DELETE FROM diagnostic_counts WHERE receipt_day < ?1").bind(shiftDays(day, -30)),
     db.prepare("DELETE FROM installs WHERE install_id NOT IN (SELECT DISTINCT install_id FROM pings)"),
   ]);
 }
 
+export async function adoptionStats(db, day) {
+  const cohorts = {};
+  for (const horizon of [1, 7, 30]) {
+    const row = await db.prepare(
+      "SELECT COUNT(*) AS eligible, COALESCE(SUM(EXISTS(SELECT 1 FROM pings p WHERE p.install_id = installs.install_id " +
+      "AND p.day > installs.first_day AND p.day <= date(installs.first_day, ?1))), 0) AS returned " +
+      "FROM installs WHERE first_day BETWEEN ?2 AND ?3",
+    ).bind(`+${horizon} days`, shiftDays(day, -horizon - 29), shiftDays(day, -horizon)).first();
+    // Do not publish a rate whose numerator or denominator is a small cell.
+    cohorts[`within_${horizon}d`] = Number(row?.eligible) >= MIN_BUCKET && Number(row?.returned) >= MIN_BUCKET
+      && (Number(row.eligible) === Number(row.returned) || Number(row.eligible) - Number(row.returned) >= MIN_BUCKET)
+      ? { eligible: Number(row.eligible), returned: Number(row.returned) } : null;
+  }
+  const rows = await db.prepare(
+    "SELECT CASE WHEN days = 1 THEN '1' WHEN days <= 3 THEN '2_3' WHEN days <= 7 THEN '4_7' " +
+    "WHEN days <= 14 THEN '8_14' ELSE '15_30' END AS key, COUNT(*) AS installs FROM " +
+    "(SELECT install_id, COUNT(*) AS days FROM pings WHERE day BETWEEN ?1 AND ?2 GROUP BY install_id) GROUP BY key",
+  ).bind(shiftDays(day, -29), day).all();
+  return { schema: "loopx_installation_return_stats_v1", generated_on: day,
+    definition: "Each cohort contains 30 first-seen UTC dates ending at least N days ago. Return means a heartbeat on a later day within N days, not exact day-N retention. Persistent installation state is not a person or organization; resets and ephemeral hosts affect counts. Small numerator/denominator or nonzero complement cohorts omitted. Horizon populations overlap.",
+    cohorts, active_day_distribution: suppressSmall(rows.results) };
+}
+
 export async function handle(request, db, now = new Date()) {
   const url = new URL(request.url);
   const day = utcDay(now);
+  if (url.pathname === "/v1/adoption-stats") {
+    if (request.method !== "GET") return json({ error: "method not allowed" }, 405);
+    return json(await adoptionStats(db, day), 200, { "cache-control": "public, max-age=3600" });
+  }
+  if (url.pathname === "/v1/diagnostic-stats") {
+    if (request.method !== "GET") return json({ error: "method not allowed" }, 405);
+    return json(await diagnosticStats(db, shiftDays(day, -29)), 200, { "cache-control": "public, max-age=3600" });
+  }
   if (url.pathname === "/v1/goal-stats") {
     if (request.method !== "GET") return json({ error: "method not allowed" }, 405);
     return json(await goalStats(db, shiftDays(day, -29)), 200, { "cache-control": "public, max-age=3600" });
@@ -187,6 +220,11 @@ export async function handle(request, db, now = new Date()) {
       return new Response(null, { status: 204 });
     }
     if (url.pathname === "/v1/aggregate") {
+      if (validDiagnostics(parsed)) {
+        if (parsed.counters.some(row => row.activity_day > day || row.activity_day < shiftDays(day, -7))) return json({ error: "activity day outside retention window" }, 400);
+        await recordDiagnostics(db, parsed, day);
+        return new Response(null, { status: 204 });
+      }
       if (!validAggregate(parsed)) return json({ error: "invalid aggregate" }, 400);
       await recordAggregate(db, parsed, day);
       return new Response(null, { status: 204 });