From 252519e684cc0d28e61a532f5eca175e1b8a21bc Mon Sep 17 00:00:00 2001 From: zJay26 <120452926+zJay26@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:24:46 +0800 Subject: [PATCH] fix: reevaluate mixed cumulative counters for v2.6.3 --- README.en.md | 6 +- README.md | 6 +- docs/accounting-v2.6.md | 12 +- docs/releases/v2.6.3.md | 29 +++++ internal/app/app.go | 2 +- internal/server/accounting_migration_test.go | 83 +++++++++++++ internal/store/accounting_migration_test.go | 97 +++++++++++++++ internal/store/store.go | 2 +- internal/store/store_test.go | 4 +- internal/store/transactions.go | 20 ++++ internal/usage/accounting.go | 22 +++- internal/usage/accounting_test.go | 118 +++++++++++++++++++ internal/usage/scanner.go | 16 ++- internal/web/static/i18n.js | 6 +- scripts/build.ps1 | 2 +- scripts/build.sh | 2 +- scripts/demo-api.js | 4 +- scripts/mock-dashboard.mjs | 2 +- 18 files changed, 406 insertions(+), 27 deletions(-) create mode 100644 docs/releases/v2.6.3.md create mode 100644 internal/server/accounting_migration_test.go create mode 100644 internal/store/accounting_migration_test.go diff --git a/README.en.md b/README.en.md index c534f2e..46d931c 100644 --- a/README.en.md +++ b/README.en.md @@ -29,7 +29,7 @@ All statistics stay on the current computer. codex-usage never stores prompts, r ## Install directly -This README covers stable **[v2.6.2](https://github.com/zJay26/codex-usage/releases/tag/v2.6.2)**; see the [release notes](docs/releases/v2.6.2.md) for changes and upgrade boundaries. Download links below always resolve to the latest stable release. +This README covers stable **[v2.6.3](https://github.com/zJay26/codex-usage/releases/tag/v2.6.3)**; see the [release notes](docs/releases/v2.6.3.md) for changes and upgrade boundaries. Download links below always resolve to the latest stable release. | System | amd64 / x64 | arm64 | |---|---|---| @@ -92,7 +92,7 @@ Starting with **v2.5.0**, the application checks GitHub for the latest stable re **Software updates** lets you set the update download directory, open the folder, or copy its path. The default is `codex-usage` inside your user Downloads directory. Version folders retain the release binaries, and the UI shows the full path of the last download. Directory changes only affect future downloads; existing files stay in place and database backups remain in the local state directory. -Upgrading to v2.6 preserves existing statistics and applies accounting fixes to newly read records. To correct earlier undercounts, back up the state and verify retained JSONL coverage before explicitly running `scan --rebuild`; rebuilding cannot recover history whose source files have been deleted. +**v2.6.3 fixes repeated accounting of old usage in mixed cumulative records processed by v2.6.0–v2.6.2.** After upgrading from those versions, existing statistics are retained and flagged for review; incremental scans pause. Back up the state and verify source JSONL coverage, then select **Rescan → Approve and rebuild** or explicitly run `codex-usage scan --rebuild` to correct stored history. Rebuilding cannot recover deleted source files, and upgrading the binary alone does not correct old totals. ## What you can see @@ -165,7 +165,7 @@ flowchart LR The tool reads the current machine's `CODEX_HOME`. It first discovers canonical session metadata from the Codex state database, then streams JSONL files under `sessions/` and `archived_sessions/`. -Token records can be cumulative per session or per turn. Persisted scope and the last token turn distinguish them: total equal to last at a new turn establishes turn scope whether it is smaller than, equal to, or larger than the preceding total. Legacy session counters retain their cross-turn baseline. At **each** `token_count` record, the scanner computes the delta against the corresponding session or turn baseline and assigns it to the record timestamp's accounting calendar day. It never moves an entire multi-day session to the session's latest update date. Stable event IDs and cursors keep repeated scans idempotent. Large prompt, response, reasoning, and tool-output records are skipped without loading the entire line into memory or writing content to the database. +Token counters can continue across turns or reset at a turn boundary. Each boundary is evaluated separately: `total_token_usage` equal to `last_token_usage` at a new turn starts a fresh baseline, whether smaller than, equal to, or larger than the preceding total. A later continuation or repeated snapshot retains its baseline; one reset never classifies all future turns. When missing snapshots prevent distinguishing continuation from another reset, the scanner conservatively differences the counter and flags potentially incomplete usage. At **each** `token_count` record, the delta is assigned to the record timestamp's accounting calendar day. It never moves an entire multi-day session to the session's latest update date. Stable event IDs and cursors keep repeated scans idempotent. Large prompt, response, reasoning, and tool-output records are skipped without loading the entire line into memory or writing content to the database. The Codex state database is used only to discover rollout paths and enrich titles, projects, and other metadata. Its `tokens_used` value never changes token totals. OpenAI's [`account/usage/read`](https://learn.chatgpt.com/docs/app-server#7-token-usage-chatgpt) is service-backed account activity; this tool counts only current-machine local JSONL, so the scopes differ. diff --git a/README.md b/README.md index 5b088f5..2fec6ec 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ ## 直接安装 -本文对应稳定版 **[v2.6.2](https://github.com/zJay26/codex-usage/releases/tag/v2.6.2)**；变更和升级边界见 [发布说明](docs/releases/v2.6.2.md)。以下下载链接始终指向最新稳定版。 +本文对应稳定版 **[v2.6.3](https://github.com/zJay26/codex-usage/releases/tag/v2.6.3)**；变更和升级边界见 [发布说明](docs/releases/v2.6.3.md)。以下下载链接始终指向最新稳定版。 | 系统 | amd64 / x64 | arm64 | |---|---|---| @@ -92,7 +92,7 @@ chmod +x codex-usage “软件更新”中可以设置**更新包下载目录**，并打开文件夹或复制路径。默认保存到用户下载目录下的 `codex-usage` 文件夹；版本文件夹内保留正式安装包，界面显示最近下载的完整路径。保存新目录只影响后续下载，已有文件留在原处；数据库备份仍保留在本机状态目录。 -v2.6 系列升级会保留现有统计，计量修复适用于新读入的记录。若要修正旧版本已经漏计的历史，需先备份并核对仍保留的 JSONL，再显式执行 `scan --rebuild`；源文件已删除的历史无法由重建恢复。 +**v2.6.3 修复了 v2.6.0–v2.6.2 在混合累计记录中重复计算旧用量的问题。** 从这些版本升级后，已有统计会保留并标记为需要核对，增量扫描暂停。先备份并确认源 JSONL 仍齐全，再点击“重新扫描”并选择“同意并重建”，或显式执行 `codex-usage scan --rebuild`，修正已入库的错误历史。源文件已删除的历史无法由重建恢复；只升级程序不会自动修正旧账。 ## 你能看到什么 @@ -165,7 +165,7 @@ flowchart LR 程序只读当前电脑的 `CODEX_HOME`。它优先从 Codex 状态库取得 session 路径、项目和 Thread 信息，再流式读取 `sessions/` 与 `archived_sessions/` 中的 JSONL。 -Token 记录可能按 Session 累计，也可能按 Turn 累计。扫描器保存计量范围与上次 Token 所属 Turn；新 Turn 的 `total_token_usage` 与 `last_token_usage` 相等时建立 Turn 范围，数值小于、等于或大于上一 Turn 均重新起算。旧版 Session 累计仍保留跨 Turn 基线。程序在**每一条** `token_count` 记录处按对应 Session 或 Turn 的累计基线计算增量，归到该条记录时间戳对应的计量自然日；不会按 session 的最后更新时间把整段历史塞到同一天。重复扫描仍由稳定事件 ID 与游标去重。超大的 prompt、回复和工具输出记录会被跳过，不会整行载入内存，也不会写进数据库。 +Token 记录可能跨 Turn 连续累计，也可能在某个 Turn 重置。扫描器逐个边界核对：新 Turn 的 `total_token_usage` 与 `last_token_usage` 相等时重新起算，支持小于、等于或大于上一累计值；后续 Turn 若继续累计或重发旧快照，则保留原基线。一次重置不会决定后续所有 Turn 的计量方式。缺少快照而无法区分连续累计与再次重置时，会保守按差值计量并提示可能缺少用量。程序在**每一条** `token_count` 记录处计算增量，归到该条记录时间戳对应的计量自然日；不会按 session 的最后更新时间把整段历史塞到同一天。重复扫描仍由稳定事件 ID 与游标去重。超大的 prompt、回复和工具输出记录会被跳过，不会整行载入内存，也不会写进数据库。 Codex 状态库只用于发现 rollout 路径并补充标题、项目等 metadata；其中的 `tokens_used` 不参与 Token 总量。OpenAI 的 [`account/usage/read`](https://learn.chatgpt.com/docs/app-server#7-token-usage-chatgpt) 是服务端账号 Token 活动；本工具只统计当前电脑的本地 JSONL，两者范围不同。 diff --git a/docs/accounting-v2.6.md b/docs/accounting-v2.6.md index 9727921..61beaab 100644 --- a/docs/accounting-v2.6.md +++ b/docs/accounting-v2.6.md @@ -2,7 +2,9 @@ ## Counter scope and safe ingestion -The persisted accounting state distinguishes an unknown/session counter from a turn counter. A changed `turn_id` alone does not reset legacy session totals. A new turn whose complete `total_token_usage` equals `last_token_usage` establishes turn scope, including totals smaller than, equal to, or larger than the previous turn. Subsequent snapshots of that turn contribute only their delta. Missing subset fields retain the existing classification only within the same counter scope. +As of v2.6.3, every new turn is evaluated independently. A changed `turn_id` alone never resets cumulative totals, even after an earlier reset. A new turn whose complete `total_token_usage` equals `last_token_usage` establishes a fresh baseline, including totals smaller than, equal to, or larger than the preceding total. A later turn that continues the same series, or repeats an unchanged snapshot, retains its baseline. The duplicate path persists the new interpretation so the next incremental scan cannot inherit stale turn scope. Same-total classification corrections remain attached to the original usage. + +After a reset, a later boundary may lack enough snapshots to prove whether it continued or reset again. Positive monotonic differences are retained conservatively and marked `gap_fallback` with `cumulative_boundary_unverified`; the previous scope is never used as proof of another reset. Missing subset fields retain the existing classification only within the same counter scope. Turn-aware event identities also deduplicate copied rollouts. For proven turn counters created after migration, each file transaction checks the turn's existing ledger before reading its tail; a restored file that starts midway through a turn adds only the previously unseen increment. Session-cumulative writers keep the existing high-water behavior. Physical ownership remains the first `session_meta`; fork replay is excluded before accounting. Classification corrections for turn-scoped counters stay inside that turn, including fragments restored to another physical file. @@ -12,9 +14,13 @@ This transaction is per physical file, so a large initial import can delay anoth ## Upgrade boundary -Schema v9 is additive. Opening a v2.5.0 database preserves token amounts, timestamps, pricing classification, original local date labels and existing cursors. It adds accounting state, parent relationships, a persisted time zone and UTC hour identities. The Dashboard shows that pre-upgrade history has been retained. Metadata can be backfilled from retained sources without rebuilding token history. +**v2.6.0–v2.6.2 contain a mixed-counter regression.** After one reset, later turns could count the full previous cumulative amount again, including unchanged snapshots. Zero warnings and idempotent repeated scans did not establish accuracy for those versions. v2.6.3 corrects this interpretation and upgrades the schema to v10. + +Opening a schema-v9 database with existing JSONL events preserves the ledger and cursors, immediately records a rebuild warning, and pauses incremental ingestion until an explicit rebuild. The Dashboard's **Rescan → Approve and rebuild** flow and CLI `scan --rebuild` recalculate retained sources with the corrected parser. Merely updating the program does not repair existing event deltas. Empty databases need no historical repair. + +Upgrades directly from v2.5.0 retain the earlier additive migration: token amounts, timestamps, pricing classification, original local date labels and cursors remain intact. Accounting state, parent relationships, a persisted time zone and UTC hour identities are added. Metadata can be backfilled from retained sources without rebuilding token history. -The new parser applies to newly read records. It does not silently recalculate previously undercounted turns. Explicit `codex-usage scan --rebuild` recalculates only retained JSONL files and may lose entries whose original files have been deleted. Back up the state and verify source coverage first. A pre-upgrade session restored into a new physical file requests a rebuild when its old event identities cannot safely prove replay ownership. +Explicit `codex-usage scan --rebuild` recalculates only retained JSONL files and may lose entries whose original files have been deleted. Back up the state and verify source coverage first. A pre-upgrade session restored into a new physical file also requests a rebuild when its old event identities cannot safely prove replay ownership. ## Query consistency and performance diff --git a/docs/releases/v2.6.3.md b/docs/releases/v2.6.3.md new file mode 100644 index 0000000..e201610 --- /dev/null +++ b/docs/releases/v2.6.3.md @@ -0,0 +1,29 @@ +# codex-usage v2.6.3 + +修复 v2.6.0–v2.6.2 在历史重扫中可能大幅多计 Token 的累计范围回归。 + +- **逐个 Turn 核对累计边界**：一次重置不再使后续所有 Turn 都从零计数；连续累计和新 Turn 重发的旧快照均保留正确基线。`100/100 → 20/20 → 50/30`（累计值／本次增量）现在合计 150，修复旧版算成 170 的问题。 +- **跨扫描一致性**：即使记录没有新增 Token，也保存其累计解释，避免下一次扫描重新继承旧范围；分类修正仍回到原有事件。 +- **明确不确定边界**：重置后若缺少快照，无法区分连续累计与再次重置，保守按差值计量并提示可能缺少用量，不再仅凭旧范围推断。 +- **旧账修复提示**：schema v9 的现有统计升级到 v10 后保留事件和游标、立即显示核对提示并暂停增量扫描，等待显式重建。空数据库无需修复；从 v2.5.0 直接升级保留原有迁移行为。 +- 更新中英文 README、重建对话框和计量技术说明，保留 v2.6.2 的更新包下载目录设置。 + +**升级后修正历史：** 先备份并确认源 JSONL 仍齐全，再在 Dashboard 点击“重新扫描 → 同意并重建”，或运行 `codex-usage scan --rebuild`。仅更新程序不会自动消除已入库的多计记录；已删除源文件的历史无法通过重建恢复。 + +**验证：** 固定同一时点的 766 个 rollout，提取计量所需的数值、metadata 和 Turn 边界后在隔离目录重建；742 个有用量会话与逐会话参考值完全匹配，总量由 19,646,071,899 修正到 14,941,760,325，消除 23 个会话的 4,704,311,574 重复 Token。独立核对覆盖了变化会话的原始记录，未变化会话沿用快照参考值；这不是所有历史的独立审计或账单验证。再次扫描新增事件为 0，总量不变。 + +回归覆盖真实数值向量、混合累计、重复快照、分类修正、扫描重启、缺失快照、旧库保留和 API 确认重建。完整平台及 Dashboard 检查通过后发布六个平台程序与 `SHA256SUMS`。 + +--- + +Fix a counter-scope regression in v2.6.0–v2.6.2 that could substantially overcount tokens during historical rescans. + +- Evaluate each turn boundary independently. One reset no longer causes every later turn to count the entire cumulative total again. Continuations and repeated old snapshots retain their baseline. The `total/last` sequence `100/100 → 20/20 → 50/30` now correctly totals 150 instead of 170. +- Persist the interpretation of duplicate snapshots across scans and keep classification corrections attached to the original usage. +- Mark incomplete boundary evidence conservatively instead of inferring another reset from an earlier one. +- Migrate schema v9 to v10 without deleting history. Existing JSONL ledgers immediately show a rebuild notice and pause incremental ingestion until an explicit rebuild. Empty stores need no repair; direct v2.5.0 upgrades retain the earlier migration behavior. +- Update both READMEs, the rebuild dialog, and accounting documentation. Retain v2.6.2 download-directory settings. + +**Repair stored history after upgrading:** back up and verify retained JSONL coverage, then choose **Rescan → Approve and rebuild** or run `codex-usage scan --rebuild`. Updating the binary alone does not fix stored overcounts, and rebuilding cannot recover deleted source files. + +**Validation:** an isolated rebuild of accounting-only projections from 766 frozen rollouts matched reference totals for all 742 sessions with usage. The total changed from 19,646,071,899 to 14,941,760,325, removing 4,704,311,574 repeated tokens across 23 sessions. Changed sessions were independently reconciled against their source records; unchanged sessions retained snapshot reference totals. This is not an independent audit of all history or a billing reconciliation. A repeated scan inserted zero events and left totals unchanged. Regression coverage includes mixed counters, duplicate snapshots, corrections, restarts, missing evidence, migration preservation, and the API approval flow. diff --git a/internal/app/app.go b/internal/app/app.go index eebecd4..7c53861 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -31,7 +31,7 @@ import ( ) var ( - Version = "2.6.2" + Version = "2.6.3" Commit = "dev" BuildDate = "unknown" ) diff --git a/internal/server/accounting_migration_test.go b/internal/server/accounting_migration_test.go new file mode 100644 index 0000000..e707843 --- /dev/null +++ b/internal/server/accounting_migration_test.go @@ -0,0 +1,83 @@ +package server + +import ( + "context" + "database/sql" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/zJay26/codex-usage/internal/model" + "github.com/zJay26/codex-usage/internal/store" + "github.com/zJay26/codex-usage/internal/usage" +) + +func TestCounterScopeUpgradeCanOnlyRecalculateThroughApprovedRescan(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + home := filepath.Join(root, "codex") + if err := os.MkdirAll(filepath.Join(home, "sessions"), 0700); err != nil { + t.Fatal(err) + } + content := `{"type":"session_meta","payload":{"id":"mixed-counter"}}` + "\n" + for index, counters := range [][2]int{{100, 100}, {20, 20}, {50, 30}} { + content += fmt.Sprintf(`{"type":"turn_context","payload":{"turn_id":"turn-%d","model":"gpt-5.4"}}`+"\n", index) + content += fmt.Sprintf(`{"timestamp":"2026-09-08T01:00:0%dZ","type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":%d,"total_tokens":%d},"last_token_usage":{"input_tokens":%d,"total_tokens":%d}}}}`+"\n", + index, counters[0], counters[0], counters[1], counters[1]) + } + if err := os.WriteFile(filepath.Join(home, "sessions", "mixed.jsonl"), []byte(content), 0600); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "usage.sqlite") + st, err := store.Open(path) + if err != nil { + t.Fatal(err) + } + if _, err := st.InsertEvent(ctx, model.UsageEvent{ID: "old-overcount", SessionID: "mixed-counter", Timestamp: time.Now(), + Usage: model.TokenUsage{Input: 170, Total: 170}, Provenance: model.ProvenanceSessionJSONL, Confidence: model.ConfidenceExact, + }, "mixed.jsonl"); err != nil { + t.Fatal(err) + } + st.Close() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE meta SET value='9' WHERE key='schema_version'`); err != nil { + t.Fatal(err) + } + db.Close() + st, err = store.Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + srv := &Server{Store: st, Scanner: &usage.Scanner{Store: st}, Homes: func() ([]string, error) { return []string{home}, nil }} + for _, step := range []struct { + rebuild bool + status int + total int64 + }{ + {false, 409, 170}, // Preserve retained history until the user confirms. + {true, 200, 150}, // Rebuild removes the repeated baseline. + {false, 200, 150}, // Normal incremental operation resumes idempotently. + } { + request := httptest.NewRequest("POST", "http://127.0.0.1/api/v1/rescan", strings.NewReader(fmt.Sprintf(`{"rebuild":%v}`, step.rebuild))) + response := httptest.NewRecorder() + srv.Handler().ServeHTTP(response, request) + if response.Code != step.status { + t.Fatalf("rescan status=%d want=%d: %s", response.Code, step.status, response.Body.String()) + } + if step.status == 409 && !strings.Contains(response.Body.String(), `"rebuild_required":true`) { + t.Fatalf("Dashboard was not offered a rebuild: %s", response.Body.String()) + } + summary, err := st.Summary(ctx, model.Filter{}) + if err != nil || summary.GrandTotal != step.total || summary.CoverageIncomplete != (step.status == 409) { + t.Fatalf("history/quality after rescan: %+v, %v", summary, err) + } + } +} diff --git a/internal/store/accounting_migration_test.go b/internal/store/accounting_migration_test.go new file mode 100644 index 0000000..527f15a --- /dev/null +++ b/internal/store/accounting_migration_test.go @@ -0,0 +1,97 @@ +package store + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/zJay26/codex-usage/internal/model" +) + +func TestCounterScopeMigrationPreservesHistoryAndRequiresExplicitRebuild(t *testing.T) { + for _, version := range []int{8, 9} { + for _, history := range []bool{false, true} { + t.Run(fmt.Sprintf("v%d/history=%v", version, history), func(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "usage.sqlite") + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + if history { + if _, err := st.InsertEvent(ctx, model.UsageEvent{ + ID: "existing", SessionID: "session", TurnID: "turn", Timestamp: time.Now(), + Usage: model.TokenUsage{Input: 170, Total: 170}, + Provenance: model.ProvenanceSessionJSONL, Confidence: model.ConfidenceExact, + }, "original.jsonl"); err != nil { + t.Fatal(err) + } + if err := st.PutCursor(ctx, FileCursor{Path: "original.jsonl", SessionID: "session", Offset: 123, + Cumulative: model.TokenUsage{Input: 50, Total: 50}, + Accounting: AccountingState{Scope: "turn", LastTurnID: "turn"}, + }); err != nil { + t.Fatal(err) + } + } + if _, err := st.db.Exec(`UPDATE meta SET value=? WHERE key='schema_version'`, fmt.Sprint(version)); err != nil { + t.Fatal(err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + wantPending := version == 9 && history + // Reopening must preserve both the existing ledger and the marker. + for reopen := 0; reopen < 2; reopen++ { + st, err = Open(path) + if err != nil { + t.Fatal(err) + } + summary, err := st.Summary(ctx, model.Filter{}) + if err != nil { + t.Fatal(err) + } + var wantTotal int64 + if history { + wantTotal = 170 + cursor, ok, err := st.GetCursor(ctx, "original.jsonl") + if err != nil || !ok || cursor.Offset != 123 || cursor.Cumulative.Total != 50 { + t.Fatalf("migration changed the retained cursor: %+v, %v", cursor, err) + } + } + if summary.GrandTotal != wantTotal || summary.CoverageIncomplete != wantPending { + t.Fatalf("retained history or immediate quality notice: %+v", summary) + } + reason, pending, err := st.HistoricalRebuildReason(ctx) + if err != nil || pending != wantPending || (pending && !strings.Contains(reason, "v2.6.0")) { + t.Fatalf("pending=%v reason=%q err=%v", pending, reason, err) + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + } + st, err = Open(path) + if err != nil { + t.Fatal(err) + } + if err := st.ResetHistorical(ctx); err != nil { + t.Fatal(err) + } + st.Close() + st, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if _, pending, err := st.HistoricalRebuildReason(ctx); err != nil || pending { + t.Fatalf("explicit rebuild left a pending marker: %v, %v", pending, err) + } + if warnings, err := st.Warnings(ctx, 100); err != nil || len(warnings) != 0 { + t.Fatalf("explicit rebuild left stale warnings: %v, %v", warnings, err) + } + }) + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go index b23007e..d7c055b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -23,7 +23,7 @@ import ( ) const ( - schemaVersion = 9 + schemaVersion = 10 historicalRebuildReasonKey = "historical_rebuild_required" pricingAggregatableEventSQL = `e.input_tokens>=0 AND e.cached_input_tokens>=0 AND e.cache_write_input_tokens>=0 AND e.output_tokens>=0 AND e.reasoning_output_tokens>=0 AND e.total_tokens>=0 diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 7dd1f4d..2803f77 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -648,8 +648,8 @@ func TestV4MigrationPreservesHistoryUntilSingleMetadataRebuildApproved(t *testin if err := st.db.QueryRow(`SELECT value FROM meta WHERE key='schema_version'`).Scan(&version); err != nil { t.Fatal(err) } - if version != "9" { - t.Fatalf("schema version = %q, want 9", version) + if version != "10" { + t.Fatalf("schema version = %q, want 10", version) } if err := st.ResetHistorical(ctx); err != nil { t.Fatal(err) diff --git a/internal/store/transactions.go b/internal/store/transactions.go index 07f4923..e8bcec5 100644 --- a/internal/store/transactions.go +++ b/internal/store/transactions.go @@ -119,6 +119,26 @@ func migrateAccounting(ctx context.Context, tx *sql.Tx, version int) error { } } } + if version == 9 { + var hasHistory bool + if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM usage_events WHERE provenance='session_jsonl')`).Scan(&hasHistory); err != nil { + return err + } + if hasHistory { + // v2.6.0-v2.6.2 could repeat an entire cumulative baseline after a + // turn change. New parser code cannot correct those existing deltas. + // Keep the ledger and cursors until an explicit rebuild is requested, + // and expose the issue before any background scan has run. + reason := "v2.6.0–v2.6.2 可能在新 Turn 重复累计历史用量。现有统计已保留；请先备份并核对源 JSONL，再确认重建以修正。已删除源文件的历史无法通过重建恢复。" + if _, err := tx.ExecContext(ctx, `INSERT INTO meta(key,value) VALUES(?,?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value`, historicalRebuildReasonKey, reason); err != nil { + return err + } + if err := (&Store{tx: tx}).AddWarning(ctx, "schema_upgrade_rebuild", "", reason); err != nil { + return err + } + } + } if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO meta VALUES('data_revision','1')`); err != nil { return err } diff --git a/internal/usage/accounting.go b/internal/usage/accounting.go index fb085f1..32ab763 100644 --- a/internal/usage/accounting.go +++ b/internal/usage/accounting.go @@ -31,23 +31,33 @@ func (s *Scanner) scanFile(ctx context.Context, home, path string, meta model.Se // A context boundary alone is insufficient: legacy writers keep a session // counter across turns. Exact total==last at a NEW turn proves a fresh counter -// even when it equals or exceeds the previous turn's total. Once a writer has -// established turn scope, a missing first snapshot still starts at zero. -func selectCounterScope(cursor *store.FileCursor, info tokenInfo) bool { +// even when it equals or exceeds the previous turn's total. A reset only proves +// the current boundary: the next turn may continue that same cumulative series +// or initially repeat its last snapshot. Never reset from the saved scope alone. +// The result marks a boundary where preserving the baseline is conservative, +// but missing snapshots prevent distinguishing a continuation from a new reset. +func selectCounterScope(cursor *store.FileCursor, info tokenInfo) (unverified bool) { current, last := info.Total.usage(), info.Last.usage() newTurn := cursor.TurnID != "" && cursor.Accounting.LastTurnID != "" && cursor.TurnID != cursor.Accounting.LastTurnID - reset := newTurn && !current.IsZero() && (cursor.Accounting.Scope == "turn" || (!last.IsZero() && current.Equal(last))) + reset := newTurn && !current.IsZero() && !last.IsZero() && current.Equal(last) if reset { cursor.Accounting.Scope = "turn" cursor.Segment++ cursor.Cumulative = model.TokenUsage{} cursor.LastEventID = "" cursor.InheritedBaseline = false - } else if newTurn && !last.IsZero() && current.Sub(last).Equal(cursor.Cumulative) { + } else if newTurn { + continued := info.Total.withMissingSubsets(cursor.Cumulative) + unverified = cursor.Accounting.Scope == "turn" && current.Total > cursor.Cumulative.Total && + continued.MonotonicFrom(cursor.Cumulative) && !continued.Sub(cursor.Cumulative).Equal(last) + // Preserve the preceding baseline for a continuing series, a repeated + // snapshot, or a boundary without enough evidence to prove a reset. + // processRecord still handles an actual regression conservatively and + // warns when last_token_usage cannot explain the whole reset. cursor.Accounting.Scope = "session" } cursor.Accounting.LastTurnID = cursor.TurnID - return reset + return unverified } func isRecordError(err error) bool { diff --git a/internal/usage/accounting_test.go b/internal/usage/accounting_test.go index e8fc7a5..6266f56 100644 --- a/internal/usage/accounting_test.go +++ b/internal/usage/accounting_test.go @@ -92,6 +92,124 @@ func TestTurnCountersSmallerEqualLargerAndRestart(t *testing.T) { } } +func TestMixedCounterBoundariesReevaluateEachTurn(t *testing.T) { + for _, incremental := range []bool{false, true} { + t.Run(fmt.Sprintf("incremental=%v", incremental), func(t *testing.T) { + st, home, path := accountingFixture(t) + ctx := context.Background() + appendAccounting(t, path, accountingMeta) + steps := []struct { + turn string + total, last int64 + want int64 + }{ + {"one", 100, 100, 100}, + {"two", 20, 20, 120}, // A real reset does not classify later turns. + {"three", 50, 30, 150}, // Continue the preceding cumulative series. + {"four", 50, 30, 150}, // A new turn can initially repeat the old snapshot. + {"four", 60, 10, 160}, // Only this new increment belongs to turn four. + {"five", 200, 200, 360}, // Reset larger than the preceding total. + {"six", 250, 50, 410}, // Session continuity can return after that reset. + {"seven", 300, 50, 460}, + {"eight", 300, 300, 760}, // Equal totals can still be a proven fresh counter. + {"nine", 300, 300, 1060}, + } + for _, step := range steps { + appendAccounting(t, path, accountingTurn(step.turn)+accountingToken(step.total, step.last)) + if incremental { + if _, err := (&Scanner{Store: st}).Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, step.want) + } + } + for repeat := 0; repeat < 2; repeat++ { + if _, err := (&Scanner{Store: st}).Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, 1060) + } + if warnings, err := st.Warnings(ctx, 100); err != nil || len(warnings) != 0 { + t.Fatalf("unexpected warnings: %v, %v", warnings, err) + } + }) + } +} + +func TestMissingFirstSnapshotDoesNotSilentlyAssumeAnotherTurnReset(t *testing.T) { + st, home, path := accountingFixture(t) + ctx := context.Background() + appendAccounting(t, path, accountingMeta+accountingTurn("one")+accountingToken(100, 100)+ + accountingTurn("two")+accountingToken(20, 20)+accountingTurn("three")+accountingToken(50, 10)) + if _, err := (&Scanner{Store: st}).Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, 150) + warnings, err := st.Warnings(ctx, 100) + if err != nil || len(warnings) != 1 || warnings[0].Kind != "cumulative_boundary_unverified" { + t.Fatalf("uncertain boundary was silently treated as exact: %v, %v", warnings, err) + } + rows, err := st.Sessions(ctx, model.Filter{}, 100, 0) + if err != nil || len(rows) != 1 || rows[0].Confidence != model.ConfidenceGapFallback { + t.Fatalf("uncertain usage confidence: %v, %v", rows, err) + } +} + +func TestRepeatedSnapshotAfterResetDoesNotCreateUsageOrMoveCorrections(t *testing.T) { + st, home, path := accountingFixture(t) + ctx := context.Background() + appendAccounting(t, path, accountingMeta+accountingTurn("one")+accountingToken(100, 100)+ + accountingTurn("two")+accountingToken(20, 20)+accountingToken(50, 30)) + scan := &Scanner{Store: st} + if _, err := scan.Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + // Copied totals at a new turn must not inherit the preceding turn's scope. + appendAccounting(t, path, accountingTurn("three")+accountingToken(50, 30)) + if _, err := scan.Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, 150) + if u, err := st.TurnUsage(ctx, "scope-test", "three"); err != nil || !u.IsZero() { + t.Fatalf("repeated snapshot created new-turn usage: %+v, %v", u, err) + } + correction := tokenLine("2026-09-08T01:01:00Z", usage(50, 15, 0, 0, 0, 50), usage(0, 15, 0, 0, 0, 0)) + "\n" + next := tokenLine("2026-09-08T01:02:00Z", usage(60, 15, 0, 0, 0, 60), usage(10, 0, 0, 0, 0, 10)) + "\n" + appendAccounting(t, path, correction+next) + if _, err := scan.Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, 160) + if u, err := st.TurnUsage(ctx, "scope-test", "two"); err != nil || u.CachedInput != 15 { + t.Fatalf("classification correction missed original usage: %+v, %v", u, err) + } +} + +func TestRealMixedCounterVectorsDoNotRecountPreviousTurn(t *testing.T) { + st, home, path := accountingFixture(t) + ctx := context.Background() + // Numeric-only regression from a reconciled rollout. The final snapshot's + // six-field difference is exactly last_token_usage, despite an earlier reset. + first := usage(11388900, 11032320, 0, 64152, 29268, 11453052) + reset := usage(157821, 18176, 0, 621, 47, 158442) + previous := usage(11281076, 11060224, 0, 21940, 6881, 11303016) + current := usage(11486661, 11263744, 0, 22866, 7397, 11509527) + last := usage(205585, 203520, 0, 926, 516, 206511) + content := accountingMeta + accountingTurn("one") + tokenLine("2026-09-03T03:00:00Z", first, first) + "\n" + + accountingTurn("two") + tokenLine("2026-09-03T03:32:08Z", reset, reset) + "\n" + + tokenLine("2026-09-03T03:56:00Z", previous, usage(11123255, 11042048, 0, 21319, 6834, 11144574)) + "\n" + + accountingTurn("three") + tokenLine("2026-09-03T03:57:18Z", current, last) + "\n" + appendAccounting(t, path, content) + if _, err := (&Scanner{Store: st}).Scan(ctx, []string{home}, false); err != nil { + t.Fatal(err) + } + accountingTotal(t, st, 22962579) + want := model.TokenUsage{Input: 205585, CachedInput: 203520, Output: 926, ReasoningOutput: 516, Total: 206511} + if got, err := st.TurnUsage(ctx, "scope-test", "three"); err != nil || !got.Equal(want) { + t.Fatalf("new turn recounted preceding cumulative vector: %+v, %v", got, err) + } +} + func TestIngestFailureRollsBackEventModesAndCursor(t *testing.T) { for _, failure := range []struct{ name, target string }{ {"event", "INSERT ON usage_events"}, diff --git a/internal/usage/scanner.go b/internal/usage/scanner.go index 54b0bc0..e1e66b5 100644 --- a/internal/usage/scanner.go +++ b/internal/usage/scanner.go @@ -608,7 +608,7 @@ func (s *Scanner) processRecord( return &recordError{fmt.Errorf("invalid token vector")} } cursor.TurnID = firstNonEmpty(payload.TurnID, cursor.TurnID) - selectCounterScope(cursor, info) + unverifiedBoundary := selectCounterScope(cursor, info) if err := s.inheritTurnProgress(ctx, cursor); err != nil { return err } @@ -619,6 +619,12 @@ func (s *Scanner) processRecord( } if current.Equal(cursor.Cumulative) { result.Duplicates++ + // A repeated snapshot can establish that a new turn continued the + // prior counter. Persist that interpretation even without a new event; + // otherwise the next scan can inherit the previous turn's stale scope. + if cursor.SessionID != "" { + return s.Store.PutSessionProgress(ctx, cursor.SessionID, cursor.Segment, current, cursor.Accounting) + } return nil } if current.Total > 0 && current.Total == cursor.Cumulative.Total { @@ -654,6 +660,14 @@ func (s *Scanner) processRecord( } delta := current confidence := model.ConfidenceExact + if unverifiedBoundary { + confidence = model.ConfidenceGapFallback + if err := s.Store.AddWarning(ctx, "cumulative_boundary_unverified", path, + fmt.Sprintf("offset=%d：新 Turn 的首个快照无法证明累计是否重置；保留此前基线按差值计量，可能缺少用量", offset)); err != nil { + return err + } + result.Warnings++ + } if !cursor.Cumulative.IsZero() { if current.MonotonicFrom(cursor.Cumulative) { delta = current.Sub(cursor.Cumulative) diff --git a/internal/web/static/i18n.js b/internal/web/static/i18n.js index c25a0e6..543e502 100644 --- a/internal/web/static/i18n.js +++ b/internal/web/static/i18n.js @@ -348,6 +348,7 @@ "warning.rollout_rewritten": "JSONL 已重写，等待确认重建", "warning.rollout_truncated": "JSONL 已截断，等待确认重建", "warning.schema_upgrade_rebuild": "解析规则已升级，等待确认重建", + "warning.cumulative_boundary_unverified": "累计边界无法完整核对，可能缺少用量", "warning.cumulative_reset": "累计快照回退补位", "warning.cumulative_gap_fallback": "累计边界无法完整核对", "warning.timestamp": "时间戳无法归属", @@ -379,7 +380,7 @@ "pricing.modelRequired": "请输入要覆写的模型名", "scan.complete": "扫描完成：新增 {inserted} 个事件，忽略 {duplicates} 个重复", "rebuild.title": "需要重建统计数据", - "rebuild.intro": "检测到本地 JSONL 历史发生变化。重建会先清空当前派生统计，再仅从当前仍存在的 JSONL 重新计算；已删除的 JSONL 数据可能从统计中消失。", + "rebuild.intro": "计量规则或本地 JSONL 历史发生变化。请先备份并核对源文件：重建会清空当前派生统计，再仅从仍存在的 JSONL 重新计算；已删除源文件的历史可能从统计中消失。", "rebuild.keep": "保留现有统计", "rebuild.confirm": "同意并重建" }, @@ -728,6 +729,7 @@ "warning.rollout_rewritten": "Rewritten JSONL awaiting rebuild approval", "warning.rollout_truncated": "Truncated JSONL awaiting rebuild approval", "warning.schema_upgrade_rebuild": "Parser upgrade awaiting rebuild approval", + "warning.cumulative_boundary_unverified": "Unverified counter boundary; usage may be incomplete", "warning.cumulative_reset": "Cumulative snapshot reset fallback", "warning.cumulative_gap_fallback": "Cumulative boundary could not be fully verified", "warning.timestamp": "Timestamp could not be attributed", @@ -759,7 +761,7 @@ "pricing.modelRequired": "Enter a model name to override", "scan.complete": "Scan complete: {inserted} events added, {duplicates} duplicates ignored", "rebuild.title": "Usage history rebuild required", - "rebuild.intro": "Local JSONL history has changed. Rebuilding first clears the current derived statistics, then recalculates only from JSONL files that still exist. Data from deleted JSONL files may disappear.", + "rebuild.intro": "Accounting rules or local JSONL history have changed. Back up and check source coverage first: rebuilding clears derived statistics and recalculates only retained JSONL. History whose source files were deleted may disappear.", "rebuild.keep": "Keep current statistics", "rebuild.confirm": "Approve and rebuild" } diff --git a/scripts/build.ps1 b/scripts/build.ps1 index f558471..74d1de9 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -1,5 +1,5 @@ param( - [string]$Version = "2.6.2" + [string]$Version = "2.6.3" ) $ErrorActionPreference = "Stop" diff --git a/scripts/build.sh b/scripts/build.sh index 13a7e3a..9e4536d 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -3,7 +3,7 @@ set -euo pipefail project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" dist="$project_root/dist" -version="${VERSION:-2.6.2}" +version="${VERSION:-2.6.3}" build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" commit="$(git -C "$project_root" rev-parse --short HEAD 2>/dev/null || printf source)" if git -C "$project_root" rev-parse --is-inside-work-tree >/dev/null 2>&1 && diff --git a/scripts/demo-api.js b/scripts/demo-api.js index daa6051..0dba894 100644 --- a/scripts/demo-api.js +++ b/scripts/demo-api.js @@ -300,7 +300,7 @@ }; } - const demoUpdates = { current_version: "2.6.2-demo", latest_version: "2.6.2", release_url: "https://github.com/zJay26/codex-usage/releases", auto_check: true, available: false, can_install: false, phase: "idle", download_dir: "C:\\Users\\Demo\\Downloads\\codex-usage", default_download_dir: "C:\\Users\\Demo\\Downloads\\codex-usage", custom_download_dir: "", can_open_download_dir: false }; + const demoUpdates = { current_version: "2.6.3-demo", latest_version: "2.6.3", release_url: "https://github.com/zJay26/codex-usage/releases", auto_check: true, available: false, can_install: false, phase: "idle", download_dir: "C:\\Users\\Demo\\Downloads\\codex-usage", default_download_dir: "C:\\Users\\Demo\\Downloads\\codex-usage", custom_download_dir: "", can_open_download_dir: false }; async function syntheticFetch(input, init = {}) { const raw = typeof input === "string" ? input : input.url; const url = new URL(raw, root.location.href); @@ -317,7 +317,7 @@ return jsonResponse(demoUpdates); } if (endpoint === "/api/v1/status") return jsonResponse({ - version: "2.6.2-demo", scanning: false, + version: "2.6.3-demo", scanning: false, status: { machine: { id: "synthetic-machine", label: "Synthetic Windows · demo", hostname: "synthetic-host", os: "windows", arch: "amd64" }, last_scan: now.toISOString(), accounting_mode: "jsonl_only", otel_active: false, diff --git a/scripts/mock-dashboard.mjs b/scripts/mock-dashboard.mjs index ca4586b..d3f2b90 100644 --- a/scripts/mock-dashboard.mjs +++ b/scripts/mock-dashboard.mjs @@ -86,7 +86,7 @@ function costEstimate(url) { const server = http.createServer(async (request, response) => { const url = new URL(request.url, `http://127.0.0.1:${port}`); if (url.pathname === "/api/v1/status") return json(response, { - version: "2.6.2-preview", scanning: false, + version: "2.6.3-preview", scanning: false, status: { machine: { id: "62c0172d-36c4-4ec9-a074-02b9ec2b45e1", label: "WORKSTATION-19 · windows", hostname: "WORKSTATION-19", os: "windows", arch: "amd64" }, last_scan: now.toISOString(), accounting_mode: "jsonl_only", otel_active: false,