From 59e12a9eac01f04c02187888793264a7934072b8 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:13:26 +0800 Subject: [PATCH 1/4] fix(usage): avoid idle history rescans --- desktop/backend/services.go | 8 +- desktop/backend/usage_auto_sync.go | 102 +++- desktop/backend/usage_auto_sync_test.go | 140 +++-- docs/codex/usage-cost.md | 2 + docs/grok-build/usage-cost.md | 6 +- docs/zh/codex/usage-cost.md | 2 + docs/zh/grok-build/usage-cost.md | 6 +- internal/bootstrap/service_test.go | 66 ++- internal/cli/command_test.go | 2 +- internal/store/integrity.go | 60 ++- ...2609110001_usage_incremental_checkpoint.go | 144 +++++ internal/store/migrations/migrations_test.go | 130 +++++ internal/store/schema_contract.go | 89 +++ internal/store/store_test.go | 21 + internal/store/usage.go | 172 +++++- internal/store/usage_codex.go | 133 ++++- internal/store/usage_compact_test.go | 57 ++ internal/store/usage_grok_build.go | 134 ++++- internal/usage/codex.go | 167 +++--- internal/usage/codex_sync.go | 440 ++++++++++++--- internal/usage/file_identity_other.go | 13 + internal/usage/file_identity_unix.go | 29 + internal/usage/file_identity_windows.go | 49 ++ internal/usage/grok_build.go | 7 +- internal/usage/grok_build_sync.go | 436 +++++++++++---- internal/usage/grok_build_sync_test.go | 69 ++- internal/usage/incremental_reader.go | 506 ++++++++++++++++++ internal/usage/incremental_reader_test.go | 198 +++++++ internal/usage/registry.go | 17 +- internal/usage/registry_test.go | 9 +- internal/usage/service.go | 44 +- internal/usage/service_test.go | 292 +++++++++- internal/usage/usage.go | 42 +- 33 files changed, 3177 insertions(+), 415 deletions(-) create mode 100644 internal/store/migrations/202609110001_usage_incremental_checkpoint.go create mode 100644 internal/usage/file_identity_other.go create mode 100644 internal/usage/file_identity_unix.go create mode 100644 internal/usage/file_identity_windows.go create mode 100644 internal/usage/incremental_reader.go create mode 100644 internal/usage/incremental_reader_test.go diff --git a/desktop/backend/services.go b/desktop/backend/services.go index f4335fc..5c874d9 100644 --- a/desktop/backend/services.go +++ b/desktop/backend/services.go @@ -293,8 +293,8 @@ func NewServices(application *app.Application, info app.Info, env Environment, s UsageSyncIntervalSeconds: value.UsageSyncIntervalSeconds, }, err }, - func(ctx context.Context) (usage.UsageSyncResult, error) { - return application.Usage().SyncProviderBackground(ctx, codexconfig.ProviderID) + func(ctx context.Context, onWorkDetected func()) (usage.BackgroundSyncOutcome, error) { + return application.Usage().SyncProviderBackground(ctx, codexconfig.ProviderID, onWorkDetected) }, ) grokBuildUsageSync := newUsageAutoSyncRuntime( @@ -305,8 +305,8 @@ func NewServices(application *app.Application, info app.Info, env Environment, s UsageSyncIntervalSeconds: value.UsageSyncIntervalSeconds, }, err }, - func(ctx context.Context) (usage.UsageSyncResult, error) { - return application.Usage().SyncProviderBackground(ctx, grokconfig.ProviderID) + func(ctx context.Context, onWorkDetected func()) (usage.BackgroundSyncOutcome, error) { + return application.Usage().SyncProviderBackground(ctx, grokconfig.ProviderID, onWorkDetected) }, ) quota := newCodexQuotaRuntime(application.Codex().ListAutomationTargets, application.Codex().RunCredentialJob) diff --git a/desktop/backend/usage_auto_sync.go b/desktop/backend/usage_auto_sync.go index e54661f..2f53a32 100644 --- a/desktop/backend/usage_auto_sync.go +++ b/desktop/backend/usage_auto_sync.go @@ -53,6 +53,8 @@ type realUsageAutoSyncTicker struct { ticker *time.Ticker } +type backgroundUsageSync func(context.Context, func()) (usage.BackgroundSyncOutcome, error) + func (t realUsageAutoSyncTicker) C() <-chan time.Time { return t.ticker.C } @@ -71,6 +73,7 @@ type usageAutoSyncRuntime struct { emitter func(UsageAutoSyncStatus) intervalRevision uint64 syncRevision uint64 + syncRunning bool lifecycleMu sync.Mutex started bool @@ -88,13 +91,13 @@ type usageAutoSyncRuntime struct { startDelayFunc func() time.Duration timeout time.Duration loadSettings func(context.Context) (usage.ProviderSyncSettings, error) - syncProvider func(context.Context) (usage.UsageSyncResult, error) + syncProvider backgroundUsageSync } func newUsageAutoSyncRuntime( providerID string, loadSettings func(context.Context) (usage.ProviderSyncSettings, error), - syncProvider func(context.Context) (usage.UsageSyncResult, error), + syncProvider backgroundUsageSync, ) *usageAutoSyncRuntime { providerID = strings.TrimSpace(providerID) return &usageAutoSyncRuntime{ @@ -245,11 +248,12 @@ func (r *usageAutoSyncRuntime) SyncNow(ctx context.Context) UsageAutoSyncStatus r.startSync(r.runCtx) r.mu.RLock() done := r.syncDone + running := r.syncRunning status := cloneUsageAutoSyncStatus(r.status) r.mu.RUnlock() r.lifecycleMu.Unlock() - if done == nil || !status.Syncing { + if done == nil || !running { return status } select { @@ -342,20 +346,15 @@ func (r *usageAutoSyncRuntime) startSyncWithRevision( requireRevision bool, ) bool { r.mu.Lock() - if r.status.Syncing || parent.Err() != nil || + if r.syncRunning || parent.Err() != nil || (requireRevision && r.syncRevision != expectedRevision) { r.mu.Unlock() return false } r.syncRevision++ - r.status.Syncing = true - r.status.Outcome = UsageAutoSyncOutcomeSyncing - r.status.Error = nil - r.status.LastStartedAtUnixMS = r.now().UnixMilli() - r.status.Revision++ + r.syncRunning = true r.syncDone = make(chan struct{}) r.mu.Unlock() - r.emitStatus() r.workerWG.Add(1) go func() { @@ -366,13 +365,15 @@ func (r *usageAutoSyncRuntime) startSyncWithRevision( } ctx, cancel := context.WithTimeout(usage.WithPhaseTimeout(parent, timeout), 2*timeout) defer cancel() - result, err := r.syncProvider(ctx) + outcome, err := r.runProviderSync(ctx) if parent.Err() != nil { r.mu.Lock() - r.status.Syncing = false - r.status.Outcome = UsageAutoSyncOutcomeIdle - r.status.Error = nil - r.status.Revision++ + if r.status.Syncing { + r.status.Syncing = false + r.status.Outcome = UsageAutoSyncOutcomeIdle + r.status.Error = nil + r.status.Revision++ + } r.finishSyncLocked() r.mu.Unlock() return @@ -381,11 +382,75 @@ func (r *usageAutoSyncRuntime) startSyncWithRevision( r.completeWithError(err) return } - r.completeWithResult(result) + if !outcome.Performed { + r.completeWithoutWork(outcome.Result) + return + } + r.markWorkDetected() + r.completeWithResult(outcome.Result) }() return true } +func (r *usageAutoSyncRuntime) runProviderSync(ctx context.Context) (usage.BackgroundSyncOutcome, error) { + if r.syncProvider == nil { + return usage.BackgroundSyncOutcome{}, errors.New("usage sync provider is unavailable") + } + return r.syncProvider(ctx, r.markWorkDetected) +} + +func (r *usageAutoSyncRuntime) markWorkDetected() { + r.mu.Lock() + if !r.syncRunning || r.status.Syncing { + r.mu.Unlock() + return + } + r.status.Syncing = true + r.status.Outcome = UsageAutoSyncOutcomeSyncing + r.status.Error = nil + r.status.LastStartedAtUnixMS = r.now().UnixMilli() + r.status.Revision++ + r.mu.Unlock() + r.emitStatus() +} + +func (r *usageAutoSyncRuntime) completeWithoutWork(result usage.UsageSyncResult) { + r.mu.Lock() + if len(result.Errors) == 0 { + if r.status.Outcome == UsageAutoSyncOutcomeError || + r.status.Outcome == UsageAutoSyncOutcomeWarning || + r.status.Error != nil || r.status.ImportErrorCount != 0 { + r.status.Syncing = false + r.status.Outcome = UsageAutoSyncOutcomeIdle + r.status.ImportErrorCount = 0 + r.status.Error = nil + r.status.Revision++ + r.finishSyncLocked() + r.mu.Unlock() + r.emitStatus() + return + } + r.finishSyncLocked() + r.mu.Unlock() + return + } + count := int64(len(result.Errors)) + if r.status.Outcome == UsageAutoSyncOutcomeWarning && + r.status.ImportErrorCount == count && r.status.Error == nil { + r.finishSyncLocked() + r.mu.Unlock() + return + } + r.status.Syncing = false + r.status.Outcome = UsageAutoSyncOutcomeWarning + r.status.ImportErrorCount = count + r.status.Error = nil + r.status.Revision++ + r.finishSyncLocked() + r.mu.Unlock() + r.emitStatus() +} + func (r *usageAutoSyncRuntime) completeWithResult(result usage.UsageSyncResult) { completedAt := r.now().UnixMilli() outcome := UsageAutoSyncOutcomeSuccess @@ -393,6 +458,7 @@ func (r *usageAutoSyncRuntime) completeWithResult(result usage.UsageSyncResult) outcome = UsageAutoSyncOutcomeWarning } r.mu.Lock() + r.syncRunning = false r.status.Syncing = false r.status.Outcome = outcome r.status.LastCompletedAtUnixMS = completedAt @@ -408,6 +474,7 @@ func (r *usageAutoSyncRuntime) completeWithResult(result usage.UsageSyncResult) func (r *usageAutoSyncRuntime) completeWithError(err error) { completedAt := r.now().UnixMilli() r.mu.Lock() + r.syncRunning = false r.status.Syncing = false r.status.Outcome = UsageAutoSyncOutcomeError r.status.LastCompletedAtUnixMS = completedAt @@ -424,7 +491,7 @@ func (r *usageAutoSyncRuntime) reportStartupError(err error, expectedSyncRevisio r.mu.Lock() // SyncNow can join the runtime while its startup settings read is still in // flight. That read must not complete or supersede the active Provider sync. - if r.status.Syncing || r.syncRevision != expectedSyncRevision { + if r.syncRunning || r.syncRevision != expectedSyncRevision { r.mu.Unlock() return } @@ -438,6 +505,7 @@ func (r *usageAutoSyncRuntime) reportStartupError(err error, expectedSyncRevisio } func (r *usageAutoSyncRuntime) finishSyncLocked() { + r.syncRunning = false if r.syncDone == nil { return } diff --git a/desktop/backend/usage_auto_sync_test.go b/desktop/backend/usage_auto_sync_test.go index ed07c93..d0e08f5 100644 --- a/desktop/backend/usage_auto_sync_test.go +++ b/desktop/backend/usage_auto_sync_test.go @@ -2,6 +2,7 @@ package backend import ( "context" + "errors" "fmt" "strings" "sync" @@ -19,14 +20,14 @@ func TestUsageAutoSyncStartsImmediatelyAndSkipsOverlappingTicks(t *testing.T) { started := make(chan int32, 3) releaseFirst := make(chan struct{}) var calls atomic.Int32 - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { call := calls.Add(1) started <- call if call == 1 { <-releaseFirst } return usage.UsageSyncResult{}, nil - } + }) statuses := make(chan UsageAutoSyncStatus, 16) runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) t.Cleanup(runtime.Stop) @@ -51,13 +52,80 @@ func TestUsageAutoSyncStartsImmediatelyAndSkipsOverlappingTicks(t *testing.T) { } } +func TestUsageAutoSyncNoopDoesNotPublishOrAdvanceStatus(t *testing.T) { + runtime, ticker := newTestUsageAutoSyncRuntime() + called := make(chan struct{}, 2) + runtime.syncProvider = func(context.Context, func()) (usage.BackgroundSyncOutcome, error) { + called <- struct{}{} + return usage.BackgroundSyncOutcome{ + Result: usage.UsageSyncResult{ProviderID: codexconfig.ProviderID}, + }, nil + } + statuses := make(chan UsageAutoSyncStatus, 4) + runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) + t.Cleanup(runtime.Stop) + waitUsageSyncSignal(t, called) + + initial := runtime.Status() + if initial.Revision != 0 || initial.Syncing || initial.Outcome != UsageAutoSyncOutcomeIdle || + initial.LastStartedAtUnixMS != 0 || initial.LastCompletedAtUnixMS != 0 { + t.Fatalf("no-op startup changed status: %#v", initial) + } + select { + case status := <-statuses: + t.Fatalf("no-op startup published status: %#v", status) + case <-time.After(50 * time.Millisecond): + } + + ticker.tick() + waitUsageSyncSignal(t, called) + if status := runtime.Status(); status != initial { + t.Fatalf("repeated no-op advanced status: before=%#v after=%#v", initial, status) + } + select { + case status := <-statuses: + t.Fatalf("repeated no-op published status: %#v", status) + case <-time.After(50 * time.Millisecond): + } +} + +func TestUsageAutoSyncNoopClearsRecoveredError(t *testing.T) { + runtime, ticker := newTestUsageAutoSyncRuntime() + called := make(chan struct{}, 2) + var calls atomic.Int32 + runtime.syncProvider = func(context.Context, func()) (usage.BackgroundSyncOutcome, error) { + called <- struct{}{} + if calls.Add(1) == 1 { + return usage.BackgroundSyncOutcome{}, errors.New("temporary failure") + } + return usage.BackgroundSyncOutcome{}, nil + } + statuses := make(chan UsageAutoSyncStatus, 4) + runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) + t.Cleanup(runtime.Stop) + waitUsageSyncSignal(t, called) + waitUsageAutoSyncStatus(t, statuses, func(status UsageAutoSyncStatus) bool { + return status.Outcome == UsageAutoSyncOutcomeError + }) + + ticker.tick() + waitUsageSyncSignal(t, called) + recovered := waitUsageAutoSyncStatus(t, statuses, func(status UsageAutoSyncStatus) bool { + return status.Outcome == UsageAutoSyncOutcomeIdle + }) + if recovered.Error != nil || recovered.ImportErrorCount != 0 || + recovered.LastSuccessAtUnixMS != 0 { + t.Fatalf("no-op recovery status = %#v", recovered) + } +} + func TestUsageAutoSyncResetsIntervalWithoutImmediateSync(t *testing.T) { runtime, ticker := newTestUsageAutoSyncRuntime() started := make(chan struct{}, 2) - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { started <- struct{}{} return usage.UsageSyncResult{}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) @@ -92,10 +160,10 @@ func TestUsageAutoSyncStartDelayDefersFirstSync(t *testing.T) { return ch } syncStarted := make(chan struct{}, 1) - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { syncStarted <- struct{}{} return usage.UsageSyncResult{ProviderID: "codex"}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) @@ -127,10 +195,10 @@ func TestUsageAutoSyncSyncNowDoesNotWaitForStartDelay(t *testing.T) { return ch } syncStarted := make(chan struct{}, 1) - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { syncStarted <- struct{}{} return usage.UsageSyncResult{ProviderID: "codex"}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) t.Cleanup(func() { close(releaseDelay) }) @@ -161,18 +229,18 @@ func TestUsageAutoSyncStatusIsProviderScoped(t *testing.T) { func(context.Context) (usage.ProviderSyncSettings, error) { return usage.ProviderSyncSettings{UsageSyncIntervalSeconds: 5}, nil }, - func(context.Context) (usage.UsageSyncResult, error) { + performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{ProviderID: "codex"}, nil - }, + }), ) grok := newUsageAutoSyncRuntime( "grok-build", func(context.Context) (usage.ProviderSyncSettings, error) { return usage.ProviderSyncSettings{UsageSyncIntervalSeconds: 60}, nil }, - func(context.Context) (usage.UsageSyncResult, error) { + performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{ProviderID: "grok-build"}, nil - }, + }), ) codex.SetInterval(15) grok.SetInterval(30) @@ -200,9 +268,9 @@ func TestUsageAutoSyncStartupLoadDoesNotOverwriteNewerInterval(t *testing.T) { createdIntervals <- interval return ticker } - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) select { @@ -243,11 +311,11 @@ func TestUsageAutoSyncStartupLoadFailureDoesNotSupersedeSyncNow(t *testing.T) { syncStarted := make(chan int32, 2) releaseSync := make(chan struct{}) var calls atomic.Int32 - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { syncStarted <- calls.Add(1) <-releaseSync return usage.UsageSyncResult{}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) t.Cleanup(func() { @@ -311,10 +379,10 @@ func TestUsageAutoSyncStartupLoadFailureDoesNotOverwriteCompletedSyncNow(t *test return ticker } var calls atomic.Int32 - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { calls.Add(1) return usage.UsageSyncResult{}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) t.Cleanup(func() { @@ -346,14 +414,14 @@ func TestUsageAutoSyncRetriesAfterTimeout(t *testing.T) { runtime.timeout = 20 * time.Millisecond releaseRetry := make(chan struct{}) var calls atomic.Int32 - runtime.syncProvider = func(ctx context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(ctx context.Context) (usage.UsageSyncResult, error) { if calls.Add(1) == 1 { <-ctx.Done() return usage.UsageSyncResult{}, ctx.Err() } <-releaseRetry return usage.UsageSyncResult{}, nil - } + }) statuses := make(chan UsageAutoSyncStatus, 16) runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) t.Cleanup(runtime.Stop) @@ -392,14 +460,14 @@ func TestUsageAutoSyncSyncNowStartsAndWaitsForProviderResult(t *testing.T) { started := make(chan int32, 2) releaseRequested := make(chan struct{}) var calls atomic.Int32 - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { call := calls.Add(1) started <- call if call == 2 { <-releaseRequested } return usage.UsageSyncResult{}, nil - } + }) statuses := make(chan UsageAutoSyncStatus, 16) runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) t.Cleanup(runtime.Stop) @@ -450,12 +518,12 @@ func TestUsageAutoSyncSyncNowJoinsActiveProviderSync(t *testing.T) { started := make(chan struct{}, 1) release := make(chan struct{}) var calls atomic.Int32 - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { calls.Add(1) started <- struct{}{} <-release return usage.UsageSyncResult{}, nil - } + }) runtime.Start(context.Background(), nil) t.Cleanup(runtime.Stop) t.Cleanup(func() { @@ -494,9 +562,9 @@ func TestUsageAutoSyncSyncNowJoinsActiveProviderSync(t *testing.T) { func TestUsageAutoSyncReportsWarningsAndRedactsFatalErrors(t *testing.T) { t.Run("warning", func(t *testing.T) { runtime, _ := newTestUsageAutoSyncRuntime() - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{Errors: []usage.UsageImportError{{SourceKey: "/private/session.jsonl", Message: "raw session error"}}}, nil - } + }) statuses := make(chan UsageAutoSyncStatus, 8) runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) t.Cleanup(runtime.Stop) @@ -512,9 +580,9 @@ func TestUsageAutoSyncReportsWarningsAndRedactsFatalErrors(t *testing.T) { t.Run("fatal", func(t *testing.T) { runtime, _ := newTestUsageAutoSyncRuntime() raw := "/Users/alice/.codex/sessions/private.jsonl" - runtime.syncProvider = func(context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{}, fmt.Errorf("failed to read %s", raw) - } + }) statuses := make(chan UsageAutoSyncStatus, 8) runtime.Start(context.Background(), func(status UsageAutoSyncStatus) { statuses <- status }) t.Cleanup(runtime.Stop) @@ -535,12 +603,12 @@ func TestUsageAutoSyncStopCancelsRunningSync(t *testing.T) { runtime, _ := newTestUsageAutoSyncRuntime() started := make(chan struct{}) stopped := make(chan struct{}) - runtime.syncProvider = func(ctx context.Context) (usage.UsageSyncResult, error) { + runtime.syncProvider = performedUsageSync(func(ctx context.Context) (usage.UsageSyncResult, error) { close(started) <-ctx.Done() close(stopped) return usage.UsageSyncResult{}, ctx.Err() - } + }) runtime.Start(context.Background(), nil) waitUsageSyncSignal(t, started) runtime.Stop() @@ -588,6 +656,18 @@ func newTestUsageAutoSyncRuntime() (*usageAutoSyncRuntime, *fakeUsageAutoSyncTic return runtime, ticker } +func performedUsageSync( + sync func(context.Context) (usage.UsageSyncResult, error), +) backgroundUsageSync { + return func(ctx context.Context, onWorkDetected func()) (usage.BackgroundSyncOutcome, error) { + if onWorkDetected != nil { + onWorkDetected() + } + result, err := sync(ctx) + return usage.BackgroundSyncOutcome{Result: result, Performed: true}, err + } +} + func waitUsageSyncCall(t *testing.T, calls <-chan int32) int32 { t.Helper() select { diff --git a/docs/codex/usage-cost.md b/docs/codex/usage-cost.md index e765239..c92f1b2 100644 --- a/docs/codex/usage-cost.md +++ b/docs/codex/usage-cost.md @@ -8,6 +8,8 @@ The Desktop app syncs after startup and continues while ProfileDeck is open or i To change the interval, open **Codex → Settings → Usage reports → Update frequency** and choose 5, 15, 30, or 60 seconds. The default is 15 seconds. The Usage page shows the latest sync result and reports files it could not read. +When session files have not changed, background sync checks their metadata without reading their contents. Normal appends read only a small integrity boundary and the new part of each file. If an existing file is truncated or ProfileDeck detects a change to earlier usage, it keeps the previously imported history and skips that file version. It checks the file again after it changes or when you run a CLI sync. + ## Sync from the CLI Run: diff --git a/docs/grok-build/usage-cost.md b/docs/grok-build/usage-cost.md index ab33b2e..9a405d8 100644 --- a/docs/grok-build/usage-cost.md +++ b/docs/grok-build/usage-cost.md @@ -8,6 +8,8 @@ The Desktop app syncs after startup and continues while ProfileDeck is open or i To change the interval, open **Grok Build → Settings → Usage reports → Update frequency** and choose 5, 15, 30, or 60 seconds. The default is 15 seconds. Codex and Grok Build use separate intervals and sync status. +When session files have not changed, background sync checks their metadata without reading their contents. Normal appends read only a small integrity boundary and the new part of each file. + Background sync uses the existing Grok Build Provider. If it has not been created yet, open **Grok Build → Profiles** to create a Profile, or run an explicit CLI sync. ## Sync from the CLI @@ -34,9 +36,9 @@ ProfileDeck reads ordinary files matching: It does not follow symbolic links. Nested `subagents` records are excluded because Grok Build already includes successful child-agent usage in the completed parent turn. -You can repeat a sync safely; previously imported usage is not counted again. Records with missing, empty, or incomplete usage are skipped. If a file changes while it is being read, contains an oversized or malformed terminal record, or has an unrecognized terminal format, ProfileDeck leaves all new data from that file uncommitted and retries it during a later sync. The source file is never moved or changed. +You can repeat a sync safely; previously imported usage is not counted again. Records with missing, empty, or incomplete usage are skipped. If a file changes while it is being read, contains an oversized or malformed terminal record, or has an unrecognized terminal format, ProfileDeck leaves all new data from that file uncommitted. Background sync checks that file again after it changes; a CLI sync checks it immediately. The source file is never moved or changed. -Copied fork history is counted once. When the same completed turn appears in multiple sessions with identical usage, ProfileDeck assigns it to one stable derived session without storing the original session identifier. Conflicting usage for that same turn causes the affected file to be retried instead. +Copied fork history is counted once. When the same completed turn appears in multiple sessions with identical usage, ProfileDeck assigns it to one stable derived session without storing the original session identifier. Conflicting usage for that same turn leaves the affected file uncommitted until it changes or you run another CLI sync. Deleting the Grok Build Provider also deletes its saved usage reports, import progress, and sync setting. Desktop background sync will not recreate a deleted Provider. Running this CLI sync is an explicit request: it can set up the Provider again and reimport usage still present in local session files. diff --git a/docs/zh/codex/usage-cost.md b/docs/zh/codex/usage-cost.md index f3a467d..233a9bb 100644 --- a/docs/zh/codex/usage-cost.md +++ b/docs/zh/codex/usage-cost.md @@ -8,6 +8,8 @@ ProfileDeck 读取本地 Codex 会话数据，展示令牌用量、活动情况 如需调整间隔，请打开 **Codex → 设置 → 用量报告 → 更新频率**，选择 5、15、30 或 60 秒；默认值为 15 秒。用量页面会显示最近一次同步结果，并报告无法读取的文件。 +会话文件没有变化时，后台同步只检查文件信息，不会读取文件内容。文件正常追加后，只读取一小段完整性校验边界和新增部分。如果现有文件被截短，或 ProfileDeck 检测到之前的用量记录发生变化，它会保留已导入的历史并跳过该文件版本；文件再次变化或手动运行 CLI 同步后会重新检查。 + ## 使用 CLI 同步 运行： diff --git a/docs/zh/grok-build/usage-cost.md b/docs/zh/grok-build/usage-cost.md index 99c4ffa..ca7f166 100644 --- a/docs/zh/grok-build/usage-cost.md +++ b/docs/zh/grok-build/usage-cost.md @@ -8,6 +8,8 @@ ProfileDeck 读取本地 Grok Build 会话记录，展示令牌用量、活动 如需调整间隔，请打开 **Grok Build → 设置 → 用量报告 → 更新频率**，选择 5、15、30 或 60 秒；默认值为 15 秒。Codex 与 Grok Build 分别使用独立的间隔和同步状态。 +会话文件没有变化时，后台同步只检查文件信息，不会读取文件内容。文件正常追加后，只读取一小段完整性校验边界和新增部分。 + 后台同步只使用现有 Grok Build Provider。如果尚未创建，请打开 **Grok Build → Profiles** 创建一个 Profile，或明确运行一次 CLI 同步。 ## 使用 CLI 同步 @@ -34,9 +36,9 @@ ProfileDeck 读取以下位置匹配的普通文件： 它不会跟随符号链接。嵌套的 `subagents` 记录会被排除，因为 Grok Build 已把成功子代理的用量计入完成的父回合。 -你可以安全地重复同步，已导入的用量不会再次计数。缺少用量、空用量或用量不完整的记录会被跳过。如果文件在读取期间发生变化，包含过大或格式错误的终态记录，或出现无法识别的终态格式，ProfileDeck 不会提交该文件中的任何新数据，并会在后续同步时重试。源文件不会被移动或修改。 +你可以安全地重复同步，已导入的用量不会再次计数。缺少用量、空用量或用量不完整的记录会被跳过。如果文件在读取期间发生变化，包含过大或格式错误的终态记录，或出现无法识别的终态格式，ProfileDeck 不会提交该文件中的任何新数据。文件发生变化后，后台同步会重新检查；CLI 同步会立即检查。源文件不会被移动或修改。 -Fork 复制的历史只计算一次。同一个已完成回合以相同用量出现在多个会话时，ProfileDeck 会将其稳定归入一个派生会话，不保存原始会话标识；如果同一回合的用量冲突，则改为重试受影响的文件。 +Fork 复制的历史只计算一次。同一个已完成回合以相同用量出现在多个会话时，ProfileDeck 会将其稳定归入一个派生会话，不保存原始会话标识；如果同一回合的用量冲突，则不会提交受影响的文件，直到文件发生变化或再次运行 CLI 同步。 删除 Grok Build Provider 也会删除已保存的用量报告、导入进度和同步设置。桌面端后台同步不会重新创建已删除的 Provider；再次运行这条 CLI 同步属于明确的用户操作，它可以重新建立 Provider，并重新导入本地会话文件中仍然存在的用量。 diff --git a/internal/bootstrap/service_test.go b/internal/bootstrap/service_test.go index bab689f..43de129 100644 --- a/internal/bootstrap/service_test.go +++ b/internal/bootstrap/service_test.go @@ -186,7 +186,8 @@ func TestInitializeBacksUpValidatedMarkerGapBeforeReapplyingMarker(t *testing.T) } snapshot := inspectDatabaseSnapshot(t, runtimeService.Paths().Database) if len(snapshot.markers) != 0 || - !snapshot.usageTable || snapshot.grokUsageTable || !snapshot.pathKeyIndex { + !snapshot.usageTable || snapshot.grokUsageTable || + snapshot.usageObservationTable || !snapshot.pathKeyIndex { t.Fatalf("database changed before backup: %#v", snapshot) } }, @@ -202,7 +203,8 @@ func TestInitializeBacksUpValidatedMarkerGapBeforeReapplyingMarker(t *testing.T) } snapshot := inspectDatabaseSnapshot(t, runtimeService.Paths().Database) if len(snapshot.markers) != migrationCount || - !snapshot.usageTable || !snapshot.grokUsageTable || !snapshot.pathKeyIndex || + !snapshot.usageTable || !snapshot.grokUsageTable || + !snapshot.usageObservationTable || !snapshot.pathKeyIndex || snapshot.setting != `{"kept":true}` { t.Fatalf("database after upgrade = %#v", snapshot) } @@ -289,10 +291,10 @@ func TestInitializeRejectsMarkerGapSchemaDriftBeforeBackup(t *testing.T) { } } -func TestInitializeBacksUpStableBaselineBeforeGrokBuildUsageMigration(t *testing.T) { +func TestInitializeBacksUpPreviousBaselineBeforeIncrementalUsageMigration(t *testing.T) { ctx := context.Background() runtimeService := newRuntimeService(t) - createPreviousStableBaseline(t, ctx, runtimeService) + createPreviousBaseline(t, ctx, runtimeService) insertSetting(t, ctx, runtimeService.Paths().Database, "upgrade-data", `{"kept":true}`) backups := &recordingBackupCreator{ inspect: func(req appbackup.CreateRequest) { @@ -301,7 +303,8 @@ func TestInitializeBacksUpStableBaselineBeforeGrokBuildUsageMigration(t *testing } snapshot := inspectDatabaseSnapshot(t, runtimeService.Paths().Database) if len(snapshot.markers) != len(storemigrations.Migrations.Sorted())-1 || - !snapshot.usageTable || snapshot.grokUsageTable || !snapshot.pathKeyIndex || + !snapshot.usageTable || !snapshot.grokUsageTable || + snapshot.usageObservationTable || !snapshot.pathKeyIndex || snapshot.setting != `{"kept":true}` { t.Fatalf("stable baseline changed before backup: %#v", snapshot) } @@ -310,14 +313,15 @@ func TestInitializeBacksUpStableBaselineBeforeGrokBuildUsageMigration(t *testing result, err := NewService(runtimeService, backups, nil).Initialize(ctx) if err != nil { - t.Fatalf("apply Grok Build usage migration: %v", err) + t.Fatalf("apply incremental usage migration: %v", err) } if result.MigrationsApplied != 1 || backups.calls != 1 { t.Fatalf("upgrade result = %#v, backups = %d", result, backups.calls) } snapshot := inspectDatabaseSnapshot(t, runtimeService.Paths().Database) if len(snapshot.markers) != len(storemigrations.Migrations.Sorted()) || - !snapshot.usageTable || !snapshot.grokUsageTable || !snapshot.pathKeyIndex || + !snapshot.usageTable || !snapshot.grokUsageTable || + !snapshot.usageObservationTable || !snapshot.pathKeyIndex || snapshot.setting != `{"kept":true}` { t.Fatalf("database after upgrade = %#v", snapshot) } @@ -377,12 +381,13 @@ func (creator *recordingBackupCreator) Create( } type databaseSnapshot struct { - schemaVersion int - markers []string - usageTable bool - grokUsageTable bool - pathKeyIndex bool - setting string + schemaVersion int + markers []string + usageTable bool + grokUsageTable bool + usageObservationTable bool + pathKeyIndex bool + setting string } func newRuntimeService(t *testing.T) *runtime.Service { @@ -408,25 +413,48 @@ func createStableMarkerGap(t *testing.T, ctx context.Context, runtimeService *ru if len(registered) < 2 { t.Fatalf("registered migrations = %d, want at least 2", len(registered)) } + dropIncrementalUsageSchema(t, runtimeService.Paths().Database) execDatabaseStatements(t, runtimeService.Paths().Database, `DROP TABLE grok_build_usage_import_files`, `DELETE FROM bun_migrations`, ) } -func createPreviousStableBaseline(t *testing.T, ctx context.Context, runtimeService *runtime.Service) { +func createPreviousBaseline(t *testing.T, ctx context.Context, runtimeService *runtime.Service) { t.Helper() createCurrentBaseline(t, ctx, runtimeService) registered := storemigrations.Migrations.Sorted() if len(registered) < 2 { t.Fatalf("registered migrations = %d, want at least 2", len(registered)) } + dropIncrementalUsageSchema(t, runtimeService.Paths().Database) execDatabaseStatements(t, runtimeService.Paths().Database, - `DROP TABLE grok_build_usage_import_files`, `DELETE FROM bun_migrations WHERE name = '`+registered[len(registered)-1].Name+`'`, ) } +func dropIncrementalUsageSchema(t *testing.T, path string) { + t.Helper() + execDatabaseStatements(t, path, + `DROP TABLE usage_import_observations`, + `ALTER TABLE codex_usage_import_files DROP COLUMN parser_state_json`, + `ALTER TABLE codex_usage_import_files DROP COLUMN checkpoint_event_digest`, + `ALTER TABLE codex_usage_import_files DROP COLUMN boundary_digest`, + `ALTER TABLE codex_usage_import_files DROP COLUMN file_identity_digest`, + `ALTER TABLE codex_usage_import_files DROP COLUMN metadata_digest`, + `ALTER TABLE codex_usage_import_files DROP COLUMN processed_bytes`, + `ALTER TABLE codex_usage_import_files DROP COLUMN checkpoint_revision`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN parser_state_json`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN checkpoint_event_digest`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN boundary_digest`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN file_identity_digest`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN metadata_digest`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN processed_bytes`, + `ALTER TABLE grok_build_usage_import_files DROP COLUMN checkpoint_revision`, + `ALTER TABLE usage_sources DROP COLUMN completed_generation`, + ) +} + func insertSetting(t *testing.T, ctx context.Context, path, key, value string) { t.Helper() db, err := store.Open(ctx, path, false) @@ -479,6 +507,14 @@ func inspectDatabaseSnapshot(t *testing.T, path string) databaseSnapshot { t.Fatal(err) } snapshot.grokUsageTable = grokUsageCount > 0 + var usageObservationCount int + if err := db.QueryRow(` + SELECT COUNT(1) FROM sqlite_master + WHERE type = 'table' AND name = 'usage_import_observations' + `).Scan(&usageObservationCount); err != nil { + t.Fatal(err) + } + snapshot.usageObservationTable = usageObservationCount > 0 var pathKeyIndexCount int if err := db.QueryRow(`SELECT COUNT(1) FROM sqlite_master WHERE type = 'index' AND name = 'idx_profile_targets_path_key'`).Scan(&pathKeyIndexCount); err != nil { t.Fatal(err) diff --git a/internal/cli/command_test.go b/internal/cli/command_test.go index 9af9dab..7c3e5b9 100644 --- a/internal/cli/command_test.go +++ b/internal/cli/command_test.go @@ -720,7 +720,7 @@ func TestUsageSyncGrokBuildUsesGlobalHomeAndOmitsFileIdentifiers(t *testing.T) { configDir := t.TempDir() grokHome := t.TempDir() writeCLIGrokBuildUsageFixture(t, grokHome, "workspace", "valid", `{"timestamp":1750000000,"method":"_x.ai/session/update","params":{"sessionId":"synthetic-session","update":{"sessionUpdate":"turn_completed","prompt_id":"synthetic-prompt","stop_reason":"end_turn","agent_result":"discarded synthetic result","usage":{"inputTokens":100,"outputTokens":20,"totalTokens":120,"cachedReadTokens":40,"reasoningTokens":5,"modelCalls":1,"apiDurationMs":10,"costUsdTicks":999,"costIsPartial":false,"modelUsage":{"grok-build-latest":{"inputTokens":100,"outputTokens":20,"totalTokens":120,"cachedReadTokens":40,"reasoningTokens":5,"modelCalls":1,"apiDurationMs":10,"costUsdTicks":999,"costIsPartial":false}},"numTurns":1}}}}`) - writeCLIGrokBuildUsageFixture(t, grokHome, "workspace", "malformed", `{"timestamp":`) + writeCLIGrokBuildUsageFixture(t, grokHome, "workspace", "malformed", "{\"timestamp\":\n") base := []string{"--config-dir", configDir, "--grok-home", grokHome} if _, err := runCLI(t, append(base, "init", "--json")...); err != nil { diff --git a/internal/store/integrity.go b/internal/store/integrity.go index 5a6b15a..7eb5cf5 100644 --- a/internal/store/integrity.go +++ b/internal/store/integrity.go @@ -235,7 +235,54 @@ var schemaContracts = func() []schemaContract { AND usage_sources.identity_revision = value.identity_revision )`, ) - return []schemaContract{stable, grokBuild} + incremental := grokBuild + incremental.migrationKey = "usage_incremental_checkpoint" + incremental.tableSpecs = replaceTableSpec(incremental.tableSpecs, usageIncrementalSourceTableSpec) + incremental.tableSpecs = replaceTableSpec(incremental.tableSpecs, usageIncrementalCodexImportTableSpec) + incremental.tableSpecs = replaceTableSpec(incremental.tableSpecs, usageIncrementalGrokBuildImportTableSpec) + incremental.tableSpecs = append(incremental.tableSpecs, usageImportObservationTableSpec) + incremental.jsonQueries = append( + append([]string(nil), grokBuild.jsonQueries...), + `SELECT COUNT(1) FROM codex_usage_import_files + WHERE NOT (`+jsonObjectExpression("parser_state_json")+`)`, + `SELECT COUNT(1) FROM grok_build_usage_import_files + WHERE NOT (`+jsonObjectExpression("parser_state_json")+`)`, + ) + incremental.referenceQueries = append( + append([]string(nil), grokBuild.referenceQueries...), + `SELECT COUNT(1) FROM usage_import_observations AS value + WHERE NOT EXISTS ( + SELECT 1 FROM usage_sources WHERE usage_sources.id = value.source_id + )`, + ) + incremental.stateQueries = append( + append([]string(nil), grokBuild.stateQueries...), + `SELECT COUNT(1) FROM usage_sources + WHERE completed_generation < 0 OR completed_generation > sync_generation`, + `SELECT COUNT(1) FROM codex_usage_import_files + WHERE checkpoint_revision NOT IN (0, 1) + OR (checkpoint_revision = 0 AND ( + processed_bytes <> 0 OR metadata_digest <> zeroblob(32) + OR file_identity_digest <> zeroblob(32) OR boundary_digest <> zeroblob(32) + OR checkpoint_event_digest <> zeroblob(32) OR parser_state_json <> '{}' + )) + OR (checkpoint_revision = 1 AND ( + metadata_digest = zeroblob(32) OR boundary_digest = zeroblob(32) + OR checkpoint_event_digest = zeroblob(32) + ))`, + `SELECT COUNT(1) FROM grok_build_usage_import_files + WHERE checkpoint_revision NOT IN (0, 1) + OR (checkpoint_revision = 0 AND ( + processed_bytes <> 0 OR metadata_digest <> zeroblob(32) + OR file_identity_digest <> zeroblob(32) OR boundary_digest <> zeroblob(32) + OR checkpoint_event_digest <> zeroblob(32) OR parser_state_json <> '{}' + )) + OR (checkpoint_revision = 1 AND ( + metadata_digest = zeroblob(32) OR boundary_digest = zeroblob(32) + OR checkpoint_event_digest = zeroblob(32) OR parser_state_json <> '{}' + ))`, + ) + return []schemaContract{stable, grokBuild, incremental} }() func jsonObjectExpression(column string) string { @@ -393,6 +440,17 @@ func (s *Store) InspectIntegrity(ctx context.Context, scope IntegrityScope) (Int if err != nil { return IntegrityReport{}, err } + if scope == IntegrityAppliedBaseline && len(issues) != 0 && state.Pending > 0 { + nextContract, nextErr := schemaContractForApplied(state.Applied + 1) + if nextErr != nil { + return IntegrityReport{}, nextErr + } + if replayIssues, replayErr := s.inspectContractIntegrity(ctx, nextContract); replayErr != nil { + return IntegrityReport{}, replayErr + } else if len(replayIssues) == 0 { + issues = nil + } + } if scope == IntegrityCurrentBaseline && !state.Current { issues = addIntegrityIssue(issues, IntegrityIssueSchema, 1) } diff --git a/internal/store/migrations/202609110001_usage_incremental_checkpoint.go b/internal/store/migrations/202609110001_usage_incremental_checkpoint.go new file mode 100644 index 0000000..25d1afd --- /dev/null +++ b/internal/store/migrations/202609110001_usage_incremental_checkpoint.go @@ -0,0 +1,144 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/uptrace/bun" +) + +const zeroUsageDigestSQL = "X'0000000000000000000000000000000000000000000000000000000000000000'" + +func init() { + Migrations.MustRegister(upUsageIncrementalCheckpoint, downUsageIncrementalCheckpoint) +} + +func upUsageIncrementalCheckpoint(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return applyUsageIncrementalCheckpoint(ctx, tx) + }) +} + +func applyUsageIncrementalCheckpoint(ctx context.Context, db bun.IDB) error { + completedGenerationExists, err := usageColumnExists(ctx, db, "usage_sources", "completed_generation") + if err != nil { + return err + } + if !completedGenerationExists { + if err := addUsageColumnIfMissing(ctx, db, "usage_sources", "completed_generation", + "INTEGER NOT NULL DEFAULT 0 CHECK (completed_generation >= 0 AND completed_generation <= sync_generation)"); err != nil { + return err + } + if _, err := db.ExecContext(ctx, ` + UPDATE usage_sources + SET completed_generation = sync_generation + `); err != nil { + return err + } + } + + for _, table := range []string{"codex_usage_import_files", "grok_build_usage_import_files"} { + columns := []struct { + name string + definition string + }{ + {"checkpoint_revision", "INTEGER NOT NULL DEFAULT 0 CHECK (checkpoint_revision >= 0)"}, + {"processed_bytes", "INTEGER NOT NULL DEFAULT 0 CHECK (processed_bytes >= 0 AND processed_bytes <= size_bytes)"}, + {"metadata_digest", "BLOB NOT NULL DEFAULT " + zeroUsageDigestSQL + " CHECK (typeof(metadata_digest) = 'blob' AND length(metadata_digest) = 32)"}, + {"file_identity_digest", "BLOB NOT NULL DEFAULT " + zeroUsageDigestSQL + " CHECK (typeof(file_identity_digest) = 'blob' AND length(file_identity_digest) = 32)"}, + {"boundary_digest", "BLOB NOT NULL DEFAULT " + zeroUsageDigestSQL + " CHECK (typeof(boundary_digest) = 'blob' AND length(boundary_digest) = 32)"}, + {"checkpoint_event_digest", "BLOB NOT NULL DEFAULT " + zeroUsageDigestSQL + " CHECK (typeof(checkpoint_event_digest) = 'blob' AND length(checkpoint_event_digest) = 32)"}, + {"parser_state_json", "TEXT NOT NULL DEFAULT '{}' CHECK (CASE WHEN json_valid(parser_state_json) THEN json_type(parser_state_json) = 'object' AND length(CAST(parser_state_json AS BLOB)) <= 1048576 ELSE 0 END)"}, + } + for _, column := range columns { + if err := addUsageColumnIfMissing(ctx, db, table, column.name, column.definition); err != nil { + return err + } + } + } + + _, err = db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS usage_import_observations ( + source_id INTEGER NOT NULL, + file_key BLOB NOT NULL CHECK ( + typeof(file_key) = 'blob' AND length(file_key) = 32 AND file_key <> zeroblob(32) + ), + metadata_digest BLOB NOT NULL CHECK ( + typeof(metadata_digest) = 'blob' AND length(metadata_digest) = 32 AND metadata_digest <> zeroblob(32) + ), + status TEXT NOT NULL CHECK (status IN ('history_changed', 'unavailable', 'fact_conflict')), + updated_at_unix_ms INTEGER NOT NULL CHECK (updated_at_unix_ms >= 0), + PRIMARY KEY (source_id, file_key), + FOREIGN KEY (source_id) REFERENCES usage_sources(id) + ON UPDATE RESTRICT ON DELETE CASCADE + ) STRICT, WITHOUT ROWID`) + return err +} + +func downUsageIncrementalCheckpoint(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + if _, err := tx.ExecContext(ctx, `DROP TABLE IF EXISTS usage_import_observations`); err != nil { + return err + } + for _, table := range []string{"grok_build_usage_import_files", "codex_usage_import_files"} { + for _, column := range []string{ + "parser_state_json", + "checkpoint_event_digest", + "boundary_digest", + "file_identity_digest", + "metadata_digest", + "processed_bytes", + "checkpoint_revision", + } { + if err := dropUsageColumnIfExists(ctx, tx, table, column); err != nil { + return err + } + } + } + return dropUsageColumnIfExists(ctx, tx, "usage_sources", "completed_generation") + }) +} + +func addUsageColumnIfMissing( + ctx context.Context, + db bun.IDB, + table string, + column string, + definition string, +) error { + exists, err := usageColumnExists(ctx, db, table, column) + if err != nil || exists { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, definition)) + return err +} + +func dropUsageColumnIfExists(ctx context.Context, db bun.IDB, table, column string) error { + exists, err := usageColumnExists(ctx, db, table, column) + if err != nil || !exists { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", table, column)) + return err +} + +func usageColumnExists(ctx context.Context, db bun.IDB, table, column string) (bool, error) { + rows, err := db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var cid, notNull, primaryKey int + var name, columnType string + var defaultValue sql.NullString + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} diff --git a/internal/store/migrations/migrations_test.go b/internal/store/migrations/migrations_test.go index 0d50939..dde4b5e 100644 --- a/internal/store/migrations/migrations_test.go +++ b/internal/store/migrations/migrations_test.go @@ -38,3 +38,133 @@ func TestExecStatementsRollsBackWholeMigrationCallback(t *testing.T) { t.Fatalf("failed migration left %d partial tables", count) } } + +func TestUsageIncrementalCheckpointRollsBackPartialUpgrade(t *testing.T) { + ctx := context.Background() + sqlDB, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "migration.db")) + if err != nil { + t.Fatal(err) + } + db := bun.NewDB(sqlDB, sqlitedialect.New()) + defer db.Close() + if err := upStableBaseline(ctx, db); err != nil { + t.Fatalf("create stable baseline: %v", err) + } + if err := upGrokBuildUsageImport(ctx, db); err != nil { + t.Fatalf("create Grok Build baseline: %v", err) + } + if _, err := sqlDB.ExecContext(ctx, `DROP TABLE grok_build_usage_import_files`); err != nil { + t.Fatalf("damage migration prerequisite: %v", err) + } + + if err := upUsageIncrementalCheckpoint(ctx, db); err == nil { + t.Fatal("incremental checkpoint migration unexpectedly succeeded") + } + for table, column := range map[string]string{ + "usage_sources": "completed_generation", + "codex_usage_import_files": "checkpoint_revision", + } { + exists, err := usageColumnExists(ctx, db, table, column) + if err != nil { + t.Fatalf("inspect %s.%s: %v", table, column, err) + } + if exists { + t.Fatalf("failed migration retained %s.%s", table, column) + } + } + var observations int + if err := sqlDB.QueryRowContext(ctx, ` + SELECT COUNT(1) FROM sqlite_master + WHERE type = 'table' AND name = 'usage_import_observations' + `).Scan(&observations); err != nil { + t.Fatal(err) + } + if observations != 0 { + t.Fatal("failed migration retained usage_import_observations") + } +} + +func TestUsageIncrementalCheckpointPreservesCompletedLegacyProgress(t *testing.T) { + ctx := context.Background() + sqlDB, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "migration.db")) + if err != nil { + t.Fatal(err) + } + db := bun.NewDB(sqlDB, sqlitedialect.New()) + defer db.Close() + if err := upStableBaseline(ctx, db); err != nil { + t.Fatalf("create stable baseline: %v", err) + } + if err := upGrokBuildUsageImport(ctx, db); err != nil { + t.Fatalf("create Grok Build baseline: %v", err) + } + if _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO providers (id, name, adapter_id, created_at_unix_ms, updated_at_unix_ms) + VALUES ('codex', 'Codex', 'codex', 1, 1) + `); err != nil { + t.Fatalf("create legacy Provider: %v", err) + } + if _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO usage_sources ( + provider_id, source_key, identity_revision, sync_generation, + last_completed_at_unix_ms + ) VALUES ('codex', 'codex-session-jsonl', 2, 7, 100) + `); err != nil { + t.Fatalf("create legacy source: %v", err) + } + if _, err := sqlDB.ExecContext(ctx, ` + INSERT INTO codex_usage_import_files ( + source_id, file_key, parser_revision, identity_revision, + event_digest, updated_at_unix_ms + ) VALUES ( + 1, + X'0101010101010101010101010101010101010101010101010101010101010101', + 1, + 2, + X'0202020202020202020202020202020202020202020202020202020202020202', + 100 + ) + `); err != nil { + t.Fatalf("create legacy cursor: %v", err) + } + + if err := upUsageIncrementalCheckpoint(ctx, db); err != nil { + t.Fatalf("migrate incremental checkpoints: %v", err) + } + var completedGeneration, checkpointRevision, processedBytes int64 + var parserState string + if err := sqlDB.QueryRowContext(ctx, ` + SELECT s.completed_generation, f.checkpoint_revision, + f.processed_bytes, f.parser_state_json + FROM usage_sources AS s + JOIN codex_usage_import_files AS f ON f.source_id = s.id + `).Scan(&completedGeneration, &checkpointRevision, &processedBytes, &parserState); err != nil { + t.Fatalf("read migrated progress: %v", err) + } + if completedGeneration != 7 || checkpointRevision != 0 || processedBytes != 0 || parserState != "{}" { + t.Fatalf( + "migrated progress = completed %d checkpoint %d bytes %d state %q", + completedGeneration, + checkpointRevision, + processedBytes, + parserState, + ) + } + if _, err := sqlDB.ExecContext(ctx, ` + UPDATE usage_sources + SET sync_generation = 8, completed_generation = 7 + `); err != nil { + t.Fatalf("create interrupted current generation: %v", err) + } + if err := upUsageIncrementalCheckpoint(ctx, db); err != nil { + t.Fatalf("replay incremental checkpoint migration: %v", err) + } + if err := sqlDB.QueryRowContext(ctx, ` + SELECT completed_generation FROM usage_sources + `).Scan(&completedGeneration); err != nil { + t.Fatalf("read replayed progress: %v", err) + } + if completedGeneration != 7 { + t.Fatalf("migration replay completed interrupted generation: got %d", completedGeneration) + } +} diff --git a/internal/store/schema_contract.go b/internal/store/schema_contract.go index 978022c..08034a8 100644 --- a/internal/store/schema_contract.go +++ b/internal/store/schema_contract.go @@ -421,6 +421,95 @@ var grokBuildUsageImportTableSpec = tableSpec{ }, } +var usageIncrementalSourceTableSpec = func() tableSpec { + spec := copyTableSpec(stableBaselineTableSpec("usage_sources")) + spec.columns = append(spec.columns, + columnSpec{name: "completed_generation", columnType: "INTEGER", notNull: true, requireDefault: true, defaultValue: "0"}, + ) + spec.checks = append(spec.checks, + "CHECK (completed_generation >= 0 AND completed_generation <= sync_generation)", + ) + return spec +}() + +var usageIncrementalCodexImportTableSpec = extendUsageImportTableSpec( + stableBaselineTableSpec("codex_usage_import_files"), +) + +var usageIncrementalGrokBuildImportTableSpec = extendUsageImportTableSpec( + grokBuildUsageImportTableSpec, +) + +var usageImportObservationTableSpec = tableSpec{ + name: "usage_import_observations", + strict: true, + columns: []columnSpec{ + {name: "source_id", columnType: "INTEGER", notNull: true, primaryKey: true}, + {name: "file_key", columnType: "BLOB", notNull: true, primaryKey: true}, + {name: "metadata_digest", columnType: "BLOB", notNull: true}, + {name: "status", columnType: "TEXT", notNull: true}, + {name: "updated_at_unix_ms", columnType: "INTEGER", notNull: true}, + }, + checks: []string{ + "CHECK (typeof(file_key) = 'blob' AND length(file_key) = 32 AND file_key <> zeroblob(32))", + "CHECK (typeof(metadata_digest) = 'blob' AND length(metadata_digest) = 32 AND metadata_digest <> zeroblob(32))", + "CHECK (status IN ('history_changed', 'unavailable', 'fact_conflict'))", + "CHECK (updated_at_unix_ms >= 0)", + "FOREIGN KEY (source_id) REFERENCES usage_sources(id) ON UPDATE RESTRICT ON DELETE CASCADE", + "WITHOUT ROWID", + }, +} + +func stableBaselineTableSpec(name string) tableSpec { + for _, spec := range stableBaselineTableSpecs { + if spec.name == name { + return spec + } + } + panic("missing stable baseline table spec: " + name) +} + +func copyTableSpec(spec tableSpec) tableSpec { + spec.columns = append([]columnSpec(nil), spec.columns...) + spec.checks = append([]string(nil), spec.checks...) + return spec +} + +func extendUsageImportTableSpec(spec tableSpec) tableSpec { + spec = copyTableSpec(spec) + spec.columns = append(spec.columns, + columnSpec{name: "checkpoint_revision", columnType: "INTEGER", notNull: true, requireDefault: true, defaultValue: "0"}, + columnSpec{name: "processed_bytes", columnType: "INTEGER", notNull: true, requireDefault: true, defaultValue: "0"}, + columnSpec{name: "metadata_digest", columnType: "BLOB", notNull: true, requireDefault: true, defaultValue: "X'0000000000000000000000000000000000000000000000000000000000000000'"}, + columnSpec{name: "file_identity_digest", columnType: "BLOB", notNull: true, requireDefault: true, defaultValue: "X'0000000000000000000000000000000000000000000000000000000000000000'"}, + columnSpec{name: "boundary_digest", columnType: "BLOB", notNull: true, requireDefault: true, defaultValue: "X'0000000000000000000000000000000000000000000000000000000000000000'"}, + columnSpec{name: "checkpoint_event_digest", columnType: "BLOB", notNull: true, requireDefault: true, defaultValue: "X'0000000000000000000000000000000000000000000000000000000000000000'"}, + columnSpec{name: "parser_state_json", columnType: "TEXT", notNull: true, requireDefault: true, defaultValue: "'{}'"}, + ) + spec.checks = append(spec.checks, + "CHECK (checkpoint_revision >= 0)", + "CHECK (processed_bytes >= 0 AND processed_bytes <= size_bytes)", + "CHECK (typeof(metadata_digest) = 'blob' AND length(metadata_digest) = 32)", + "CHECK (typeof(file_identity_digest) = 'blob' AND length(file_identity_digest) = 32)", + "CHECK (typeof(boundary_digest) = 'blob' AND length(boundary_digest) = 32)", + "CHECK (typeof(checkpoint_event_digest) = 'blob' AND length(checkpoint_event_digest) = 32)", + "CHECK (CASE WHEN json_valid(parser_state_json) THEN json_type(parser_state_json) = 'object' AND length(CAST(parser_state_json AS BLOB)) <= 1048576 ELSE 0 END)", + ) + return spec +} + +func replaceTableSpec(specs []tableSpec, replacement tableSpec) []tableSpec { + result := make([]tableSpec, len(specs)) + copy(result, specs) + for index := range result { + if result[index].name == replacement.name { + result[index] = replacement + return result + } + } + panic("missing table spec replacement: " + replacement.name) +} + var stableBaselineIndexSpecs = []indexSpec{ {name: "idx_providers_adapter_id", table: "providers", columns: []string{"adapter_id"}}, {name: "idx_provider_profile_settings_provider_id", table: "provider_profile_settings", columns: []string{"provider_id"}}, diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 3faae40..3171dae 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -233,6 +233,27 @@ func TestAppliedSchemaUsesItsVersionedContract(t *testing.T) { if _, err := db.db.DB.ExecContext(ctx, `DELETE FROM bun_migrations`); err != nil { t.Fatalf("remove Stable marker: %v", err) } + if _, err := db.db.DB.ExecContext(ctx, `DROP TABLE usage_import_observations`); err != nil { + t.Fatalf("remove incremental usage observation table: %v", err) + } + for _, table := range []string{"codex_usage_import_files", "grok_build_usage_import_files"} { + for _, column := range []string{ + "parser_state_json", + "checkpoint_event_digest", + "boundary_digest", + "file_identity_digest", + "metadata_digest", + "processed_bytes", + "checkpoint_revision", + } { + if _, err := db.db.DB.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", table, column)); err != nil { + t.Fatalf("remove incremental usage column %s.%s: %v", table, column, err) + } + } + } + if _, err := db.db.DB.ExecContext(ctx, `ALTER TABLE usage_sources DROP COLUMN completed_generation`); err != nil { + t.Fatalf("remove usage completion generation: %v", err) + } if _, err := db.db.DB.ExecContext(ctx, `DROP TABLE grok_build_usage_import_files`); err != nil { t.Fatalf("restore unmarked Stable schema: %v", err) } diff --git a/internal/store/usage.go b/internal/store/usage.go index b92c5fa..de994a0 100644 --- a/internal/store/usage.go +++ b/internal/store/usage.go @@ -11,7 +11,10 @@ import ( "time" ) -const usageKeySize = 32 +const ( + usageKeySize = 32 + maxUsageParserStateSize = 1024 * 1024 +) // UsageUnknownModelKey is the persisted label for absent or unsafe models. const UsageUnknownModelKey = "unknown" @@ -97,6 +100,7 @@ type UsageSource struct { SourceKey string IdentityRevision int64 SyncGeneration int64 + CompletedGeneration int64 LastCompletedAtUnixMS int64 TrackedUnits int64 InvalidRecords int64 @@ -133,6 +137,23 @@ type CompleteUsageSyncParams struct { Generation int64 CompletedAtUnixMS int64 Finalization UsageSyncFinalization + Observations []UsageImportObservation +} + +type UsageImportObservationStatus string + +const ( + UsageImportObservationHistoryChanged UsageImportObservationStatus = "history_changed" + UsageImportObservationUnavailable UsageImportObservationStatus = "unavailable" + UsageImportObservationFactConflict UsageImportObservationStatus = "fact_conflict" +) + +type UsageImportObservation struct { + SourceID int64 + FileKey UsageKey + MetadataDigest UsageKey + Status UsageImportObservationStatus + UpdatedAtUnixMS int64 } // UsageSyncFinalization is sealed so provider-specific checkpoints remain @@ -302,7 +323,8 @@ func (s *Store) GetUsageSource(ctx context.Context, providerID, sourceKey string func (s *Store) getUsageSource(ctx context.Context, providerID, sourceKey string) (UsageSource, error) { row := s.executor().QueryRowContext(ctx, ` SELECT id, provider_id, source_key, identity_revision, sync_generation, - last_completed_at_unix_ms, tracked_units, invalid_records, unsupported_records + completed_generation, last_completed_at_unix_ms, tracked_units, + invalid_records, unsupported_records FROM usage_sources WHERE provider_id = ? AND source_key = ? `, providerID, sourceKey) @@ -312,7 +334,8 @@ func (s *Store) getUsageSource(ctx context.Context, providerID, sourceKey string func (s *Store) getUsageSourceByID(ctx context.Context, sourceID int64) (UsageSource, error) { row := s.executor().QueryRowContext(ctx, ` SELECT id, provider_id, source_key, identity_revision, sync_generation, - last_completed_at_unix_ms, tracked_units, invalid_records, unsupported_records + completed_generation, last_completed_at_unix_ms, tracked_units, + invalid_records, unsupported_records FROM usage_sources WHERE id = ? `, sourceID) @@ -327,6 +350,7 @@ func scanUsageSource(row rowScanner) (UsageSource, error) { &source.SourceKey, &source.IdentityRevision, &source.SyncGeneration, + &source.CompletedGeneration, &source.LastCompletedAtUnixMS, &source.TrackedUnits, &source.InvalidRecords, @@ -713,6 +737,9 @@ func (s *Store) completeUsageSync(ctx context.Context, params CompleteUsageSyncP if result.trackedUnits < 0 || result.invalidRecords < 0 || result.unsupportedRecords < 0 { return errors.New("usage sync finalization is invalid") } + if err := s.upsertUsageImportObservations(ctx, source, params.Observations); err != nil { + return err + } return s.updateUsageSyncCompletion(ctx, params, result) } @@ -743,6 +770,13 @@ func validateUsageSyncCompletion(params CompleteUsageSyncParams) error { if err := params.Finalization.validateUsageSyncFinalization(); err != nil { return err } + for _, observation := range params.Observations { + if observation.SourceID != params.SourceID || observation.FileKey.IsZero() || + observation.MetadataDigest.IsZero() || !observation.Status.valid() || + observation.UpdatedAtUnixMS < 0 { + return errors.New("usage import observation is invalid") + } + } return nil } @@ -757,10 +791,10 @@ func (s *Store) updateUsageSyncCompletion( } update, err := s.executor().ExecContext(ctx, ` UPDATE usage_sources - SET last_completed_at_unix_ms = ?, tracked_units = ?, + SET completed_generation = ?, last_completed_at_unix_ms = ?, tracked_units = ?, invalid_records = ?, unsupported_records = ? WHERE id = ? AND sync_generation = ? - `, completedAt, result.trackedUnits, result.invalidRecords, result.unsupportedRecords, params.SourceID, params.Generation) + `, params.Generation, completedAt, result.trackedUnits, result.invalidRecords, result.unsupportedRecords, params.SourceID, params.Generation) if err != nil { return err } @@ -774,6 +808,134 @@ func (s *Store) updateUsageSyncCompletion( return nil } +func (s *Store) ListUsageImportObservations(ctx context.Context, sourceID int64) ([]UsageImportObservation, error) { + if sourceID <= 0 { + return nil, errors.New("usage import observation query is invalid") + } + rows, err := s.executor().QueryContext(ctx, ` + SELECT source_id, file_key, metadata_digest, status, updated_at_unix_ms + FROM usage_import_observations + WHERE source_id = ? + ORDER BY file_key + `, sourceID) + if err != nil { + return nil, err + } + defer rows.Close() + observations := make([]UsageImportObservation, 0) + for rows.Next() { + var observation UsageImportObservation + if err := rows.Scan( + &observation.SourceID, + &observation.FileKey, + &observation.MetadataDigest, + &observation.Status, + &observation.UpdatedAtUnixMS, + ); err != nil { + return nil, err + } + observations = append(observations, observation) + } + return observations, rows.Err() +} + +func (s *Store) upsertUsageImportObservations( + ctx context.Context, + source UsageSource, + observations []UsageImportObservation, +) error { + if len(observations) == 0 { + return nil + } + stmt, err := s.executor().PrepareContext(ctx, ` + INSERT INTO usage_import_observations ( + source_id, file_key, metadata_digest, status, updated_at_unix_ms + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_id, file_key) DO UPDATE SET + metadata_digest = excluded.metadata_digest, + status = excluded.status, + updated_at_unix_ms = excluded.updated_at_unix_ms + `) + if err != nil { + return err + } + defer stmt.Close() + now := time.Now().UnixMilli() + for _, observation := range observations { + if observation.SourceID != source.ID || observation.FileKey.IsZero() || + observation.MetadataDigest.IsZero() || !observation.Status.valid() || + observation.UpdatedAtUnixMS < 0 { + return errors.New("usage import observation is invalid") + } + updatedAt := observation.UpdatedAtUnixMS + if updatedAt == 0 { + updatedAt = now + } + if _, err := stmt.ExecContext( + ctx, + observation.SourceID, + observation.FileKey, + observation.MetadataDigest, + observation.Status, + updatedAt, + ); err != nil { + return err + } + } + return nil +} + +func (status UsageImportObservationStatus) valid() bool { + switch status { + case UsageImportObservationHistoryChanged, + UsageImportObservationUnavailable, + UsageImportObservationFactConflict: + return true + default: + return false + } +} + +func (s *Store) deleteUsageImportObservation(ctx context.Context, sourceID int64, fileKey UsageKey) error { + _, err := s.executor().ExecContext(ctx, ` + DELETE FROM usage_import_observations WHERE source_id = ? AND file_key = ? + `, sourceID, fileKey) + return err +} + +func (s *Store) deleteMissingUsageImportObservations( + ctx context.Context, + sourceID int64, + discovered map[UsageKey]struct{}, +) error { + rows, err := s.executor().QueryContext(ctx, ` + SELECT file_key FROM usage_import_observations WHERE source_id = ? + `, sourceID) + if err != nil { + return err + } + var stale []UsageKey + for rows.Next() { + var fileKey UsageKey + if err := rows.Scan(&fileKey); err != nil { + _ = rows.Close() + return err + } + if _, ok := discovered[fileKey]; !ok { + stale = append(stale, fileKey) + } + } + if err := rows.Close(); err != nil { + return err + } + for _, fileKey := range stale { + if err := s.deleteUsageImportObservation(ctx, sourceID, fileKey); err != nil { + return err + } + } + return nil +} + // ListUnknownUsageCostModels returns one dimension row per model that still has // unknown costs. The owning Integration decides which labels it can price. func (s *Store) ListUnknownUsageCostModels(ctx context.Context, providerID string) ([]UsageUnknownCostModel, error) { diff --git a/internal/store/usage_codex.go b/internal/store/usage_codex.go index 7d2d462..88f73c4 100644 --- a/internal/store/usage_codex.go +++ b/internal/store/usage_codex.go @@ -4,23 +4,31 @@ import ( "context" "database/sql" "errors" + "strings" "time" ) const codexUsageProviderID = "codex" type CodexUsageImportFile struct { - SourceID int64 - FileKey UsageKey - ModifiedUnixMS int64 - SizeBytes int64 - ImportedFacts int64 - InvalidLines int64 - UnsupportedLines int64 - ParserRevision int64 - IdentityRevision int64 - EventDigest UsageKey - UpdatedAtUnixMS int64 + SourceID int64 + FileKey UsageKey + ModifiedUnixMS int64 + SizeBytes int64 + ImportedFacts int64 + InvalidLines int64 + UnsupportedLines int64 + ParserRevision int64 + IdentityRevision int64 + EventDigest UsageKey + CheckpointRevision int64 + ProcessedBytes int64 + MetadataDigest UsageKey + FileIdentityDigest UsageKey + BoundaryDigest UsageKey + CheckpointEventDigest UsageKey + ParserStateJSON string + UpdatedAtUnixMS int64 } type CommitCodexUsageImportParams struct { @@ -80,7 +88,9 @@ func (s *Store) GetCodexUsageImportFile(ctx context.Context, sourceID int64, fil row := s.executor().QueryRowContext(ctx, ` SELECT source_id, file_key, modified_unix_ms, size_bytes, imported_facts, invalid_lines, unsupported_lines, parser_revision, identity_revision, - event_digest, updated_at_unix_ms + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms FROM codex_usage_import_files WHERE source_id = ? AND file_key = ? `, sourceID, fileKey) @@ -91,6 +101,35 @@ func (s *Store) GetCodexUsageImportFile(ctx context.Context, sourceID int64, fil return cursor, err } +func (s *Store) ListCodexUsageImportFiles(ctx context.Context, sourceID int64) ([]CodexUsageImportFile, error) { + if sourceID <= 0 { + return nil, errors.New("Codex usage import query is invalid") + } + rows, err := s.executor().QueryContext(ctx, ` + SELECT source_id, file_key, modified_unix_ms, size_bytes, imported_facts, + invalid_lines, unsupported_lines, parser_revision, identity_revision, + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms + FROM codex_usage_import_files + WHERE source_id = ? + ORDER BY file_key + `, sourceID) + if err != nil { + return nil, err + } + defer rows.Close() + files := make([]CodexUsageImportFile, 0) + for rows.Next() { + file, err := scanCodexUsageImportFile(rows) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, rows.Err() +} + func scanCodexUsageImportFile(row rowScanner) (CodexUsageImportFile, error) { var cursor CodexUsageImportFile if err := row.Scan( @@ -104,6 +143,13 @@ func scanCodexUsageImportFile(row rowScanner) (CodexUsageImportFile, error) { &cursor.ParserRevision, &cursor.IdentityRevision, &cursor.EventDigest, + &cursor.CheckpointRevision, + &cursor.ProcessedBytes, + &cursor.MetadataDigest, + &cursor.FileIdentityDigest, + &cursor.BoundaryDigest, + &cursor.CheckpointEventDigest, + &cursor.ParserStateJSON, &cursor.UpdatedAtUnixMS, ); err != nil { return CodexUsageImportFile{}, err @@ -139,6 +185,9 @@ func (s *Store) commitCodexUsageImport(ctx context.Context, params CommitCodexUs if err := s.upsertCodexUsageImportFileCAS(ctx, params.File, params.Expected); err != nil { return UsageInsertResult{}, err } + if err := s.deleteUsageImportObservation(ctx, params.File.SourceID, params.File.FileKey); err != nil { + return UsageInsertResult{}, err + } return result, nil } @@ -176,6 +225,9 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa if err := validateCodexUsageImportFile(file); err != nil { return err } + if file.CheckpointRevision == 0 && strings.TrimSpace(file.ParserStateJSON) == "" { + file.ParserStateJSON = "{}" + } source, err := s.getUsageSourceByID(ctx, file.SourceID) if err != nil { return err @@ -190,8 +242,10 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa INSERT INTO codex_usage_import_files ( source_id, file_key, modified_unix_ms, size_bytes, imported_facts, invalid_lines, unsupported_lines, parser_revision, identity_revision, - event_digest, updated_at_unix_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, file.SourceID, file.FileKey, @@ -203,6 +257,13 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa file.ParserRevision, file.IdentityRevision, file.EventDigest, + file.CheckpointRevision, + file.ProcessedBytes, + file.MetadataDigest, + file.FileIdentityDigest, + file.BoundaryDigest, + file.CheckpointEventDigest, + file.ParserStateJSON, updatedAt, ) if isSQLiteConstraintError(err) { @@ -220,11 +281,17 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa UPDATE codex_usage_import_files SET modified_unix_ms = ?, size_bytes = ?, imported_facts = ?, invalid_lines = ?, unsupported_lines = ?, parser_revision = ?, - identity_revision = ?, event_digest = ?, updated_at_unix_ms = ? + identity_revision = ?, event_digest = ?, checkpoint_revision = ?, + processed_bytes = ?, metadata_digest = ?, file_identity_digest = ?, + boundary_digest = ?, checkpoint_event_digest = ?, parser_state_json = ?, + updated_at_unix_ms = ? WHERE source_id = ? AND file_key = ? AND modified_unix_ms = ? AND size_bytes = ? AND imported_facts = ? AND invalid_lines = ? AND unsupported_lines = ? AND parser_revision = ? - AND identity_revision = ? AND event_digest = ? AND updated_at_unix_ms = ? + AND identity_revision = ? AND event_digest = ? AND checkpoint_revision = ? + AND processed_bytes = ? AND metadata_digest = ? AND file_identity_digest = ? + AND boundary_digest = ? AND checkpoint_event_digest = ? AND parser_state_json = ? + AND updated_at_unix_ms = ? `, file.ModifiedUnixMS, file.SizeBytes, @@ -234,6 +301,13 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa file.ParserRevision, file.IdentityRevision, file.EventDigest, + file.CheckpointRevision, + file.ProcessedBytes, + file.MetadataDigest, + file.FileIdentityDigest, + file.BoundaryDigest, + file.CheckpointEventDigest, + file.ParserStateJSON, updatedAt, file.SourceID, file.FileKey, @@ -245,6 +319,13 @@ func (s *Store) upsertCodexUsageImportFileCAS(ctx context.Context, file CodexUsa expected.ParserRevision, expected.IdentityRevision, expected.EventDigest, + expected.CheckpointRevision, + expected.ProcessedBytes, + expected.MetadataDigest, + expected.FileIdentityDigest, + expected.BoundaryDigest, + expected.CheckpointEventDigest, + expected.ParserStateJSON, expected.UpdatedAtUnixMS, ) if err != nil { @@ -264,9 +345,24 @@ func validateCodexUsageImportFile(file CodexUsageImportFile) error { if file.SourceID <= 0 || file.FileKey.IsZero() || file.EventDigest.IsZero() || file.ModifiedUnixMS < 0 || file.SizeBytes < 0 || file.ImportedFacts < 0 || file.InvalidLines < 0 || file.UnsupportedLines < 0 || file.ParserRevision <= 0 || file.IdentityRevision <= 0 || - file.UpdatedAtUnixMS < 0 { + file.UpdatedAtUnixMS < 0 || file.CheckpointRevision < 0 || + file.ProcessedBytes < 0 || file.ProcessedBytes > file.SizeBytes { return errors.New("Codex usage import file is invalid") } + if file.CheckpointRevision == 0 { + if file.ProcessedBytes != 0 || !file.MetadataDigest.IsZero() || + !file.FileIdentityDigest.IsZero() || !file.BoundaryDigest.IsZero() || + !file.CheckpointEventDigest.IsZero() || file.ParserStateJSON != "" && file.ParserStateJSON != "{}" { + return errors.New("Codex usage import legacy checkpoint is invalid") + } + return nil + } + if file.CheckpointRevision != 1 || file.MetadataDigest.IsZero() || + file.BoundaryDigest.IsZero() || file.CheckpointEventDigest.IsZero() || + strings.TrimSpace(file.ParserStateJSON) == "" || + len(file.ParserStateJSON) > maxUsageParserStateSize { + return errors.New("Codex usage import checkpoint is invalid") + } return nil } @@ -315,6 +411,9 @@ func (finalization *CodexUsageSyncFinalization) applyUsageSyncFinalization( return usageSyncFinalizationResult{}, err } } + if err := store.deleteMissingUsageImportObservations(ctx, source.ID, discovered); err != nil { + return usageSyncFinalizationResult{}, err + } var result usageSyncFinalizationResult if err := store.executor().QueryRowContext(ctx, ` diff --git a/internal/store/usage_compact_test.go b/internal/store/usage_compact_test.go index 186e85b..ab59514 100644 --- a/internal/store/usage_compact_test.go +++ b/internal/store/usage_compact_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "database/sql" "errors" + "fmt" "path/filepath" "sync" "testing" @@ -488,6 +489,62 @@ func (executor *usageQueryCountingExecutor) QueryRowContext(ctx context.Context, return executor.dbExecutor.QueryRowContext(ctx, query, args...) } +func TestUsageSyncPreflightQueryCountDoesNotScaleWithFiles(t *testing.T) { + ctx := context.Background() + var expectedQueries int + for _, fileCount := range []int{1, 1_500} { + t.Run(fmt.Sprintf("files-%d", fileCount), func(t *testing.T) { + db := migratedTestStore(t, ctx) + defer closeTestStore(t, db) + createUsageProviderFixture(t, ctx, db, "codex") + source, err := db.BeginUsageSync(ctx, "codex", "codex-session-jsonl", 1) + if err != nil { + t.Fatalf("begin usage sync: %v", err) + } + insert, err := db.executor().PrepareContext(ctx, ` + INSERT INTO codex_usage_import_files ( + source_id, file_key, parser_revision, identity_revision, + event_digest, updated_at_unix_ms + ) VALUES (?, ?, 1, 1, ?, 1) + `) + if err != nil { + t.Fatalf("prepare cursor fixtures: %v", err) + } + for index := range fileCount { + key := testUsageKey(fmt.Sprintf("file-%d", index)) + if _, err := insert.ExecContext(ctx, source.ID, key, testUsageKey("digest")); err != nil { + _ = insert.Close() + t.Fatalf("insert cursor fixture %d: %v", index, err) + } + } + if err := insert.Close(); err != nil { + t.Fatalf("close cursor fixture statement: %v", err) + } + + counter := &usageQueryCountingExecutor{dbExecutor: db.executor()} + db.exec = counter + if _, err := db.GetUsageSource(ctx, "codex", "codex-session-jsonl"); err != nil { + t.Fatalf("read usage source: %v", err) + } + if _, err := db.ListCodexUsageImportFiles(ctx, source.ID); err != nil { + t.Fatalf("list cursors: %v", err) + } + if _, err := db.ListUsageImportObservations(ctx, source.ID); err != nil { + t.Fatalf("list observations: %v", err) + } + if _, err := db.ListUnknownUsageCostModels(ctx, "codex"); err != nil { + t.Fatalf("list unknown-cost models: %v", err) + } + if expectedQueries == 0 { + expectedQueries = counter.queries + } + if counter.queries != expectedQueries { + t.Fatalf("preflight queries = %d, want %d", counter.queries, expectedQueries) + } + }) + } +} + func TestUsageReportQueryCountDoesNotScaleWithBuckets(t *testing.T) { ctx := context.Background() db := migratedTestStore(t, ctx) diff --git a/internal/store/usage_grok_build.go b/internal/store/usage_grok_build.go index 8d05602..db14478 100644 --- a/internal/store/usage_grok_build.go +++ b/internal/store/usage_grok_build.go @@ -11,17 +11,24 @@ import ( ) type GrokBuildUsageImportFile struct { - SourceID int64 - FileKey UsageKey - ModifiedUnixMS int64 - SizeBytes int64 - ImportedFacts int64 - InvalidLines int64 - UnsupportedLines int64 - ParserRevision int64 - IdentityRevision int64 - EventDigest UsageKey - UpdatedAtUnixMS int64 + SourceID int64 + FileKey UsageKey + ModifiedUnixMS int64 + SizeBytes int64 + ImportedFacts int64 + InvalidLines int64 + UnsupportedLines int64 + ParserRevision int64 + IdentityRevision int64 + EventDigest UsageKey + CheckpointRevision int64 + ProcessedBytes int64 + MetadataDigest UsageKey + FileIdentityDigest UsageKey + BoundaryDigest UsageKey + CheckpointEventDigest UsageKey + ParserStateJSON string + UpdatedAtUnixMS int64 } type CommitGrokBuildUsageImportParams struct { @@ -93,7 +100,9 @@ func (s *Store) GetGrokBuildUsageImportFile( row := s.executor().QueryRowContext(ctx, ` SELECT source_id, file_key, modified_unix_ms, size_bytes, imported_facts, invalid_lines, unsupported_lines, parser_revision, identity_revision, - event_digest, updated_at_unix_ms + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms FROM grok_build_usage_import_files WHERE source_id = ? AND file_key = ? `, sourceID, fileKey) @@ -104,6 +113,38 @@ func (s *Store) GetGrokBuildUsageImportFile( return cursor, err } +func (s *Store) ListGrokBuildUsageImportFiles( + ctx context.Context, + sourceID int64, +) ([]GrokBuildUsageImportFile, error) { + if sourceID <= 0 { + return nil, errors.New("invalid Grok Build usage import query") + } + rows, err := s.executor().QueryContext(ctx, ` + SELECT source_id, file_key, modified_unix_ms, size_bytes, imported_facts, + invalid_lines, unsupported_lines, parser_revision, identity_revision, + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms + FROM grok_build_usage_import_files + WHERE source_id = ? + ORDER BY file_key + `, sourceID) + if err != nil { + return nil, err + } + defer rows.Close() + files := make([]GrokBuildUsageImportFile, 0) + for rows.Next() { + file, err := scanGrokBuildUsageImportFile(rows) + if err != nil { + return nil, err + } + files = append(files, file) + } + return files, rows.Err() +} + func scanGrokBuildUsageImportFile(row rowScanner) (GrokBuildUsageImportFile, error) { var cursor GrokBuildUsageImportFile if err := row.Scan( @@ -117,6 +158,13 @@ func scanGrokBuildUsageImportFile(row rowScanner) (GrokBuildUsageImportFile, err &cursor.ParserRevision, &cursor.IdentityRevision, &cursor.EventDigest, + &cursor.CheckpointRevision, + &cursor.ProcessedBytes, + &cursor.MetadataDigest, + &cursor.FileIdentityDigest, + &cursor.BoundaryDigest, + &cursor.CheckpointEventDigest, + &cursor.ParserStateJSON, &cursor.UpdatedAtUnixMS, ); err != nil { return GrokBuildUsageImportFile{}, err @@ -160,6 +208,9 @@ func (s *Store) commitGrokBuildUsageImport( if err := s.upsertGrokBuildUsageImportFileCAS(ctx, params.File, params.Expected); err != nil { return UsageInsertResult{}, err } + if err := s.deleteUsageImportObservation(ctx, params.File.SourceID, params.File.FileKey); err != nil { + return UsageInsertResult{}, err + } return result, nil } @@ -218,6 +269,9 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( if err := validateGrokBuildUsageImportFile(file); err != nil { return err } + if file.CheckpointRevision == 0 && strings.TrimSpace(file.ParserStateJSON) == "" { + file.ParserStateJSON = "{}" + } source, err := s.getUsageSourceByID(ctx, file.SourceID) if err != nil { return err @@ -232,8 +286,10 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( INSERT INTO grok_build_usage_import_files ( source_id, file_key, modified_unix_ms, size_bytes, imported_facts, invalid_lines, unsupported_lines, parser_revision, identity_revision, - event_digest, updated_at_unix_ms - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + event_digest, checkpoint_revision, processed_bytes, metadata_digest, + file_identity_digest, boundary_digest, checkpoint_event_digest, + parser_state_json, updated_at_unix_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, file.SourceID, file.FileKey, @@ -245,6 +301,13 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( file.ParserRevision, file.IdentityRevision, file.EventDigest, + file.CheckpointRevision, + file.ProcessedBytes, + file.MetadataDigest, + file.FileIdentityDigest, + file.BoundaryDigest, + file.CheckpointEventDigest, + file.ParserStateJSON, updatedAt, ) if isSQLiteConstraintError(err) { @@ -262,11 +325,17 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( UPDATE grok_build_usage_import_files SET modified_unix_ms = ?, size_bytes = ?, imported_facts = ?, invalid_lines = ?, unsupported_lines = ?, parser_revision = ?, - identity_revision = ?, event_digest = ?, updated_at_unix_ms = ? + identity_revision = ?, event_digest = ?, checkpoint_revision = ?, + processed_bytes = ?, metadata_digest = ?, file_identity_digest = ?, + boundary_digest = ?, checkpoint_event_digest = ?, parser_state_json = ?, + updated_at_unix_ms = ? WHERE source_id = ? AND file_key = ? AND modified_unix_ms = ? AND size_bytes = ? AND imported_facts = ? AND invalid_lines = ? AND unsupported_lines = ? AND parser_revision = ? - AND identity_revision = ? AND event_digest = ? AND updated_at_unix_ms = ? + AND identity_revision = ? AND event_digest = ? AND checkpoint_revision = ? + AND processed_bytes = ? AND metadata_digest = ? AND file_identity_digest = ? + AND boundary_digest = ? AND checkpoint_event_digest = ? AND parser_state_json = ? + AND updated_at_unix_ms = ? `, file.ModifiedUnixMS, file.SizeBytes, @@ -276,6 +345,13 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( file.ParserRevision, file.IdentityRevision, file.EventDigest, + file.CheckpointRevision, + file.ProcessedBytes, + file.MetadataDigest, + file.FileIdentityDigest, + file.BoundaryDigest, + file.CheckpointEventDigest, + file.ParserStateJSON, updatedAt, file.SourceID, file.FileKey, @@ -287,6 +363,13 @@ func (s *Store) upsertGrokBuildUsageImportFileCAS( expected.ParserRevision, expected.IdentityRevision, expected.EventDigest, + expected.CheckpointRevision, + expected.ProcessedBytes, + expected.MetadataDigest, + expected.FileIdentityDigest, + expected.BoundaryDigest, + expected.CheckpointEventDigest, + expected.ParserStateJSON, expected.UpdatedAtUnixMS, ) if err != nil { @@ -307,9 +390,23 @@ func validateGrokBuildUsageImportFile(file GrokBuildUsageImportFile) error { file.ModifiedUnixMS < 0 || file.SizeBytes < 0 || file.ImportedFacts < 0 || file.InvalidLines < 0 || file.UnsupportedLines < 0 || file.ParserRevision <= 0 || file.IdentityRevision <= 0 || - file.UpdatedAtUnixMS < 0 { + file.UpdatedAtUnixMS < 0 || file.CheckpointRevision < 0 || + file.ProcessedBytes < 0 || file.ProcessedBytes > file.SizeBytes { return errors.New("invalid Grok Build usage import file") } + if file.CheckpointRevision == 0 { + if file.ProcessedBytes != 0 || !file.MetadataDigest.IsZero() || + !file.FileIdentityDigest.IsZero() || !file.BoundaryDigest.IsZero() || + !file.CheckpointEventDigest.IsZero() || file.ParserStateJSON != "" && file.ParserStateJSON != "{}" { + return errors.New("invalid Grok Build legacy checkpoint") + } + return nil + } + if file.CheckpointRevision != 1 || file.MetadataDigest.IsZero() || + file.BoundaryDigest.IsZero() || file.CheckpointEventDigest.IsZero() || + file.ParserStateJSON != "{}" { + return errors.New("invalid Grok Build checkpoint") + } return nil } @@ -360,6 +457,9 @@ func (finalization *GrokBuildUsageSyncFinalization) applyUsageSyncFinalization( return usageSyncFinalizationResult{}, err } } + if err := store.deleteMissingUsageImportObservations(ctx, source.ID, discovered); err != nil { + return usageSyncFinalizationResult{}, err + } var result usageSyncFinalizationResult if err := store.executor().QueryRowContext(ctx, ` diff --git a/internal/usage/codex.go b/internal/usage/codex.go index 3daad4d..e0f387c 100644 --- a/internal/usage/codex.go +++ b/internal/usage/codex.go @@ -92,12 +92,7 @@ func collectCodexSessionTree(ctx context.Context, sessionsDir string, files *[]S if err != nil { return err } - *files = append(*files, SourceFile{ - Path: path, - SourceKey: sourceKey, - ModifiedUnixMS: info.ModTime().UnixMilli(), - SizeBytes: info.Size(), - }) + *files = append(*files, sourceFileFromInfo(path, sourceKey, info)) return nil }) } @@ -132,12 +127,7 @@ func collectArchivedCodexSessions(ctx context.Context, archivedDir string, files if err != nil { return err } - *files = append(*files, SourceFile{ - Path: path, - SourceKey: sourceKey, - ModifiedUnixMS: info.ModTime().UnixMilli(), - SizeBytes: info.Size(), - }) + *files = append(*files, sourceFileFromInfo(path, sourceKey, info)) } return nil } @@ -150,6 +140,88 @@ func ParseCodexSessionFileContext(ctx context.Context, file SourceFile) (FilePar return parseCodexSessionFile(ctx, file, maxCodexSessionLineBytes) } +type codexParserState struct { + CurrentSessionKey string `json:"current_session_key"` + HasLogSessionID bool `json:"has_log_session_id"` + ModelForStorage string `json:"model_for_storage"` + ModelForPricing string `json:"model_for_pricing"` + PreviousTotals map[string]TokenCounts `json:"previous_totals"` + UsageOrdinals map[string]int64 `json:"usage_ordinals"` +} + +func newCodexParserState(file SourceFile) codexParserState { + fallback := strings.TrimSuffix(filepath.Base(file.Path), filepath.Ext(file.Path)) + return codexParserState{ + CurrentSessionKey: storedSessionID(eventIdentitySessionID(fallback, file.SourceKey, false)), + ModelForStorage: store.UsageUnknownModelKey, + PreviousTotals: make(map[string]TokenCounts), + UsageOrdinals: make(map[string]int64), + } +} + +func (state *codexParserState) parseLine(line []byte) (*Event, bool, bool) { + var payload any + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.UseNumber() + if err := decoder.Decode(&payload); err != nil { + return nil, true, false + } + object, ok := payload.(map[string]any) + if !ok { + return nil, false, true + } + + eventType := eventTypeFromObject(object) + if found := sessionIDFromObject(object, eventType); found != "" { + state.CurrentSessionKey = storedSessionID(found) + state.HasLogSessionID = true + } + if found := modelFromObject(object); found.Stored != "" { + state.ModelForStorage = found.Stored + state.ModelForPricing = found.Pricing + } + + counts, _, cumulative, ok := tokenCountsFromObject(object) + if !ok { + return nil, false, false + } + counts.normalizeTotal() + if !counts.valid() { + return nil, false, true + } + + delta := counts + if cumulative { + previous := state.PreviousTotals[state.CurrentSessionKey] + delta = deltaFromCumulative(previous, counts) + state.PreviousTotals[state.CurrentSessionKey] = counts + } + delta.normalizeTotal() + if !delta.valid() { + return nil, false, true + } + if delta.empty() { + return nil, false, false + } + + state.UsageOrdinals[state.CurrentSessionKey]++ + usageOrdinal := state.UsageOrdinals[state.CurrentSessionKey] + costMicros, costStatus := EstimateCostMicros(state.ModelForPricing, delta) + event := Event{ + EventKey: EventID(ProviderCodex, SourceCodexSessionJSONL, usageOrdinal, state.CurrentSessionKey, state.ModelForStorage, delta), + SessionID: state.CurrentSessionKey, + Model: state.ModelForStorage, + OccurredAtUnixMS: occurredAtUnixMS(object), + InputTokens: delta.InputTokens, + CachedInputTokens: delta.CachedInputTokens, + OutputTokens: delta.OutputTokens, + TotalTokens: delta.TotalTokens, + EstimatedCostMicros: costMicros, + CostStatus: costStatus, + } + return &event, false, false +} + func parseCodexSessionFile(ctx context.Context, file SourceFile, maxLineBytes int) (FileParseResult, error) { if err := ctx.Err(); err != nil { return FileParseResult{}, err @@ -162,13 +234,7 @@ func parseCodexSessionFile(ctx context.Context, file SourceFile, maxLineBytes in result := FileParseResult{} reader := bufio.NewReaderSize(handle, 64*1024) - - sessionID := strings.TrimSuffix(filepath.Base(file.Path), filepath.Ext(file.Path)) - hasLogSessionID := false - modelForStorage := store.UsageUnknownModelKey - modelForPricing := "" - previousTotals := map[string]TokenCounts{} - usageOrdinals := map[string]int64{} + state := newCodexParserState(file) for { // Background imports must remain cancellable while scanning large local // session files so Desktop shutdown and per-run deadlines can complete. @@ -192,71 +258,18 @@ func parseCodexSessionFile(ctx context.Context, file SourceFile, maxLineBytes in continue } - var payload any - decoder := json.NewDecoder(bytes.NewReader(line)) - decoder.UseNumber() - if err := decoder.Decode(&payload); err != nil { + event, invalid, unsupported := state.parseLine(line) + if invalid { result.InvalidLines++ continue } - object, ok := payload.(map[string]any) - if !ok { + if unsupported { result.UnsupportedLines++ continue } - - eventType := eventTypeFromObject(object) - if found := sessionIDFromObject(object, eventType); found != "" { - sessionID = found - hasLogSessionID = true - } - if found := modelFromObject(object); found.Stored != "" { - modelForStorage = found.Stored - modelForPricing = found.Pricing + if event != nil { + result.Events = append(result.Events, *event) } - - counts, _, cumulative, ok := tokenCountsFromObject(object) - if !ok { - continue - } - counts.normalizeTotal() - if !counts.valid() { - result.UnsupportedLines++ - continue - } - - delta := counts - if cumulative { - previous := previousTotals[sessionID] - delta = deltaFromCumulative(previous, counts) - previousTotals[sessionID] = counts - } - delta.normalizeTotal() - if !delta.valid() { - result.UnsupportedLines++ - continue - } - if delta.empty() { - continue - } - - sessionIdentity := eventIdentitySessionID(sessionID, file.SourceKey, hasLogSessionID) - storedSessionIdentity := storedSessionID(sessionIdentity) - usageOrdinals[storedSessionIdentity]++ - usageOrdinal := usageOrdinals[storedSessionIdentity] - costMicros, costStatus := EstimateCostMicros(modelForPricing, delta) - result.Events = append(result.Events, Event{ - EventKey: EventID(ProviderCodex, SourceCodexSessionJSONL, usageOrdinal, storedSessionIdentity, modelForStorage, delta), - SessionID: storedSessionIdentity, - Model: modelForStorage, - OccurredAtUnixMS: occurredAtUnixMS(object), - InputTokens: delta.InputTokens, - CachedInputTokens: delta.CachedInputTokens, - OutputTokens: delta.OutputTokens, - TotalTokens: delta.TotalTokens, - EstimatedCostMicros: costMicros, - CostStatus: costStatus, - }) } return result, nil } diff --git a/internal/usage/codex_sync.go b/internal/usage/codex_sync.go index 3d36e06..46f4460 100644 --- a/internal/usage/codex_sync.go +++ b/internal/usage/codex_sync.go @@ -15,6 +15,8 @@ import ( const codexHistoryChangedMessage = "This Codex session file changed before the saved import point, so it was skipped to protect existing usage history." +const codexFactConflictMessage = "This Codex session file conflicts with previously imported usage, so it was skipped." + type codexIntegration struct { codexDir string provisioner ProviderProvisioner @@ -48,74 +50,113 @@ func (codexIntegration) PricingInfo() UsagePricingInfo { func (integration codexIntegration) Sync( ctx context.Context, stores store.Factory, - mode SyncProvisionMode, -) (UsageSyncResult, error) { + options SyncOptions, +) (SyncOutcome, error) { + files, cursors, observed, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) + if err != nil { + return SyncOutcome{}, err + } + if !work { + return SyncOutcome{Result: noWorkResult}, nil + } + if options.OnWorkDetected != nil { + options.OnWorkDetected() + } db, err := stores.OpenHealthy(ctx, false) if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } defer db.Close() - - source, err := beginCodexUsageSync(ctx, db, integration.provisioner, mode) + source, err := beginCodexUsageSync(ctx, db, integration.provisioner, options.ProvisionMode) if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } - files, err := ListCodexSessionFilesContext(ctx, integration.codexDir) - if err != nil { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "failed to list Codex session files", err) + if cursors == nil { + cursorRows, err := db.ListCodexUsageImportFiles(ctx, source.ID) + if err != nil { + return SyncOutcome{}, err + } + cursors = codexCursorMap(cursorRows) + } + if observed == nil { + observationRows, err := db.ListUsageImportObservations(ctx, source.ID) + if err != nil { + return SyncOutcome{}, err + } + observed = observationMap(observationRows) } result := UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL} discoveredFileKeys := make([]store.UsageKey, 0, len(files)) + observations := make([]store.UsageImportObservation, 0) for _, file := range files { if err := ctx.Err(); err != nil { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "usage import canceled", err) + return SyncOutcome{}, apperror.Wrap(apperror.UsageImportFailed, "usage import canceled", err) } discoveredFileKeys = append(discoveredFileKeys, file.SourceKey) result.ScannedFiles++ - - cursor, hasCursor, err := codexUsageCursor(ctx, db, source.ID, file.SourceKey) - if err != nil { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "failed to inspect usage import progress", err) + cursor, hasCursor := cursors[file.SourceKey] + if hasCursor && codexCursorMatchesFile(cursor, file) { + result.SkippedUnchangedFiles++ + continue } - if hasCursor && cursor.ModifiedUnixMS == file.ModifiedUnixMS && cursor.SizeBytes == file.SizeBytes && - cursor.ParserRevision == CodexUsageParserRevision && cursor.IdentityRevision == CodexUsageIdentityRevision { + if observation, ok := observed[file.SourceKey]; ok && + observation.MetadataDigest == file.MetadataDigest && !options.ForceObservedRetry { result.SkippedUnchangedFiles++ + result.Errors = append(result.Errors, codexObservationError(file, observation.Status)) + continue + } + if hasCursor && codexFileIsShorterThanCheckpoint(cursor, file) { + result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationHistoryChanged)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) continue } - parsed, err := ParseCodexSessionFileContext(ctx, file) + parsed, fullParse, err := parseCodexUsageChange(ctx, file, cursor, hasCursor, options) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "usage import canceled", ctxErr) + return SyncOutcome{}, apperror.Wrap(apperror.UsageImportFailed, "usage import canceled", ctxErr) } - result.Errors = append(result.Errors, UsageImportError{ - SourceKey: file.SourceKey.String(), - FileName: filepath.Base(file.Path), - Message: sanitizedUsageImportError(err), - }) + result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationUnavailable)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationUnavailable)) continue } - - eventsToStore, safe := codexEventsAfterCursor(parsed.Events, cursor, hasCursor, file) - if !safe { - result.Errors = append(result.Errors, UsageImportError{ - SourceKey: file.SourceKey.String(), - FileName: filepath.Base(file.Path), - Message: codexHistoryChangedMessage, - }) - continue + eventsToStore := parsed.Events + importedFacts := int64(len(parsed.Events)) + invalidLines := parsed.InvalidLines + unsupportedLines := parsed.UnsupportedLines + if hasCursor { + if fullParse && !codexCheckpointPrefixMatches(parsed.Events, cursor) { + result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationHistoryChanged)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) + continue + } + if fullParse { + eventsToStore = parsed.Events[cursor.ImportedFacts:] + importedFacts = int64(len(parsed.Events)) + } else { + importedFacts = cursor.ImportedFacts + int64(len(parsed.Events)) + invalidLines += cursor.InvalidLines + unsupportedLines += cursor.UnsupportedLines + } } desired := store.CodexUsageImportFile{ - SourceID: source.ID, - FileKey: file.SourceKey, - ModifiedUnixMS: file.ModifiedUnixMS, - SizeBytes: file.SizeBytes, - ImportedFacts: int64(len(parsed.Events)), - InvalidLines: parsed.InvalidLines, - UnsupportedLines: parsed.UnsupportedLines, - ParserRevision: CodexUsageParserRevision, - IdentityRevision: CodexUsageIdentityRevision, - EventDigest: EventDigest(parsed.Events, int64(len(parsed.Events))), + SourceID: source.ID, + FileKey: file.SourceKey, + ModifiedUnixMS: file.ModifiedUnixMS, + SizeBytes: file.SizeBytes, + ImportedFacts: importedFacts, + InvalidLines: invalidLines, + UnsupportedLines: unsupportedLines, + ParserRevision: CodexUsageParserRevision, + IdentityRevision: CodexUsageIdentityRevision, + EventDigest: parsed.CheckpointEventDigest, + CheckpointRevision: usageCheckpointRevision, + ProcessedBytes: parsed.ProcessedBytes, + MetadataDigest: file.MetadataDigest, + FileIdentityDigest: file.FileIdentityDigest, + BoundaryDigest: parsed.BoundaryDigest, + CheckpointEventDigest: parsed.CheckpointEventDigest, + ParserStateJSON: parsed.ParserStateJSON, } var expected *store.CodexUsageImportFile if hasCursor { @@ -131,22 +172,27 @@ func (integration codexIntegration) Sync( current, readErr := db.GetCodexUsageImportFile(ctx, source.ID, file.SourceKey) if readErr == nil && sameCodexUsageImportProgress(current, desired) { result.SkippedDuplicateEvents += int64(len(eventsToStore)) - result.InvalidLines += parsed.InvalidLines - result.UnsupportedLines += parsed.UnsupportedLines + result.InvalidLines += invalidLines + result.UnsupportedLines += unsupportedLines continue } if readErr != nil && !errors.Is(readErr, store.ErrNotFound) { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "failed to inspect concurrent usage sync", readErr) + return SyncOutcome{}, apperror.Wrap(apperror.UsageImportFailed, "failed to inspect concurrent usage sync", readErr) } - return UsageSyncResult{}, err + return SyncOutcome{}, err + } + if errors.Is(err, store.ErrUsageFactConflict) { + result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationFactConflict)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationFactConflict)) + continue } if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } result.ImportedEvents += int64(insertResult.Inserted) result.SkippedDuplicateEvents += int64(insertResult.Duplicates) - result.InvalidLines += parsed.InvalidLines - result.UnsupportedLines += parsed.UnsupportedLines + result.InvalidLines += invalidLines + result.UnsupportedLines += unsupportedLines } if err := db.CompleteUsageSync(ctx, store.CompleteUsageSyncParams{ @@ -156,13 +202,268 @@ func (integration codexIntegration) Sync( Finalization: &store.CodexUsageSyncFinalization{ DiscoveredFileKeys: discoveredFileKeys, }, + Observations: observations, }); err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err + } + if options.ProvisionMode == SyncProvisionProvider || priceBackfill { + if err := backfillPartialUsageCosts(ctx, db); err != nil { + return SyncOutcome{}, apperror.Wrap(apperror.UsageImportFailed, "failed to update usage pricing", err) + } + } + return SyncOutcome{Result: result, Performed: true}, nil +} + +func (integration codexIntegration) preflight( + ctx context.Context, + stores store.Factory, + options SyncOptions, +) ( + []SourceFile, + map[store.UsageKey]store.CodexUsageImportFile, + map[store.UsageKey]store.UsageImportObservation, + bool, + UsageSyncResult, + bool, + error, +) { + if options.ProvisionMode == SyncProvisionProvider { + files, err := ListCodexSessionFilesContext(ctx, integration.codexDir) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, + apperror.Wrap(apperror.UsageImportFailed, "failed to list Codex session files", err) + } + result := UsageSyncResult{ + ProviderID: ProviderCodex, + Source: SourceCodexSessionJSONL, + ScannedFiles: int64(len(files)), + } + return files, nil, nil, true, result, true, nil + } + db, err := stores.OpenHealthy(ctx, true) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + defer db.Close() + if integration.provisioner == nil { + return nil, nil, nil, false, UsageSyncResult{}, false, + errors.New("usage Provider provisioner for Codex is required") + } + if err := integration.provisioner.Ensure(ctx, db, SyncExistingProvider); err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + source, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if errors.Is(err, store.ErrNotFound) { + files, listErr := ListCodexSessionFilesContext(ctx, integration.codexDir) + if listErr != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, + apperror.Wrap(apperror.UsageImportFailed, "failed to list Codex session files", listErr) + } + result := UsageSyncResult{ + ProviderID: ProviderCodex, + Source: SourceCodexSessionJSONL, + ScannedFiles: int64(len(files)), + } + return files, nil, nil, false, result, true, nil + } + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + if source.IdentityRevision != CodexUsageIdentityRevision { + return nil, nil, nil, false, UsageSyncResult{}, false, store.ErrUsageIdentityRevision + } + cursorRows, err := db.ListCodexUsageImportFiles(ctx, source.ID) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + observationRows, err := db.ListUsageImportObservations(ctx, source.ID) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + unknownModels, err := db.ListUnknownUsageCostModels(ctx, ProviderCodex) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + files, err := ListCodexSessionFilesContext(ctx, integration.codexDir) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, + apperror.Wrap(apperror.UsageImportFailed, "failed to list Codex session files", err) + } + result := UsageSyncResult{ + ProviderID: ProviderCodex, + Source: SourceCodexSessionJSONL, + ScannedFiles: int64(len(files)), + } + cursors := codexCursorMap(cursorRows) + observations := observationMap(observationRows) + priceBackfill := hasSupportedUnknownUsageModel(unknownModels, codexPriceCatalog.Supports) + work := source.SyncGeneration != source.CompletedGeneration || priceBackfill + discovered := make(map[store.UsageKey]struct{}, len(files)) + for _, file := range files { + discovered[file.SourceKey] = struct{}{} + if cursor, ok := cursors[file.SourceKey]; ok { + if cursor.IdentityRevision != CodexUsageIdentityRevision { + return nil, nil, nil, false, UsageSyncResult{}, false, store.ErrUsageIdentityRevision + } + if codexCursorMatchesFile(cursor, file) { + result.SkippedUnchangedFiles++ + continue + } + } + if observation, ok := observations[file.SourceKey]; ok && + observation.MetadataDigest == file.MetadataDigest && !options.ForceObservedRetry { + result.SkippedUnchangedFiles++ + result.Errors = append(result.Errors, codexObservationError(file, observation.Status)) + continue + } + work = true } - if err := backfillPartialUsageCosts(ctx, db); err != nil { - return UsageSyncResult{}, apperror.Wrap(apperror.UsageImportFailed, "failed to update usage pricing", err) + for key := range cursors { + if _, ok := discovered[key]; !ok { + work = true + } + } + for key := range observations { + if _, ok := discovered[key]; !ok { + work = true + } + } + return files, cursors, observations, priceBackfill, result, work, nil +} + +func parseCodexUsageChange( + ctx context.Context, + file SourceFile, + cursor store.CodexUsageImportFile, + hasCursor bool, + options SyncOptions, +) (checkpointParseResult, bool, error) { + if hasCursor && cursor.CheckpointRevision == usageCheckpointRevision && + cursor.ParserRevision == CodexUsageParserRevision && + cursor.IdentityRevision == CodexUsageIdentityRevision && + !cursor.FileIdentityDigest.IsZero() && + cursor.FileIdentityDigest == file.FileIdentityDigest && + file.SizeBytes > cursor.SizeBytes { + state, err := decodeCodexParserState(cursor.ParserStateJSON) + if err == nil { + parsed, parseErr := parseCodexCheckpointFile( + ctx, + file, + cursor.ProcessedBytes, + state, + cursor.CheckpointEventDigest, + cursor.BoundaryDigest, + options.fileSystem, + options.Observer, + ) + if parseErr == nil { + return parsed, false, nil + } + if !errors.Is(parseErr, errUsageBoundaryChanged) { + return checkpointParseResult{}, false, parseErr + } + } + } + parsed, err := parseCodexCheckpointFile( + ctx, + file, + 0, + newCodexParserState(file), + store.UsageKey{}, + store.UsageKey{}, + options.fileSystem, + options.Observer, + ) + return parsed, true, err +} + +func codexCheckpointPrefixMatches(events []Event, cursor store.CodexUsageImportFile) bool { + if cursor.ImportedFacts < 0 || cursor.ImportedFacts > int64(len(events)) || + cursor.IdentityRevision != CodexUsageIdentityRevision { + return false + } + if cursor.CheckpointRevision == usageCheckpointRevision { + return checkpointEventDigest(ProviderCodex, events, cursor.ImportedFacts) == cursor.CheckpointEventDigest + } + return cursor.CheckpointRevision == 0 && EventDigest(events, cursor.ImportedFacts) == cursor.EventDigest +} + +func codexCursorMatchesFile(cursor store.CodexUsageImportFile, file SourceFile) bool { + if cursor.ParserRevision != CodexUsageParserRevision || + cursor.IdentityRevision != CodexUsageIdentityRevision { + return false + } + if cursor.CheckpointRevision == usageCheckpointRevision { + return cursor.MetadataDigest == file.MetadataDigest + } + return cursor.CheckpointRevision == 0 && + cursor.ModifiedUnixMS == file.ModifiedUnixMS && cursor.SizeBytes == file.SizeBytes +} + +func codexFileIsShorterThanCheckpoint(cursor store.CodexUsageImportFile, file SourceFile) bool { + if cursor.CheckpointRevision == usageCheckpointRevision { + return file.SizeBytes < cursor.ProcessedBytes + } + return file.SizeBytes < cursor.SizeBytes +} + +func codexCursorMap(rows []store.CodexUsageImportFile) map[store.UsageKey]store.CodexUsageImportFile { + result := make(map[store.UsageKey]store.CodexUsageImportFile, len(rows)) + for _, row := range rows { + result[row.FileKey] = row + } + return result +} + +func observationMap(rows []store.UsageImportObservation) map[store.UsageKey]store.UsageImportObservation { + result := make(map[store.UsageKey]store.UsageImportObservation, len(rows)) + for _, row := range rows { + result[row.FileKey] = row + } + return result +} + +func hasSupportedUnknownUsageModel( + models []store.UsageUnknownCostModel, + supports func(string) bool, +) bool { + for _, model := range models { + if supports(model.Model) { + return true + } + } + return false +} + +func newUsageObservation( + sourceID int64, + file SourceFile, + status store.UsageImportObservationStatus, +) store.UsageImportObservation { + return store.UsageImportObservation{ + SourceID: sourceID, + FileKey: file.SourceKey, + MetadataDigest: file.MetadataDigest, + Status: status, + } +} + +func codexObservationError( + file SourceFile, + status store.UsageImportObservationStatus, +) UsageImportError { + message := sanitizedUsageImportError(errors.New("usage file unavailable")) + switch status { + case store.UsageImportObservationHistoryChanged: + message = codexHistoryChangedMessage + case store.UsageImportObservationFactConflict: + message = codexFactConflictMessage + } + return UsageImportError{ + SourceKey: file.SourceKey.String(), + FileName: filepath.Base(file.Path), + Message: message, } - return result, nil } func beginCodexUsageSync( @@ -241,36 +542,19 @@ func (provisioner codexProviderProvisioner) Ensure( return nil } -func codexUsageCursor(ctx context.Context, db *store.Store, sourceID int64, fileKey store.UsageKey) (store.CodexUsageImportFile, bool, error) { - cursor, err := db.GetCodexUsageImportFile(ctx, sourceID, fileKey) - if errors.Is(err, store.ErrNotFound) { - return store.CodexUsageImportFile{}, false, nil - } - return cursor, err == nil, err -} - -func codexEventsAfterCursor(events []Event, cursor store.CodexUsageImportFile, hasCursor bool, file SourceFile) ([]Event, bool) { - if !hasCursor { - return events, true - } - // A shorter or rewritten append-only file is ambiguous, so retain its - // checkpoint and facts unchanged instead of reinterpreting prior history. - if cursor.IdentityRevision != CodexUsageIdentityRevision || cursor.ImportedFacts < 0 || - cursor.ImportedFacts > int64(len(events)) || file.SizeBytes < cursor.SizeBytes { - return nil, false - } - if EventDigest(events, cursor.ImportedFacts) != cursor.EventDigest { - return nil, false - } - return events[cursor.ImportedFacts:], true -} - func sameCodexUsageImportProgress(left, right store.CodexUsageImportFile) bool { return left.SourceID == right.SourceID && left.FileKey == right.FileKey && left.ModifiedUnixMS == right.ModifiedUnixMS && left.SizeBytes == right.SizeBytes && left.ImportedFacts == right.ImportedFacts && left.InvalidLines == right.InvalidLines && left.UnsupportedLines == right.UnsupportedLines && left.ParserRevision == right.ParserRevision && - left.IdentityRevision == right.IdentityRevision && left.EventDigest == right.EventDigest + left.IdentityRevision == right.IdentityRevision && left.EventDigest == right.EventDigest && + left.CheckpointRevision == right.CheckpointRevision && + left.ProcessedBytes == right.ProcessedBytes && + left.MetadataDigest == right.MetadataDigest && + left.FileIdentityDigest == right.FileIdentityDigest && + left.BoundaryDigest == right.BoundaryDigest && + left.CheckpointEventDigest == right.CheckpointEventDigest && + left.ParserStateJSON == right.ParserStateJSON } func usageEventsToFactParams(sourceID int64, events []Event) []store.CreateUsageFactParams { diff --git a/internal/usage/file_identity_other.go b/internal/usage/file_identity_other.go new file mode 100644 index 0000000..1d70018 --- /dev/null +++ b/internal/usage/file_identity_other.go @@ -0,0 +1,13 @@ +//go:build !darwin && !linux && !windows + +package usage + +import ( + "os" + + "github.com/strahe/profiledeck/internal/store" +) + +func sourceFileIdentityDigest(string, os.FileInfo) store.UsageKey { + return store.UsageKey{} +} diff --git a/internal/usage/file_identity_unix.go b/internal/usage/file_identity_unix.go new file mode 100644 index 0000000..e6e01b8 --- /dev/null +++ b/internal/usage/file_identity_unix.go @@ -0,0 +1,29 @@ +//go:build darwin || linux + +package usage + +import ( + "crypto/sha256" + "encoding/binary" + "os" + "syscall" + + "github.com/strahe/profiledeck/internal/store" +) + +func sourceFileIdentityDigest(_ string, info os.FileInfo) store.UsageKey { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return store.UsageKey{} + } + hash := sha256.New() + _, _ = hash.Write([]byte("profiledeck-usage-file-identity-v1\x00")) + var encoded [8]byte + binary.BigEndian.PutUint64(encoded[:], uint64(stat.Dev)) + _, _ = hash.Write(encoded[:]) + binary.BigEndian.PutUint64(encoded[:], uint64(stat.Ino)) + _, _ = hash.Write(encoded[:]) + var digest store.UsageKey + copy(digest[:], hash.Sum(nil)) + return digest +} diff --git a/internal/usage/file_identity_windows.go b/internal/usage/file_identity_windows.go new file mode 100644 index 0000000..334398d --- /dev/null +++ b/internal/usage/file_identity_windows.go @@ -0,0 +1,49 @@ +//go:build windows + +package usage + +import ( + "crypto/sha256" + "encoding/binary" + "os" + + "golang.org/x/sys/windows" + + "github.com/strahe/profiledeck/internal/store" +) + +func sourceFileIdentityDigest(path string, _ os.FileInfo) store.UsageKey { + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return store.UsageKey{} + } + handle, err := windows.CreateFile( + name, + 0, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + return store.UsageKey{} + } + defer windows.CloseHandle(handle) + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + return store.UsageKey{} + } + + hash := sha256.New() + _, _ = hash.Write([]byte("profiledeck-usage-file-identity-v1\x00")) + var encoded [8]byte + binary.BigEndian.PutUint64(encoded[:], uint64(info.VolumeSerialNumber)) + _, _ = hash.Write(encoded[:]) + fileIndex := uint64(info.FileIndexHigh)<<32 | uint64(info.FileIndexLow) + binary.BigEndian.PutUint64(encoded[:], fileIndex) + _, _ = hash.Write(encoded[:]) + var digest store.UsageKey + copy(digest[:], hash.Sum(nil)) + return digest +} diff --git a/internal/usage/grok_build.go b/internal/usage/grok_build.go index fde8edd..a229c0d 100644 --- a/internal/usage/grok_build.go +++ b/internal/usage/grok_build.go @@ -121,12 +121,7 @@ func ListGrokBuildSessionFilesContext(ctx context.Context, grokHome string) ([]S if err != nil { return nil, err } - files = append(files, SourceFile{ - Path: updatePath, - SourceKey: sourceKey, - ModifiedUnixMS: updateInfo.ModTime().UnixMilli(), - SizeBytes: updateInfo.Size(), - }) + files = append(files, sourceFileFromInfo(updatePath, sourceKey, updateInfo)) } } sort.Slice(files, func(i, j int) bool { diff --git a/internal/usage/grok_build_sync.go b/internal/usage/grok_build_sync.go index a715738..5a4cd5c 100644 --- a/internal/usage/grok_build_sync.go +++ b/internal/usage/grok_build_sync.go @@ -13,9 +13,9 @@ import ( ) const ( - grokBuildFileUnavailableMessage = "A Grok Build session file could not be imported and will be retried." - grokBuildHistoryChangedMessage = "A Grok Build session file changed before the saved import point and will be retried." - grokBuildFactConflictMessage = "A Grok Build session file conflicts with previously imported usage and will be retried." + grokBuildFileUnavailableMessage = "A Grok Build session file could not be imported. It will be checked again after the file changes or when you sync manually." + grokBuildHistoryChangedMessage = "A Grok Build session file changed before the saved import point. It will be checked again after the file changes or when you sync manually." + grokBuildFactConflictMessage = "A Grok Build session file conflicts with previously imported usage. It will be checked again after the file changes or when you sync manually." ) type grokBuildIntegration struct { @@ -57,34 +57,50 @@ func (grokBuildIntegration) PricingInfo() UsagePricingInfo { func (integration grokBuildIntegration) Sync( ctx context.Context, stores store.Factory, - mode SyncProvisionMode, -) (UsageSyncResult, error) { + options SyncOptions, +) (SyncOutcome, error) { + files, cursors, observed, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) + if err != nil { + return SyncOutcome{}, err + } + if !work { + return SyncOutcome{Result: noWorkResult}, nil + } + if options.OnWorkDetected != nil { + options.OnWorkDetected() + } db, err := stores.OpenHealthy(ctx, false) if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } defer db.Close() - - source, err := beginGrokBuildUsageSync(ctx, db, integration.provisioner, mode) + source, err := beginGrokBuildUsageSync(ctx, db, integration.provisioner, options.ProvisionMode) if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } - files, err := ListGrokBuildSessionFilesContext(ctx, integration.grokHome) - if err != nil { - return UsageSyncResult{}, apperror.Wrap( - apperror.UsageImportFailed, - "failed to list Grok Build session files", - err, - ) + if cursors == nil { + cursorRows, err := db.ListGrokBuildUsageImportFiles(ctx, source.ID) + if err != nil { + return SyncOutcome{}, err + } + cursors = grokBuildCursorMap(cursorRows) + } + if observed == nil { + observationRows, err := db.ListUsageImportObservations(ctx, source.ID) + if err != nil { + return SyncOutcome{}, err + } + observed = observationMap(observationRows) } result := UsageSyncResult{ ProviderID: grokconfig.ProviderID, Source: SourceGrokBuildSessionJSONL, } discoveredFileKeys := make([]store.UsageKey, 0, len(files)) + observations := make([]store.UsageImportObservation, 0) for _, file := range files { if err := ctx.Err(); err != nil { - return UsageSyncResult{}, apperror.Wrap( + return SyncOutcome{}, apperror.Wrap( apperror.UsageImportFailed, "usage import canceled", err, @@ -93,57 +109,75 @@ func (integration grokBuildIntegration) Sync( discoveredFileKeys = append(discoveredFileKeys, file.SourceKey) result.ScannedFiles++ - cursor, hasCursor, err := grokBuildUsageCursor(ctx, db, source.ID, file.SourceKey) - if err != nil { - return UsageSyncResult{}, apperror.Wrap( - apperror.UsageImportFailed, - "failed to inspect usage import progress", - err, - ) + cursor, hasCursor := cursors[file.SourceKey] + if hasCursor && grokBuildCursorMatchesFile(cursor, file) { + result.SkippedUnchangedFiles++ + continue } - if hasCursor && - cursor.ModifiedUnixMS == file.ModifiedUnixMS && - cursor.SizeBytes == file.SizeBytes && - cursor.ParserRevision == GrokBuildUsageParserRevision && - cursor.IdentityRevision == GrokBuildUsageIdentityRevision { + if observation, ok := observed[file.SourceKey]; ok && + observation.MetadataDigest == file.MetadataDigest && !options.ForceObservedRetry { result.SkippedUnchangedFiles++ + result.Errors = append(result.Errors, grokBuildObservationError(observation.Status)) + continue + } + if hasCursor && grokBuildFileIsShorterThanCheckpoint(cursor, file) { + result.Errors = append(result.Errors, grokBuildObservationError(store.UsageImportObservationHistoryChanged)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) continue } - parsed, err := ParseGrokBuildSessionFileContext(ctx, file) + parsed, fullParse, err := parseGrokBuildUsageChange(ctx, file, cursor, hasCursor, options) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return UsageSyncResult{}, apperror.Wrap( + return SyncOutcome{}, apperror.Wrap( apperror.UsageImportFailed, "usage import canceled", ctxErr, ) } result.InvalidLines++ - result.Errors = append(result.Errors, UsageImportError{ - Message: grokBuildFileUnavailableMessage, - }) + result.Errors = append(result.Errors, grokBuildObservationError(store.UsageImportObservationUnavailable)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationUnavailable)) continue } - eventsToStore, safe := grokBuildEventsAfterCursor(parsed.Events, cursor, hasCursor, file) - if !safe { - result.Errors = append(result.Errors, UsageImportError{ - Message: grokBuildHistoryChangedMessage, - }) - continue + eventsToStore := parsed.Events + importedFacts := int64(len(parsed.Events)) + invalidLines := parsed.InvalidLines + unsupportedLines := parsed.UnsupportedLines + if hasCursor { + if fullParse && !grokBuildCheckpointPrefixMatches(parsed.Events, cursor) { + result.Errors = append(result.Errors, grokBuildObservationError(store.UsageImportObservationHistoryChanged)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) + continue + } + if fullParse { + eventsToStore = parsed.Events[cursor.ImportedFacts:] + importedFacts = int64(len(parsed.Events)) + } else { + importedFacts = cursor.ImportedFacts + int64(len(parsed.Events)) + invalidLines += cursor.InvalidLines + unsupportedLines += cursor.UnsupportedLines + } } desired := store.GrokBuildUsageImportFile{ - SourceID: source.ID, - FileKey: file.SourceKey, - ModifiedUnixMS: file.ModifiedUnixMS, - SizeBytes: file.SizeBytes, - ImportedFacts: int64(len(parsed.Events)), - InvalidLines: parsed.InvalidLines, - UnsupportedLines: parsed.UnsupportedLines, - ParserRevision: GrokBuildUsageParserRevision, - IdentityRevision: GrokBuildUsageIdentityRevision, - EventDigest: GrokBuildEventDigest(parsed.Events, int64(len(parsed.Events))), + SourceID: source.ID, + FileKey: file.SourceKey, + ModifiedUnixMS: file.ModifiedUnixMS, + SizeBytes: file.SizeBytes, + ImportedFacts: importedFacts, + InvalidLines: invalidLines, + UnsupportedLines: unsupportedLines, + ParserRevision: GrokBuildUsageParserRevision, + IdentityRevision: GrokBuildUsageIdentityRevision, + EventDigest: parsed.CheckpointEventDigest, + CheckpointRevision: usageCheckpointRevision, + ProcessedBytes: parsed.ProcessedBytes, + MetadataDigest: file.MetadataDigest, + FileIdentityDigest: file.FileIdentityDigest, + BoundaryDigest: parsed.BoundaryDigest, + CheckpointEventDigest: parsed.CheckpointEventDigest, + ParserStateJSON: "{}", } var expected *store.GrokBuildUsageImportFile if hasCursor { @@ -163,32 +197,31 @@ func (integration grokBuildIntegration) Sync( current, readErr := db.GetGrokBuildUsageImportFile(ctx, source.ID, file.SourceKey) if readErr == nil && sameGrokBuildUsageImportProgress(current, desired) { result.SkippedDuplicateEvents += int64(len(eventsToStore)) - result.UnsupportedLines += parsed.UnsupportedLines + result.UnsupportedLines += unsupportedLines continue } if readErr != nil && !errors.Is(readErr, store.ErrNotFound) { - return UsageSyncResult{}, apperror.Wrap( + return SyncOutcome{}, apperror.Wrap( apperror.UsageImportFailed, "failed to inspect concurrent usage sync", readErr, ) } - return UsageSyncResult{}, err + return SyncOutcome{}, err } if errors.Is(err, store.ErrUsageFactConflict) { result.InvalidLines++ - result.Errors = append(result.Errors, UsageImportError{ - Message: grokBuildFactConflictMessage, - }) + result.Errors = append(result.Errors, grokBuildObservationError(store.UsageImportObservationFactConflict)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationFactConflict)) continue } if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } result.ImportedEvents += int64(insertResult.Inserted) result.SkippedDuplicateEvents += int64(insertResult.Duplicates) - result.InvalidLines += parsed.InvalidLines - result.UnsupportedLines += parsed.UnsupportedLines + result.InvalidLines += invalidLines + result.UnsupportedLines += unsupportedLines } if err := db.CompleteUsageSync(ctx, store.CompleteUsageSyncParams{ @@ -199,23 +232,243 @@ func (integration grokBuildIntegration) Sync( ProviderID: grokconfig.ProviderID, DiscoveredFileKeys: discoveredFileKeys, }, + Observations: observations, }); err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } - if err := backfillUnknownUsageCosts( - ctx, - db, - grokconfig.ProviderID, - grokBuildPriceCatalog.Supports, - EstimateGrokBuildCostMicros, - ); err != nil { - return UsageSyncResult{}, apperror.Wrap( + if options.ProvisionMode == SyncProvisionProvider || priceBackfill { + if err := backfillUnknownUsageCosts( + ctx, + db, + grokconfig.ProviderID, + grokBuildPriceCatalog.Supports, + EstimateGrokBuildCostMicros, + ); err != nil { + return SyncOutcome{}, apperror.Wrap( + apperror.UsageImportFailed, + "failed to update usage pricing", + err, + ) + } + } + return SyncOutcome{Result: result, Performed: true}, nil +} + +func (integration grokBuildIntegration) preflight( + ctx context.Context, + stores store.Factory, + options SyncOptions, +) ( + []SourceFile, + map[store.UsageKey]store.GrokBuildUsageImportFile, + map[store.UsageKey]store.UsageImportObservation, + bool, + UsageSyncResult, + bool, + error, +) { + if options.ProvisionMode == SyncProvisionProvider { + files, err := ListGrokBuildSessionFilesContext(ctx, integration.grokHome) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, apperror.Wrap( + apperror.UsageImportFailed, + "failed to list Grok Build session files", + err, + ) + } + result := UsageSyncResult{ + ProviderID: grokconfig.ProviderID, + Source: SourceGrokBuildSessionJSONL, + ScannedFiles: int64(len(files)), + } + return files, nil, nil, true, result, true, nil + } + db, err := stores.OpenHealthy(ctx, true) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + defer db.Close() + if integration.provisioner == nil { + return nil, nil, nil, false, UsageSyncResult{}, false, + errors.New("usage Provider provisioner for Grok Build is required") + } + if err := integration.provisioner.Ensure(ctx, db, SyncExistingProvider); err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + source, err := db.GetUsageSource(ctx, grokconfig.ProviderID, SourceGrokBuildSessionJSONL) + if errors.Is(err, store.ErrNotFound) { + files, listErr := ListGrokBuildSessionFilesContext(ctx, integration.grokHome) + if listErr != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, apperror.Wrap( + apperror.UsageImportFailed, + "failed to list Grok Build session files", + listErr, + ) + } + result := UsageSyncResult{ + ProviderID: grokconfig.ProviderID, + Source: SourceGrokBuildSessionJSONL, + ScannedFiles: int64(len(files)), + } + return files, nil, nil, false, result, true, nil + } + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + if source.IdentityRevision != GrokBuildUsageIdentityRevision { + return nil, nil, nil, false, UsageSyncResult{}, false, store.ErrUsageIdentityRevision + } + cursorRows, err := db.ListGrokBuildUsageImportFiles(ctx, source.ID) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + observationRows, err := db.ListUsageImportObservations(ctx, source.ID) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + unknownModels, err := db.ListUnknownUsageCostModels(ctx, grokconfig.ProviderID) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, err + } + files, err := ListGrokBuildSessionFilesContext(ctx, integration.grokHome) + if err != nil { + return nil, nil, nil, false, UsageSyncResult{}, false, apperror.Wrap( apperror.UsageImportFailed, - "failed to update usage pricing", + "failed to list Grok Build session files", err, ) } - return result, nil + result := UsageSyncResult{ + ProviderID: grokconfig.ProviderID, + Source: SourceGrokBuildSessionJSONL, + ScannedFiles: int64(len(files)), + } + cursors := grokBuildCursorMap(cursorRows) + observations := observationMap(observationRows) + priceBackfill := hasSupportedUnknownUsageModel(unknownModels, grokBuildPriceCatalog.Supports) + work := source.SyncGeneration != source.CompletedGeneration || priceBackfill + discovered := make(map[store.UsageKey]struct{}, len(files)) + for _, file := range files { + discovered[file.SourceKey] = struct{}{} + if cursor, ok := cursors[file.SourceKey]; ok { + if cursor.IdentityRevision != GrokBuildUsageIdentityRevision { + return nil, nil, nil, false, UsageSyncResult{}, false, store.ErrUsageIdentityRevision + } + if grokBuildCursorMatchesFile(cursor, file) { + result.SkippedUnchangedFiles++ + continue + } + } + if observation, ok := observations[file.SourceKey]; ok && + observation.MetadataDigest == file.MetadataDigest && !options.ForceObservedRetry { + result.SkippedUnchangedFiles++ + result.Errors = append(result.Errors, grokBuildObservationError(observation.Status)) + continue + } + work = true + } + for key := range cursors { + if _, ok := discovered[key]; !ok { + work = true + } + } + for key := range observations { + if _, ok := discovered[key]; !ok { + work = true + } + } + return files, cursors, observations, priceBackfill, result, work, nil +} + +func parseGrokBuildUsageChange( + ctx context.Context, + file SourceFile, + cursor store.GrokBuildUsageImportFile, + hasCursor bool, + options SyncOptions, +) (checkpointParseResult, bool, error) { + if hasCursor && cursor.CheckpointRevision == usageCheckpointRevision && + cursor.ParserRevision == GrokBuildUsageParserRevision && + cursor.IdentityRevision == GrokBuildUsageIdentityRevision && + !cursor.FileIdentityDigest.IsZero() && + cursor.FileIdentityDigest == file.FileIdentityDigest && + cursor.ParserStateJSON == "{}" && file.SizeBytes > cursor.SizeBytes { + parsed, parseErr := parseGrokBuildCheckpointFile( + ctx, + file, + cursor.ProcessedBytes, + cursor.CheckpointEventDigest, + cursor.BoundaryDigest, + options.fileSystem, + options.Observer, + ) + if parseErr == nil { + return parsed, false, nil + } + if !errors.Is(parseErr, errUsageBoundaryChanged) { + return checkpointParseResult{}, false, parseErr + } + } + parsed, err := parseGrokBuildCheckpointFile( + ctx, + file, + 0, + store.UsageKey{}, + store.UsageKey{}, + options.fileSystem, + options.Observer, + ) + return parsed, true, err +} + +func grokBuildCheckpointPrefixMatches(events []Event, cursor store.GrokBuildUsageImportFile) bool { + if cursor.ImportedFacts < 0 || cursor.ImportedFacts > int64(len(events)) || + cursor.IdentityRevision != GrokBuildUsageIdentityRevision { + return false + } + if cursor.CheckpointRevision == usageCheckpointRevision { + return checkpointEventDigest(grokconfig.ProviderID, events, cursor.ImportedFacts) == cursor.CheckpointEventDigest + } + return cursor.CheckpointRevision == 0 && + GrokBuildEventDigest(events, cursor.ImportedFacts) == cursor.EventDigest +} + +func grokBuildCursorMatchesFile(cursor store.GrokBuildUsageImportFile, file SourceFile) bool { + if cursor.ParserRevision != GrokBuildUsageParserRevision || + cursor.IdentityRevision != GrokBuildUsageIdentityRevision { + return false + } + if cursor.CheckpointRevision == usageCheckpointRevision { + return cursor.MetadataDigest == file.MetadataDigest + } + return cursor.CheckpointRevision == 0 && + cursor.ModifiedUnixMS == file.ModifiedUnixMS && cursor.SizeBytes == file.SizeBytes +} + +func grokBuildFileIsShorterThanCheckpoint(cursor store.GrokBuildUsageImportFile, file SourceFile) bool { + if cursor.CheckpointRevision == usageCheckpointRevision { + return file.SizeBytes < cursor.ProcessedBytes + } + return file.SizeBytes < cursor.SizeBytes +} + +func grokBuildCursorMap(rows []store.GrokBuildUsageImportFile) map[store.UsageKey]store.GrokBuildUsageImportFile { + result := make(map[store.UsageKey]store.GrokBuildUsageImportFile, len(rows)) + for _, row := range rows { + result[row.FileKey] = row + } + return result +} + +func grokBuildObservationError(status store.UsageImportObservationStatus) UsageImportError { + message := grokBuildFileUnavailableMessage + switch status { + case store.UsageImportObservationHistoryChanged: + message = grokBuildHistoryChangedMessage + case store.UsageImportObservationFactConflict: + message = grokBuildFactConflictMessage + } + return UsageImportError{Message: message} } func beginGrokBuildUsageSync( @@ -304,40 +557,6 @@ func (provisioner grokBuildProviderProvisioner) Ensure( return nil } -func grokBuildUsageCursor( - ctx context.Context, - db *store.Store, - sourceID int64, - fileKey store.UsageKey, -) (store.GrokBuildUsageImportFile, bool, error) { - cursor, err := db.GetGrokBuildUsageImportFile(ctx, sourceID, fileKey) - if errors.Is(err, store.ErrNotFound) { - return store.GrokBuildUsageImportFile{}, false, nil - } - return cursor, err == nil, err -} - -func grokBuildEventsAfterCursor( - events []Event, - cursor store.GrokBuildUsageImportFile, - hasCursor bool, - file SourceFile, -) ([]Event, bool) { - if !hasCursor { - return events, true - } - if cursor.IdentityRevision != GrokBuildUsageIdentityRevision || - cursor.ImportedFacts < 0 || - cursor.ImportedFacts > int64(len(events)) || - file.SizeBytes < cursor.SizeBytes { - return nil, false - } - if GrokBuildEventDigest(events, cursor.ImportedFacts) != cursor.EventDigest { - return nil, false - } - return events[cursor.ImportedFacts:], true -} - func sameGrokBuildUsageImportProgress( left store.GrokBuildUsageImportFile, right store.GrokBuildUsageImportFile, @@ -351,5 +570,12 @@ func sameGrokBuildUsageImportProgress( left.UnsupportedLines == right.UnsupportedLines && left.ParserRevision == right.ParserRevision && left.IdentityRevision == right.IdentityRevision && - left.EventDigest == right.EventDigest + left.EventDigest == right.EventDigest && + left.CheckpointRevision == right.CheckpointRevision && + left.ProcessedBytes == right.ProcessedBytes && + left.MetadataDigest == right.MetadataDigest && + left.FileIdentityDigest == right.FileIdentityDigest && + left.BoundaryDigest == right.BoundaryDigest && + left.CheckpointEventDigest == right.CheckpointEventDigest && + left.ParserStateJSON == right.ParserStateJSON } diff --git a/internal/usage/grok_build_sync_test.go b/internal/usage/grok_build_sync_test.go index 435924b..5186484 100644 --- a/internal/usage/grok_build_sync_test.go +++ b/internal/usage/grok_build_sync_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "os" "path/filepath" "strings" "sync" @@ -105,6 +106,70 @@ func TestUsageSyncGrokBuildImportsIncrementallyAcrossRestart(t *testing.T) { } } +func TestBackgroundGrokBuildSyncNoopAndBoundedAppend(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + grokHome := t.TempDir() + benign := `{"timestamp":1,"method":"session/update","params":{"sessionId":"session-tail","update":{"sessionUpdate":"agent_message_chunk"}}}` + "\n" + path := writeGrokBuildUsageFixture( + t, + grokHome, + "workspace-tail", + "session-tail", + strings.Repeat(benign, 20_000)+syntheticGrokBuildUsageLine( + "session-tail", + "prompt-a", + "grok-build-latest", + 1_750_000_000, + TokenCounts{InputTokens: 100, CachedInputTokens: 20, OutputTokens: 10, TotalTokens: 110}, + ), + ) + environment := newGrokBuildUsageTestEnvironment(t, configDir, grokHome) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncGrokBuild(ctx); err != nil { + t.Fatalf("initial Grok Build sync: %v", err) + } + + idleObserver := &usageSyncReadObserver{} + idle, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: grokconfig.ProviderID}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: idleObserver, + }) + if err != nil || idle.Performed || idle.Result.SkippedUnchangedFiles != 1 { + t.Fatalf("idle Grok Build outcome = %#v, err = %v", idle, err) + } + if idleObserver.opened.Load() != 0 || idleObserver.bytes.Load() != 0 { + t.Fatalf("idle Grok Build sync read content: opens=%d bytes=%d", idleObserver.opened.Load(), idleObserver.bytes.Load()) + } + + appendedLine := syntheticGrokBuildUsageLine( + "session-tail", + "prompt-b", + "grok-4.5", + 1_750_000_001, + TokenCounts{InputTokens: 50, CachedInputTokens: 10, OutputTokens: 5, TotalTokens: 55}, + ) + appendAppUsageFixture(t, path, appendedLine) + appendObserver := &usageSyncReadObserver{} + appended, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: grokconfig.ProviderID}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: appendObserver, + }) + if err != nil || !appended.Performed || appended.Result.ImportedEvents != 1 { + t.Fatalf("Grok Build tail outcome = %#v, err = %v", appended, err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat Grok Build fixture: %v", err) + } + maxRead := usageBoundaryBytes + int64(len(appendedLine)+1) + if appendObserver.opened.Load() != 1 || appendObserver.bytes.Load() > maxRead { + t.Fatalf("Grok Build append read was not bounded: opens=%d bytes=%d size=%d", appendObserver.opened.Load(), appendObserver.bytes.Load(), info.Size()) + } +} + func TestUsageSyncGrokBuildBackfillsNewlyRecognizedUnknownModels(t *testing.T) { ctx := context.Background() environment := newGrokBuildUsageTestEnvironment(t, t.TempDir(), t.TempDir()) @@ -450,8 +515,8 @@ func TestBackgroundGrokBuildSyncDoesNotCreateOrRebindProvider(t *testing.T) { if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { t.Fatalf("initialize runtime: %v", err) } - background, err := environment.service.SyncProviderBackground(ctx, grokconfig.ProviderID) - if err != nil || background.ProviderID != grokconfig.ProviderID || background.ImportedEvents != 0 { + background, err := environment.service.SyncProviderBackground(ctx, grokconfig.ProviderID, nil) + if err != nil || background.Result.ProviderID != grokconfig.ProviderID || background.Result.ImportedEvents != 0 || background.Performed { t.Fatalf("missing Provider background sync = %#v, err = %v", background, err) } db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, true) diff --git a/internal/usage/incremental_reader.go b/internal/usage/incremental_reader.go new file mode 100644 index 0000000..49440aa --- /dev/null +++ b/internal/usage/incremental_reader.go @@ -0,0 +1,506 @@ +package usage + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "io" + "os" + "strings" + + grokconfig "github.com/strahe/profiledeck/internal/grokbuild/config" + "github.com/strahe/profiledeck/internal/store" +) + +const ( + usageCheckpointRevision = int64(1) + usageBoundaryBytes = int64(64 * 1024) + maxCodexParserStateSize = 1024 * 1024 +) + +type UsageSyncObserver interface { + UsageFileOpened() + UsageBytesRead(int64) +} + +type usageFileSystem interface { + Lstat(string) (os.FileInfo, error) + Open(string) (usageOpenFile, error) +} + +type usageOpenFile interface { + io.Reader + io.ReaderAt + io.Seeker + io.Closer + Stat() (os.FileInfo, error) +} + +type osUsageFileSystem struct{} + +func (osUsageFileSystem) Lstat(path string) (os.FileInfo, error) { + return os.Lstat(path) +} + +func (osUsageFileSystem) Open(path string) (usageOpenFile, error) { + return os.Open(path) +} + +type observedReaderAt struct { + reader io.ReaderAt + observer UsageSyncObserver +} + +func (reader observedReaderAt) ReadAt(buffer []byte, offset int64) (int, error) { + count, err := reader.reader.ReadAt(buffer, offset) + if reader.observer != nil && count > 0 { + reader.observer.UsageBytesRead(int64(count)) + } + return count, err +} + +type checkpointParseResult struct { + FileParseResult + ProcessedBytes int64 + BoundaryDigest store.UsageKey + CheckpointEventDigest store.UsageKey + ParserStateJSON string +} + +func parseCodexCheckpointFile( + ctx context.Context, + file SourceFile, + start int64, + state codexParserState, + eventDigest store.UsageKey, + expectedBoundary store.UsageKey, + fileSystem usageFileSystem, + observer UsageSyncObserver, +) (checkpointParseResult, error) { + if fileSystem == nil { + fileSystem = osUsageFileSystem{} + } + handle, err := openStableUsageFile(file, fileSystem, observer) + if err != nil { + return checkpointParseResult{}, err + } + defer handle.Close() + if start < 0 || start > file.SizeBytes { + return checkpointParseResult{}, errors.New("Codex usage checkpoint is outside the session file") + } + boundary, err := readUsageBoundaryWindow(handle, start, observer) + if err != nil { + return checkpointParseResult{}, err + } + if start > 0 { + if boundary.digest() != expectedBoundary { + return checkpointParseResult{}, errUsageBoundaryChanged + } + } + if eventDigest.IsZero() { + eventDigest = initialCheckpointEventDigest(ProviderCodex) + } + result := checkpointParseResult{ProcessedBytes: start, CheckpointEventDigest: eventDigest} + section := io.NewSectionReader(observedReaderAt{reader: handle, observer: observer}, start, file.SizeBytes-start) + reader := bufio.NewReaderSize(section, 64*1024) + for { + if err := ctx.Err(); err != nil { + return checkpointParseResult{}, err + } + rawLine, boundarySuffix, tooLong, terminated, consumed, err := readCheckpointLine( + ctx, + reader, + maxCodexSessionLineBytes, + ) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return checkpointParseResult{}, err + } + if tooLong { + result.InvalidLines++ + result.ProcessedBytes += consumed + boundary.appendSuffix(boundarySuffix, consumed) + continue + } + line := bytes.TrimSpace(rawLine) + if len(line) == 0 { + result.ProcessedBytes += consumed + boundary.appendSuffix(boundarySuffix, consumed) + continue + } + event, invalid, unsupported := state.parseLine(line) + if invalid && !terminated { + break + } + if invalid { + result.InvalidLines++ + } else if unsupported { + result.UnsupportedLines++ + } else if event != nil { + result.Events = append(result.Events, *event) + result.CheckpointEventDigest = advanceCheckpointEventDigest( + ProviderCodex, + result.CheckpointEventDigest, + *event, + ) + } + result.ProcessedBytes += consumed + boundary.appendSuffix(boundarySuffix, consumed) + } + stateJSON, err := encodeCodexParserState(state) + if err != nil { + return checkpointParseResult{}, err + } + result.ParserStateJSON = stateJSON + result.BoundaryDigest = boundary.digest() + if err := verifyStableUsageFile(file, handle, fileSystem); err != nil { + return checkpointParseResult{}, err + } + return result, nil +} + +func parseGrokBuildCheckpointFile( + ctx context.Context, + file SourceFile, + start int64, + eventDigest store.UsageKey, + expectedBoundary store.UsageKey, + fileSystem usageFileSystem, + observer UsageSyncObserver, +) (checkpointParseResult, error) { + if fileSystem == nil { + fileSystem = osUsageFileSystem{} + } + handle, err := openStableUsageFile(file, fileSystem, observer) + if err != nil { + return checkpointParseResult{}, err + } + defer handle.Close() + if start < 0 || start > file.SizeBytes { + return checkpointParseResult{}, errors.New("usage checkpoint is outside the Grok Build session file") + } + boundary, err := readUsageBoundaryWindow(handle, start, observer) + if err != nil { + return checkpointParseResult{}, err + } + if start > 0 { + if boundary.digest() != expectedBoundary { + return checkpointParseResult{}, errUsageBoundaryChanged + } + } + if eventDigest.IsZero() { + eventDigest = initialCheckpointEventDigest(grokconfig.ProviderID) + } + result := checkpointParseResult{ + ProcessedBytes: start, + CheckpointEventDigest: eventDigest, + ParserStateJSON: "{}", + } + section := io.NewSectionReader(observedReaderAt{reader: handle, observer: observer}, start, file.SizeBytes-start) + reader := bufio.NewReaderSize(section, 64*1024) + for { + if err := ctx.Err(); err != nil { + return checkpointParseResult{}, err + } + rawLine, boundarySuffix, tooLong, terminated, consumed, err := readCheckpointLine( + ctx, + reader, + maxGrokBuildSessionLineBytes, + ) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return checkpointParseResult{}, err + } + if tooLong { + return checkpointParseResult{}, errors.New("session file contains an oversized record") + } + line := bytes.TrimSpace(rawLine) + if len(line) == 0 { + result.ProcessedBytes += consumed + boundary.appendSuffix(boundarySuffix, consumed) + continue + } + events, unsupported, parseErr := parseGrokBuildSessionLine(line) + if parseErr != nil && !terminated { + break + } + if parseErr != nil { + return checkpointParseResult{}, parseErr + } + for _, event := range events { + result.Events = append(result.Events, event) + result.CheckpointEventDigest = advanceCheckpointEventDigest( + grokconfig.ProviderID, + result.CheckpointEventDigest, + event, + ) + } + if unsupported { + result.UnsupportedLines++ + } + result.ProcessedBytes += consumed + boundary.appendSuffix(boundarySuffix, consumed) + } + result.BoundaryDigest = boundary.digest() + if err := verifyStableUsageFile(file, handle, fileSystem); err != nil { + return checkpointParseResult{}, err + } + return result, nil +} + +var errUsageBoundaryChanged = errors.New("usage file changed before the saved checkpoint") + +func openStableUsageFile( + file SourceFile, + fileSystem usageFileSystem, + observer UsageSyncObserver, +) (usageOpenFile, error) { + if fileSystem == nil { + fileSystem = osUsageFileSystem{} + } + before, err := fileSystem.Lstat(file.Path) + if err != nil { + return nil, err + } + if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() || + sourceFileFromInfo(file.Path, file.SourceKey, before).MetadataDigest != file.MetadataDigest { + return nil, errors.New("usage file changed before it was read") + } + handle, err := fileSystem.Open(file.Path) + if err != nil { + return nil, err + } + if observer != nil { + observer.UsageFileOpened() + } + opened, err := handle.Stat() + if err != nil || !opened.Mode().IsRegular() || !os.SameFile(before, opened) { + _ = handle.Close() + if err != nil { + return nil, err + } + return nil, errors.New("usage file identity changed before it was read") + } + return handle, nil +} + +func verifyStableUsageFile(file SourceFile, handle usageOpenFile, fileSystem usageFileSystem) error { + opened, err := handle.Stat() + if err != nil { + return err + } + after, err := fileSystem.Lstat(file.Path) + if err != nil { + return err + } + if after.Mode()&os.ModeSymlink != 0 || !after.Mode().IsRegular() || + !os.SameFile(opened, after) || + sourceFileFromInfo(file.Path, file.SourceKey, opened).MetadataDigest != file.MetadataDigest || + sourceFileFromInfo(file.Path, file.SourceKey, after).MetadataDigest != file.MetadataDigest { + return errors.New("usage file changed while it was read") + } + return nil +} + +type usageBoundaryWindow struct { + data []byte +} + +func readUsageBoundaryWindow( + reader io.ReaderAt, + processedBytes int64, + observer UsageSyncObserver, +) (usageBoundaryWindow, error) { + if processedBytes < 0 { + return usageBoundaryWindow{}, errors.New("usage checkpoint offset is invalid") + } + start := processedBytes - usageBoundaryBytes + if start < 0 { + start = 0 + } + length := processedBytes - start + window := usageBoundaryWindow{data: make([]byte, length)} + if length == 0 { + return window, nil + } + section := io.NewSectionReader(observedReaderAt{reader: reader, observer: observer}, start, length) + if _, err := io.ReadFull(section, window.data); err != nil { + return usageBoundaryWindow{}, err + } + return window, nil +} + +func (window *usageBoundaryWindow) appendSuffix(suffix []byte, consumed int64) { + if consumed <= 0 { + return + } + if consumed >= usageBoundaryBytes { + window.data = append(window.data[:0], suffix...) + return + } + window.data = append(window.data, suffix...) + if excess := int64(len(window.data)) - usageBoundaryBytes; excess > 0 { + copy(window.data, window.data[excess:]) + window.data = window.data[:usageBoundaryBytes] + } +} + +func (window usageBoundaryWindow) digest() store.UsageKey { + hash := sha256.New() + _, _ = hash.Write([]byte("profiledeck-usage-file-boundary-v1\x00")) + _, _ = hash.Write(window.data) + var digest store.UsageKey + copy(digest[:], hash.Sum(nil)) + return digest +} + +func readCheckpointLine( + ctx context.Context, + reader *bufio.Reader, + limit int, +) ([]byte, []byte, bool, bool, int64, error) { + if limit <= 0 { + limit = maxCodexSessionLineBytes + } + var line []byte + var boundarySuffix []byte + var consumed int64 + tooLong := false + for { + if err := ctx.Err(); err != nil { + return nil, nil, false, false, consumed, err + } + fragment, err := reader.ReadSlice('\n') + consumed += int64(len(fragment)) + boundarySuffix = appendBoundarySuffix(boundarySuffix, fragment) + if !tooLong && len(line)+len(fragment) <= limit { + line = append(line, fragment...) + } else if len(fragment) > 0 { + tooLong = true + line = nil + } + switch { + case err == nil: + return line, boundarySuffix, tooLong, true, consumed, nil + case errors.Is(err, bufio.ErrBufferFull): + continue + case errors.Is(err, io.EOF) && consumed == 0: + return nil, nil, false, false, 0, io.EOF + case errors.Is(err, io.EOF): + return line, boundarySuffix, tooLong, false, consumed, nil + default: + return nil, nil, false, false, consumed, err + } + } +} + +func appendBoundarySuffix(suffix, fragment []byte) []byte { + if int64(len(fragment)) >= usageBoundaryBytes { + return append(suffix[:0], fragment[len(fragment)-int(usageBoundaryBytes):]...) + } + suffix = append(suffix, fragment...) + if excess := int64(len(suffix)) - usageBoundaryBytes; excess > 0 { + copy(suffix, suffix[excess:]) + suffix = suffix[:usageBoundaryBytes] + } + return suffix +} + +func initialCheckpointEventDigest(providerID string) store.UsageKey { + hash := sha256.Sum256([]byte("profiledeck-usage-event-checkpoint-v1\x00" + providerID)) + return store.UsageKey(hash) +} + +func advanceCheckpointEventDigest( + providerID string, + previous store.UsageKey, + event Event, +) store.UsageKey { + hash := sha256.New() + _, _ = hash.Write([]byte("profiledeck-usage-event-checkpoint-v1\x00")) + _, _ = hash.Write([]byte(providerID)) + _, _ = hash.Write([]byte{0}) + _, _ = hash.Write(previous[:]) + _, _ = hash.Write(event.EventKey[:]) + var encoded [8]byte + for _, value := range []int64{ + event.InputTokens, + event.CachedInputTokens, + event.OutputTokens, + event.TotalTokens, + } { + binary.BigEndian.PutUint64(encoded[:], uint64(value)) + _, _ = hash.Write(encoded[:]) + } + var digest store.UsageKey + copy(digest[:], hash.Sum(nil)) + return digest +} + +func checkpointEventDigest(providerID string, events []Event, limit int64) store.UsageKey { + if limit < 0 || limit > int64(len(events)) { + limit = int64(len(events)) + } + digest := initialCheckpointEventDigest(providerID) + for _, event := range events[:limit] { + digest = advanceCheckpointEventDigest(providerID, digest, event) + } + return digest +} + +func encodeCodexParserState(state codexParserState) (string, error) { + if err := validateCodexParserState(state); err != nil { + return "", err + } + encoded, err := json.Marshal(state) + if err != nil { + return "", err + } + if len(encoded) > maxCodexParserStateSize { + return "", errors.New("Codex usage parser checkpoint is invalid") + } + return string(encoded), nil +} + +func decodeCodexParserState(encoded string) (codexParserState, error) { + if len(encoded) == 0 || len(encoded) > maxCodexParserStateSize { + return codexParserState{}, errors.New("Codex usage parser checkpoint is invalid") + } + decoder := json.NewDecoder(strings.NewReader(encoded)) + decoder.DisallowUnknownFields() + var state codexParserState + if err := decoder.Decode(&state); err != nil { + return codexParserState{}, errors.New("Codex usage parser checkpoint is invalid") + } + if err := validateCodexParserState(state); err != nil { + return codexParserState{}, err + } + return state, nil +} + +func validateCodexParserState(state codexParserState) error { + if len(state.CurrentSessionKey) == 0 || len(state.CurrentSessionKey) > 256 || + store.NormalizeUsageModelKey(state.ModelForStorage) != state.ModelForStorage || + len(state.ModelForPricing) > 200 || state.PreviousTotals == nil || state.UsageOrdinals == nil { + return errors.New("Codex usage parser checkpoint is invalid") + } + for session, totals := range state.PreviousTotals { + if len(session) == 0 || len(session) > 256 || !totals.valid() { + return errors.New("Codex usage parser checkpoint is invalid") + } + } + for session, ordinal := range state.UsageOrdinals { + if len(session) == 0 || len(session) > 256 || ordinal <= 0 { + return errors.New("Codex usage parser checkpoint is invalid") + } + } + return nil +} diff --git a/internal/usage/incremental_reader_test.go b/internal/usage/incremental_reader_test.go new file mode 100644 index 0000000..7e6c98c --- /dev/null +++ b/internal/usage/incremental_reader_test.go @@ -0,0 +1,198 @@ +package usage + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/strahe/profiledeck/internal/store" +) + +type openHookUsageFileSystem struct { + hook func() +} + +func (fileSystem openHookUsageFileSystem) Lstat(path string) (os.FileInfo, error) { + return os.Lstat(path) +} + +func (fileSystem openHookUsageFileSystem) Open(path string) (usageOpenFile, error) { + handle, err := os.Open(path) + if err == nil && fileSystem.hook != nil { + fileSystem.hook() + } + return handle, err +} + +func TestCheckpointParsersRejectConcurrentAppend(t *testing.T) { + for _, test := range []struct { + name string + content string + parse func(context.Context, SourceFile, usageFileSystem) error + }{ + { + name: "Codex", + content: `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + parse: func(ctx context.Context, file SourceFile, fileSystem usageFileSystem) error { + _, err := parseCodexCheckpointFile( + ctx, + file, + 0, + newCodexParserState(file), + store.UsageKey{}, + store.UsageKey{}, + fileSystem, + nil, + ) + return err + }, + }, + { + name: "Grok Build", + content: `{"timestamp":0,"method":"session/update","params":{"sessionId":"s","update":{"sessionUpdate":"user_message_chunk"}}}`, + parse: func(ctx context.Context, file SourceFile, fileSystem usageFileSystem) error { + _, err := parseGrokBuildCheckpointFile( + ctx, + file, + 0, + store.UsageKey{}, + store.UsageKey{}, + fileSystem, + nil, + ) + return err + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "session.jsonl") + if err := os.WriteFile(path, []byte(test.content), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat fixture: %v", err) + } + file := sourceFileFromInfo(path, usageTestEventKey(test.name), info) + fileSystem := openHookUsageFileSystem{hook: func() { + handle, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open append fixture: %v", err) + } + if _, err := handle.WriteString("\n" + test.content); err != nil { + _ = handle.Close() + t.Fatalf("append fixture: %v", err) + } + if err := handle.Close(); err != nil { + t.Fatalf("close append fixture: %v", err) + } + }} + if err := test.parse(context.Background(), file, fileSystem); err == nil { + t.Fatal("concurrent append unexpectedly advanced the checkpoint") + } + }) + } +} + +func TestCheckpointParsersResumeUnterminatedRecord(t *testing.T) { + t.Run("Codex", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "session.jsonl") + partial := `{"type":"session_meta","session_id":"` + if err := os.WriteFile(path, []byte(partial), 0o600); err != nil { + t.Fatalf("write partial fixture: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat partial fixture: %v", err) + } + file := sourceFileFromInfo(path, usageTestEventKey("Codex partial"), info) + first, err := parseCodexCheckpointFile( + context.Background(), file, 0, newCodexParserState(file), + store.UsageKey{}, store.UsageKey{}, nil, nil, + ) + if err != nil || first.ProcessedBytes != 0 || first.InvalidLines != 0 { + t.Fatalf("partial Codex checkpoint = %#v, err = %v", first, err) + } + appendUsageCheckpointFixture(t, path, `session-tail"}`+"\n") + info, err = os.Stat(path) + if err != nil { + t.Fatalf("stat completed fixture: %v", err) + } + file = sourceFileFromInfo(path, file.SourceKey, info) + state, err := decodeCodexParserState(first.ParserStateJSON) + if err != nil { + t.Fatalf("decode partial parser state: %v", err) + } + completed, err := parseCodexCheckpointFile( + context.Background(), file, first.ProcessedBytes, state, + first.CheckpointEventDigest, first.BoundaryDigest, nil, nil, + ) + if err != nil || completed.ProcessedBytes != info.Size() || completed.InvalidLines != 0 { + t.Fatalf("completed Codex checkpoint = %#v, err = %v", completed, err) + } + }) + + t.Run("Grok Build", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "updates.jsonl") + partial := `{"timestamp":1,"method":"session/update","params":{"sessionId":"session-tail","update":` + if err := os.WriteFile(path, []byte(partial), 0o600); err != nil { + t.Fatalf("write partial fixture: %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat partial fixture: %v", err) + } + file := sourceFileFromInfo(path, usageTestEventKey("Grok partial"), info) + first, err := parseGrokBuildCheckpointFile( + context.Background(), file, 0, store.UsageKey{}, store.UsageKey{}, nil, nil, + ) + if err != nil || first.ProcessedBytes != 0 || first.InvalidLines != 0 { + t.Fatalf("partial Grok Build checkpoint = %#v, err = %v", first, err) + } + appendUsageCheckpointFixture(t, path, `{"sessionUpdate":"agent_message_chunk"}}}`+"\n") + info, err = os.Stat(path) + if err != nil { + t.Fatalf("stat completed fixture: %v", err) + } + file = sourceFileFromInfo(path, file.SourceKey, info) + completed, err := parseGrokBuildCheckpointFile( + context.Background(), file, first.ProcessedBytes, + first.CheckpointEventDigest, first.BoundaryDigest, nil, nil, + ) + if err != nil || completed.ProcessedBytes != info.Size() || completed.InvalidLines != 0 { + t.Fatalf("completed Grok Build checkpoint = %#v, err = %v", completed, err) + } + }) +} + +func appendUsageCheckpointFixture(t *testing.T, path, content string) { + t.Helper() + handle, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Fatalf("open fixture for append: %v", err) + } + if _, err := handle.WriteString(content); err != nil { + _ = handle.Close() + t.Fatalf("append fixture: %v", err) + } + if err := handle.Close(); err != nil { + t.Fatalf("close appended fixture: %v", err) + } +} + +func TestEncodeCodexParserStateRejectsOversizedCheckpoint(t *testing.T) { + state := newCodexParserState(SourceFile{ + Path: "session.jsonl", + SourceKey: usageTestEventKey("oversized-state"), + }) + for index := range 12_000 { + session := fmt.Sprintf("session-%05d", index) + state.PreviousTotals[session] = TokenCounts{InputTokens: 1, TotalTokens: 1} + state.UsageOrdinals[session] = 1 + } + if _, err := encodeCodexParserState(state); err == nil { + t.Fatal("oversized parser checkpoint unexpectedly encoded") + } +} diff --git a/internal/usage/registry.go b/internal/usage/registry.go index 501c8b9..f6b1c93 100644 --- a/internal/usage/registry.go +++ b/internal/usage/registry.go @@ -13,10 +13,25 @@ import ( type Integration interface { ProviderID() string SourceIDs() []string - Sync(context.Context, store.Factory, SyncProvisionMode) (UsageSyncResult, error) + Sync(context.Context, store.Factory, SyncOptions) (SyncOutcome, error) PricingInfo() UsagePricingInfo } +type SyncOptions struct { + ProvisionMode SyncProvisionMode + ForceObservedRetry bool + OnWorkDetected func() + Observer UsageSyncObserver + fileSystem usageFileSystem +} + +type SyncOutcome struct { + Result UsageSyncResult + Performed bool +} + +type BackgroundSyncOutcome = SyncOutcome + type SyncProvisionMode int const ( diff --git a/internal/usage/registry_test.go b/internal/usage/registry_test.go index 224715c..ae85957 100644 --- a/internal/usage/registry_test.go +++ b/internal/usage/registry_test.go @@ -26,12 +26,15 @@ func (integration *registryTestIntegration) SourceIDs() []string { return append([]string(nil), integration.sources...) } -func (integration *registryTestIntegration) Sync(context.Context, store.Factory, SyncProvisionMode) (UsageSyncResult, error) { +func (integration *registryTestIntegration) Sync(context.Context, store.Factory, SyncOptions) (SyncOutcome, error) { integration.called = true if integration.syncErr != nil { - return UsageSyncResult{}, integration.syncErr + return SyncOutcome{}, integration.syncErr } - return UsageSyncResult{ProviderID: integration.provider, Source: integration.sources[0]}, nil + return SyncOutcome{ + Result: UsageSyncResult{ProviderID: integration.provider, Source: integration.sources[0]}, + Performed: true, + }, nil } func (*registryTestIntegration) PricingInfo() UsagePricingInfo { diff --git a/internal/usage/service.go b/internal/usage/service.go index a22e9b3..bdbd531 100644 --- a/internal/usage/service.go +++ b/internal/usage/service.go @@ -64,37 +64,41 @@ type UsageSummaryResult struct { } func (service *Service) Sync(ctx context.Context, req UsageSyncRequest) (UsageSyncResult, error) { - return service.sync(ctx, req, SyncProvisionProvider) + outcome, err := service.sync(ctx, req, SyncOptions{ + ProvisionMode: SyncProvisionProvider, + ForceObservedRetry: true, + }) + return outcome.Result, err } func (service *Service) sync( ctx context.Context, req UsageSyncRequest, - mode SyncProvisionMode, -) (UsageSyncResult, error) { + options SyncOptions, +) (SyncOutcome, error) { providerID, integration, appErr := service.resolveIntegration(req.ProviderID) if appErr != nil { - return UsageSyncResult{}, appErr + return SyncOutcome{}, appErr } workCtx, releaseWork, err := service.acquireSyncForWork(ctx) if err != nil { - return UsageSyncResult{}, usageSyncError(providerID, err) + return SyncOutcome{}, usageSyncError(providerID, err) } defer releaseWork() - result, err := integration.Sync(workCtx, service.stores, mode) - if mode == SyncExistingProvider && errors.Is(err, store.ErrUsageProviderMissing) { - return UsageSyncResult{ + outcome, err := integration.Sync(workCtx, service.stores, options) + if options.ProvisionMode == SyncExistingProvider && errors.Is(err, store.ErrUsageProviderMissing) { + return SyncOutcome{Result: UsageSyncResult{ ProviderID: providerID, Source: summarySource(integration.SourceIDs()), Errors: []UsageImportError{}, - }, nil + }}, nil } if err != nil { - return UsageSyncResult{}, usageSyncError(providerID, err) + return SyncOutcome{}, usageSyncError(providerID, err) } - return result, nil + return outcome, nil } type phaseTimeoutKey struct{} @@ -219,8 +223,14 @@ func (service *Service) SyncCodex(ctx context.Context) (UsageSyncResult, error) // SyncCodexBackground never provisions a deleted Provider. A later explicit // sync remains the only action that may recreate it. -func (service *Service) SyncCodexBackground(ctx context.Context) (UsageSyncResult, error) { - return service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncExistingProvider) +func (service *Service) SyncCodexBackground( + ctx context.Context, + onWorkDetected func(), +) (BackgroundSyncOutcome, error) { + return service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + OnWorkDetected: onWorkDetected, + }) } func (service *Service) SyncGrokBuild(ctx context.Context) (UsageSyncResult, error) { @@ -231,8 +241,12 @@ func (service *Service) SyncGrokBuild(ctx context.Context) (UsageSyncResult, err func (service *Service) SyncProviderBackground( ctx context.Context, providerID string, -) (UsageSyncResult, error) { - return service.sync(ctx, UsageSyncRequest{ProviderID: providerID}, SyncExistingProvider) + onWorkDetected func(), +) (BackgroundSyncOutcome, error) { + return service.sync(ctx, UsageSyncRequest{ProviderID: providerID}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + OnWorkDetected: onWorkDetected, + }) } func (service *Service) Summary(ctx context.Context, req UsageSummaryRequest) (UsageSummaryResult, error) { diff --git a/internal/usage/service_test.go b/internal/usage/service_test.go index 15ce11e..dd9ec27 100644 --- a/internal/usage/service_test.go +++ b/internal/usage/service_test.go @@ -37,23 +37,23 @@ func (*pausedUsageIntegration) PricingInfo() UsagePricingInfo { func (integration *pausedUsageIntegration) Sync( ctx context.Context, stores store.Factory, - _ SyncProvisionMode, -) (UsageSyncResult, error) { + _ SyncOptions, +) (SyncOutcome, error) { close(integration.started) select { case <-ctx.Done(): - return UsageSyncResult{}, ctx.Err() + return SyncOutcome{}, ctx.Err() case <-integration.resume: } db, err := stores.OpenHealthy(ctx, false) if err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } defer db.Close() if _, err := db.BeginUsageSync(ctx, ProviderCodex, SourceCodexSessionJSONL, CodexUsageIdentityRevision); err != nil { - return UsageSyncResult{}, err + return SyncOutcome{}, err } - return UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL}, nil + return SyncOutcome{Result: UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL}, Performed: true}, nil } type serialUsageIntegration struct { @@ -74,8 +74,8 @@ func (*serialUsageIntegration) PricingInfo() UsagePricingInfo { return UsagePric func (integration *serialUsageIntegration) Sync( ctx context.Context, _ store.Factory, - _ SyncProvisionMode, -) (UsageSyncResult, error) { + _ SyncOptions, +) (SyncOutcome, error) { n := integration.inFlight.Add(1) for { cur := integration.maxFlight.Load() @@ -88,7 +88,7 @@ func (integration *serialUsageIntegration) Sync( select { case <-integration.release: case <-ctx.Done(): - return UsageSyncResult{}, ctx.Err() + return SyncOutcome{}, ctx.Err() } if integration.workHold > 0 && (integration.workHoldFromEnt == 0 || entry >= integration.workHoldFromEnt) { timer := time.NewTimer(integration.workHold) @@ -96,10 +96,10 @@ func (integration *serialUsageIntegration) Sync( select { case <-timer.C: case <-ctx.Done(): - return UsageSyncResult{}, ctx.Err() + return SyncOutcome{}, ctx.Err() } } - return UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL}, nil + return SyncOutcome{Result: UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL}, Performed: true}, nil } func TestUsageSyncSerializesConcurrentImports(t *testing.T) { @@ -283,7 +283,7 @@ func TestBackgroundUsageSyncDoesNotRecreateProviderDeletedAfterDispatch(t *testi service := NewService(environment.runtime.StoreFactory(), MustRegistry(integration)) errCh := make(chan error, 1) go func() { - _, syncErr := service.SyncCodexBackground(ctx) + _, syncErr := service.SyncCodexBackground(ctx, nil) errCh <- syncErr }() select { @@ -530,6 +530,272 @@ func TestUsageSyncCodexImportsOnlyAppendedEvents(t *testing.T) { } } +type usageSyncReadObserver struct { + opened atomic.Int64 + bytes atomic.Int64 +} + +func (observer *usageSyncReadObserver) UsageFileOpened() { + observer.opened.Add(1) +} + +func (observer *usageSyncReadObserver) UsageBytesRead(count int64) { + observer.bytes.Add(count) +} + +func TestBackgroundUsageSyncNoopDoesNotReadOrWrite(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + path := writeAppUsageFixture(t, codexDir, strings.Join([]string{ + `{"type":"session_meta","session_id":"session-idle"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + }, "\n")) + environment := newUsageTestEnvironment(t, configDir, codexDir) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncCodex(ctx); err != nil { + t.Fatalf("initial usage sync: %v", err) + } + db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("open inspection Store: %v", err) + } + sourceBefore, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if err != nil { + _ = db.Close() + t.Fatalf("read source before no-op: %v", err) + } + fileKey, err := SourceKey(path) + if err != nil { + _ = db.Close() + t.Fatalf("derive file key: %v", err) + } + cursorBefore, err := db.GetCodexUsageImportFile(ctx, sourceBefore.ID, fileKey) + if err != nil { + _ = db.Close() + t.Fatalf("read cursor before no-op: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close inspection Store: %v", err) + } + + observer := &usageSyncReadObserver{} + outcome, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: observer, + }) + if err != nil || outcome.Performed || outcome.Result.SkippedUnchangedFiles != 1 { + t.Fatalf("idle background outcome = %#v, err = %v", outcome, err) + } + if observer.opened.Load() != 0 || observer.bytes.Load() != 0 { + t.Fatalf("idle background read session content: opens=%d bytes=%d", observer.opened.Load(), observer.bytes.Load()) + } + db, err = environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("reopen inspection Store: %v", err) + } + defer db.Close() + sourceAfter, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if err != nil { + t.Fatalf("read source after no-op: %v", err) + } + cursorAfter, err := db.GetCodexUsageImportFile(ctx, sourceAfter.ID, fileKey) + if err != nil { + t.Fatalf("read cursor after no-op: %v", err) + } + if sourceAfter != sourceBefore || cursorAfter != cursorBefore { + t.Fatalf("idle background wrote usage state: source=%#v cursor=%#v", sourceAfter, cursorAfter) + } +} + +func TestBackgroundUsageSyncCompletesAfterInterruptedGeneration(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + writeAppUsageFixture(t, codexDir, strings.Join([]string{ + `{"type":"session_meta","session_id":"session-recovery"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + }, "\n")) + environment := newUsageTestEnvironment(t, configDir, codexDir) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncCodex(ctx); err != nil { + t.Fatalf("initial usage sync: %v", err) + } + db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, false) + if err != nil { + t.Fatalf("open Store: %v", err) + } + interrupted, err := db.BeginUsageSync( + ctx, + ProviderCodex, + SourceCodexSessionJSONL, + CodexUsageIdentityRevision, + ) + if closeErr := db.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + t.Fatalf("leave interrupted generation: %v", err) + } + + observer := &usageSyncReadObserver{} + outcome, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: observer, + }) + if err != nil || !outcome.Performed || outcome.Result.SkippedUnchangedFiles != 1 { + t.Fatalf("recovery sync outcome = %#v, err = %v", outcome, err) + } + if observer.opened.Load() != 0 || observer.bytes.Load() != 0 { + t.Fatalf("recovery sync read unchanged content: opens=%d bytes=%d", observer.opened.Load(), observer.bytes.Load()) + } + db, err = environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("reopen Store: %v", err) + } + defer db.Close() + completed, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if err != nil { + t.Fatalf("read recovered source: %v", err) + } + if completed.SyncGeneration <= interrupted.SyncGeneration || + completed.CompletedGeneration != completed.SyncGeneration { + t.Fatalf("interrupted generation was not completed: before=%#v after=%#v", interrupted, completed) + } +} + +func TestBackgroundUsageSyncReadsOnlyCodexAppendAndBoundary(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + padding := strings.Repeat("{\"type\":\"response_item\",\"payload\":{\"type\":\"message\"}}\n", 20_000) + path := writeAppUsageFixture(t, codexDir, padding+strings.Join([]string{ + `{"type":"session_meta","session_id":"session-tail"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":100,"output_tokens":10}}}}`, + }, "\n")) + environment := newUsageTestEnvironment(t, configDir, codexDir) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncCodex(ctx); err != nil { + t.Fatalf("initial usage sync: %v", err) + } + appended := `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":120,"output_tokens":12}}}}` + appendAppUsageFixture(t, path, appended) + + observer := &usageSyncReadObserver{} + outcome, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: observer, + }) + if err != nil || !outcome.Performed || outcome.Result.ImportedEvents != 1 { + t.Fatalf("Codex tail outcome = %#v, err = %v", outcome, err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat Codex fixture: %v", err) + } + maxRead := usageBoundaryBytes + int64(len(appended)+1) + if observer.opened.Load() != 1 || observer.bytes.Load() > maxRead { + t.Fatalf("Codex append read was not bounded: opens=%d bytes=%d size=%d", observer.opened.Load(), observer.bytes.Load(), info.Size()) + } + nextAppended := `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":130,"output_tokens":13}}}}` + appendAppUsageFixture(t, path, nextAppended) + nextObserver := &usageSyncReadObserver{} + next, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: nextObserver, + }) + if err != nil || !next.Performed || next.Result.ImportedEvents != 1 { + t.Fatalf("second Codex tail outcome = %#v, err = %v", next, err) + } + nextMaxRead := usageBoundaryBytes + int64(len(nextAppended)+1) + if nextObserver.opened.Load() != 1 || nextObserver.bytes.Load() > nextMaxRead { + t.Fatalf("second Codex append read was not bounded: opens=%d bytes=%d", nextObserver.opened.Load(), nextObserver.bytes.Load()) + } + summary, err := environment.service.Summary(ctx, UsageSummaryRequest{ProviderID: ProviderCodex}) + if err != nil || summary.EventCount != 3 || summary.TotalTokens != 143 { + t.Fatalf("Codex tail summary = %#v, err = %v", summary, err) + } +} + +func TestBackgroundUsageSyncQuarantinesSameCodexVersionOnce(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + path := writeAppUsageFixture(t, codexDir, strings.Join([]string{ + `{"type":"session_meta","session_id":"session-quarantine"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + }, "\n")) + environment := newUsageTestEnvironment(t, configDir, codexDir) + initialized, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx) + if err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncCodex(ctx); err != nil { + t.Fatalf("initial usage sync: %v", err) + } + writeAppUsageFile(t, path, "") + first, err := environment.service.SyncCodexBackground(ctx, nil) + if err != nil || !first.Performed || len(first.Result.Errors) != 1 { + t.Fatalf("first quarantine outcome = %#v, err = %v", first, err) + } + rawDB, err := sql.Open("sqlite", initialized.DatabasePath) + if err != nil { + t.Fatalf("open usage database: %v", err) + } + var generationBefore, completedBefore, observations int64 + if err := rawDB.QueryRowContext(ctx, ` + SELECT sync_generation, completed_generation, + (SELECT COUNT(1) FROM usage_import_observations) + FROM usage_sources + `).Scan(&generationBefore, &completedBefore, &observations); err != nil { + _ = rawDB.Close() + t.Fatalf("read quarantine state: %v", err) + } + if observations != 1 || generationBefore != completedBefore { + _ = rawDB.Close() + t.Fatalf("unexpected quarantine state: generation=%d completed=%d observations=%d", generationBefore, completedBefore, observations) + } + if err := rawDB.Close(); err != nil { + t.Fatalf("close usage database: %v", err) + } + + observer := &usageSyncReadObserver{} + second, err := environment.service.sync(ctx, UsageSyncRequest{ProviderID: ProviderCodex}, SyncOptions{ + ProvisionMode: SyncExistingProvider, + Observer: observer, + }) + if err != nil || second.Performed || len(second.Result.Errors) != 1 { + t.Fatalf("repeated quarantine outcome = %#v, err = %v", second, err) + } + if observer.opened.Load() != 0 || observer.bytes.Load() != 0 { + t.Fatalf("repeated quarantine read content: opens=%d bytes=%d", observer.opened.Load(), observer.bytes.Load()) + } + rawDB, err = sql.Open("sqlite", initialized.DatabasePath) + if err != nil { + t.Fatalf("reopen usage database: %v", err) + } + defer rawDB.Close() + var generationAfter, completedAfter int64 + if err := rawDB.QueryRowContext(ctx, ` + SELECT sync_generation, completed_generation FROM usage_sources + `).Scan(&generationAfter, &completedAfter); err != nil { + t.Fatalf("read repeated quarantine state: %v", err) + } + if generationAfter != generationBefore || completedAfter != completedBefore { + t.Fatalf("repeated quarantine wrote source: generation %d/%d -> %d/%d", generationBefore, completedBefore, generationAfter, completedAfter) + } +} + func TestUsageSyncCodexRejectsTruncatedAndRewrittenHistory(t *testing.T) { for _, test := range []struct { name string @@ -1076,7 +1342,7 @@ func TestProviderDeleteClearsUsageAndOnlyExplicitSyncRecreatesIt(t *testing.T) { if err != nil || summary.EventCount != 0 || summary.TotalTokens != 0 { t.Fatalf("Provider deletion retained Usage: summary=%#v err=%v", summary, err) } - if _, err := environment.service.SyncCodexBackground(ctx); err != nil { + if _, err := environment.service.SyncCodexBackground(ctx, nil); err != nil { t.Fatalf("background sync for a deleted Provider should be a no-op: %v", err) } db, err = environment.runtime.StoreFactory().OpenHealthy(ctx, true) diff --git a/internal/usage/usage.go b/internal/usage/usage.go index 8dad5aa..9d425a0 100644 --- a/internal/usage/usage.go +++ b/internal/usage/usage.go @@ -2,8 +2,10 @@ package usage import ( "crypto/sha256" + "encoding/binary" "fmt" "math" + "os" "path/filepath" "runtime" "strings" @@ -51,10 +53,14 @@ type Event struct { } type SourceFile struct { - Path string - SourceKey store.UsageKey - ModifiedUnixMS int64 - SizeBytes int64 + Path string + SourceKey store.UsageKey + ModifiedUnixMS int64 + ModifiedUnixNano int64 + SizeBytes int64 + Mode uint32 + FileIdentityDigest store.UsageKey + MetadataDigest store.UsageKey } type FileParseResult struct { @@ -217,6 +223,34 @@ func SourceKey(path string) (store.UsageKey, error) { return store.UsageKey(sum), nil } +func sourceFileFromInfo(path string, sourceKey store.UsageKey, info os.FileInfo) SourceFile { + file := SourceFile{ + Path: path, + SourceKey: sourceKey, + ModifiedUnixMS: info.ModTime().UnixMilli(), + ModifiedUnixNano: info.ModTime().UnixNano(), + SizeBytes: info.Size(), + Mode: uint32(info.Mode()), + FileIdentityDigest: sourceFileIdentityDigest(path, info), + } + file.MetadataDigest = sourceFileMetadataDigest(file) + return file +} + +func sourceFileMetadataDigest(file SourceFile) store.UsageKey { + hash := sha256.New() + _, _ = hash.Write([]byte("profiledeck-usage-file-metadata-v1\x00")) + var encoded [8]byte + for _, value := range []int64{file.SizeBytes, file.ModifiedUnixNano, int64(file.Mode)} { + binary.BigEndian.PutUint64(encoded[:], uint64(value)) + _, _ = hash.Write(encoded[:]) + } + _, _ = hash.Write(file.FileIdentityDigest[:]) + var digest store.UsageKey + copy(digest[:], hash.Sum(nil)) + return digest +} + func EventID(providerID, source string, usageOrdinal int64, sessionID, model string, tokens TokenCounts) store.UsageKey { if providerID == "" || source == "" || usageOrdinal <= 0 || sessionID == "" { return store.UsageKey{} From a2f3c8d33003dc168a9e8da8ff056c6d1d27f1eb Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:26:35 +0800 Subject: [PATCH 2/4] fix(desktop): avoid runtime side effects in error handling --- desktop/frontend/src/lib/desktop-errors.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/desktop/frontend/src/lib/desktop-errors.ts b/desktop/frontend/src/lib/desktop-errors.ts index fd014b2..efce967 100644 --- a/desktop/frontend/src/lib/desktop-errors.ts +++ b/desktop/frontend/src/lib/desktop-errors.ts @@ -1,5 +1,3 @@ -import { CancelError } from "@wailsio/runtime"; - import { translate } from "$lib/i18n"; type DesktopErrorShape = { @@ -131,7 +129,6 @@ export function isDesktopErrorCode(value: unknown, code: string): boolean { export function isCancelError(value: unknown): boolean { if (!value) return false; - if (value instanceof CancelError) return true; if (Array.isArray(value)) return value.some(isCancelError); const typed = value as DesktopErrorShape; if (typed.name === "CancelError" || typed.code === "CANCELED" || typed.Code === "CANCELED") return true; From f11bf221dbdd1af13cbc4654d5a6990e46eff536 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:47:25 +0800 Subject: [PATCH 3/4] fix(usage): preserve concurrent checkpoints and prefix facts --- internal/store/usage_codex.go | 113 +++++++++++++++++ internal/store/usage_codex_test.go | 73 +++++++++++ internal/usage/codex_sync.go | 72 +++++++---- internal/usage/grok_build_sync.go | 38 +++--- internal/usage/grok_build_sync_test.go | 74 ++++++++++++ internal/usage/service_test.go | 160 +++++++++++++++++++++++++ 6 files changed, 493 insertions(+), 37 deletions(-) diff --git a/internal/store/usage_codex.go b/internal/store/usage_codex.go index 88f73c4..108e4f7 100644 --- a/internal/store/usage_codex.go +++ b/internal/store/usage_codex.go @@ -130,6 +130,119 @@ func (s *Store) ListCodexUsageImportFiles(ctx context.Context, sourceID int64) ( return files, rows.Err() } +// CodexUsageFactPrefixMatches verifies the persisted, non-cost semantics of a +// parsed Codex prefix before the caller discards that prefix during a full +// checkpoint revalidation. +func (s *Store) CodexUsageFactPrefixMatches( + ctx context.Context, + sourceID int64, + facts []CreateUsageFactParams, +) (bool, error) { + if sourceID <= 0 { + return false, errors.New("Codex usage fact prefix source is invalid") + } + if len(facts) == 0 { + return true, nil + } + for _, fact := range facts { + if fact.SourceID != sourceID { + return false, errors.New("Codex usage fact prefix source is invalid") + } + if err := validateUsageFact(fact); err != nil { + return false, err + } + } + + const queryBatchSize = 200 + for start := 0; start < len(facts); start += queryBatchSize { + end := start + queryBatchSize + if end > len(facts) { + end = len(facts) + } + batch := facts[start:end] + placeholders := strings.TrimRight(strings.Repeat("?,", len(batch)), ",") + rows, err := s.executor().QueryContext(ctx, ` + SELECT facts.event_key, facts.source_id, + COALESCE(sessions.session_key, ''), + COALESCE(models.model_key, ''), + facts.occurred_at_unix_ms, + facts.input_tokens, facts.cached_input_tokens, + facts.output_tokens, facts.total_tokens + FROM usage_facts AS facts + LEFT JOIN usage_sessions AS sessions + ON sessions.source_id = facts.source_id AND sessions.id = facts.session_id + LEFT JOIN usage_models AS models + ON models.source_id = facts.source_id AND models.id = facts.model_id + WHERE facts.source_id = ? AND facts.event_key IN (`+placeholders+`) + `, append([]any{sourceID}, usageKeysAsArgs(batch)...)...) + if err != nil { + return false, err + } + + persisted := make(map[UsageKey]codexUsageFactSnapshot, len(batch)) + for rows.Next() { + var snapshot codexUsageFactSnapshot + if err := rows.Scan( + &snapshot.EventKey, + &snapshot.SourceID, + &snapshot.SessionKey, + &snapshot.ModelKey, + &snapshot.OccurredAtUnixMS, + &snapshot.InputTokens, + &snapshot.CachedInputTokens, + &snapshot.OutputTokens, + &snapshot.TotalTokens, + ); err != nil { + _ = rows.Close() + return false, err + } + persisted[snapshot.EventKey] = snapshot + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return false, err + } + if err := rows.Close(); err != nil { + return false, err + } + + for _, fact := range batch { + snapshot, ok := persisted[fact.EventKey] + if !ok || snapshot.SourceID != sourceID || + snapshot.SessionKey != fact.SessionKey || + snapshot.ModelKey != NormalizeUsageModelKey(fact.ModelKey) || + snapshot.OccurredAtUnixMS != fact.OccurredAtUnixMS || + snapshot.InputTokens != fact.InputTokens || + snapshot.CachedInputTokens != fact.CachedInputTokens || + snapshot.OutputTokens != fact.OutputTokens || + snapshot.TotalTokens != fact.TotalTokens { + return false, nil + } + } + } + return true, nil +} + +type codexUsageFactSnapshot struct { + EventKey UsageKey + SourceID int64 + SessionKey string + ModelKey string + OccurredAtUnixMS int64 + InputTokens int64 + CachedInputTokens int64 + OutputTokens int64 + TotalTokens int64 +} + +func usageKeysAsArgs(facts []CreateUsageFactParams) []any { + args := make([]any, len(facts)) + for index, fact := range facts { + args[index] = fact.EventKey + } + return args +} + func scanCodexUsageImportFile(row rowScanner) (CodexUsageImportFile, error) { var cursor CodexUsageImportFile if err := row.Scan( diff --git a/internal/store/usage_codex_test.go b/internal/store/usage_codex_test.go index 87fd1b6..4b41262 100644 --- a/internal/store/usage_codex_test.go +++ b/internal/store/usage_codex_test.go @@ -71,6 +71,79 @@ func TestCodexUsageImportFileUsesCompareAndSwap(t *testing.T) { } } +func TestCodexUsageFactPrefixMatchesPersistedSemantics(t *testing.T) { + ctx := context.Background() + db := migratedTestStore(t, ctx) + defer closeTestStore(t, db) + createUsageProviderFixture(t, ctx, db, "codex") + source, err := db.BeginUsageSync(ctx, "codex", "codex-session-jsonl", 1) + if err != nil { + t.Fatalf("begin usage sync: %v", err) + } + facts := []CreateUsageFactParams{ + { + EventKey: testUsageKey("prefix-first"), + SourceID: source.ID, + SessionKey: "session-prefix", + ModelKey: "gpt-5.3-codex", + OccurredAtUnixMS: 1_000, + InputTokens: 10, + OutputTokens: 2, + TotalTokens: 12, + CostStatus: UsageCostStatusUnknown, + }, + { + EventKey: testUsageKey("prefix-second"), + SourceID: source.ID, + SessionKey: "session-prefix", + ModelKey: "gpt-5.3-codex", + OccurredAtUnixMS: 2_000, + InputTokens: 20, + OutputTokens: 4, + TotalTokens: 24, + CostStatus: UsageCostStatusUnknown, + }, + } + if _, err := db.InsertUsageFacts(ctx, testUsageFactBatch(source, facts)); err != nil { + t.Fatalf("insert usage facts: %v", err) + } + + matched, err := db.CodexUsageFactPrefixMatches(ctx, source.ID, facts) + if err != nil || !matched { + t.Fatalf("matching prefix = %v, err = %v", matched, err) + } + + costChanged := append([]CreateUsageFactParams(nil), facts...) + estimatedCost := int64(123) + costChanged[0].EstimatedCostMicros = &estimatedCost + costChanged[0].CostStatus = UsageCostStatusEstimated + matched, err = db.CodexUsageFactPrefixMatches(ctx, source.ID, costChanged) + if err != nil || !matched { + t.Fatalf("derived cost change matched prefix = %v, err = %v", matched, err) + } + + timestampChanged := append([]CreateUsageFactParams(nil), facts...) + timestampChanged[0].OccurredAtUnixMS++ + matched, err = db.CodexUsageFactPrefixMatches(ctx, source.ID, timestampChanged) + if err != nil || matched { + t.Fatalf("timestamp change matched prefix = %v, err = %v", matched, err) + } + + modelChanged := append([]CreateUsageFactParams(nil), facts...) + modelChanged[0].ModelKey = "gpt-5.4" + matched, err = db.CodexUsageFactPrefixMatches(ctx, source.ID, modelChanged) + if err != nil || matched { + t.Fatalf("model change matched prefix = %v, err = %v", matched, err) + } + + missing := append([]CreateUsageFactParams(nil), facts...) + missing[1].EventKey = testUsageKey("prefix-missing") + matched, err = db.CodexUsageFactPrefixMatches(ctx, source.ID, missing) + if err != nil || matched { + t.Fatalf("missing fact matched prefix = %v, err = %v", matched, err) + } +} + func TestCodexUsageFinalizationCountsOnlyPersistedCursors(t *testing.T) { ctx := context.Background() db := migratedTestStore(t, ctx) diff --git a/internal/usage/codex_sync.go b/internal/usage/codex_sync.go index 46f4460..5496421 100644 --- a/internal/usage/codex_sync.go +++ b/internal/usage/codex_sync.go @@ -52,7 +52,7 @@ func (integration codexIntegration) Sync( stores store.Factory, options SyncOptions, ) (SyncOutcome, error) { - files, cursors, observed, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) + _, _, _, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) if err != nil { return SyncOutcome{}, err } @@ -71,19 +71,9 @@ func (integration codexIntegration) Sync( if err != nil { return SyncOutcome{}, err } - if cursors == nil { - cursorRows, err := db.ListCodexUsageImportFiles(ctx, source.ID) - if err != nil { - return SyncOutcome{}, err - } - cursors = codexCursorMap(cursorRows) - } - if observed == nil { - observationRows, err := db.ListUsageImportObservations(ctx, source.ID) - if err != nil { - return SyncOutcome{}, err - } - observed = observationMap(observationRows) + files, cursors, observed, err := integration.loadSyncSnapshot(ctx, db, source.ID) + if err != nil { + return SyncOutcome{}, err } result := UsageSyncResult{ProviderID: ProviderCodex, Source: SourceCodexSessionJSONL} discoveredFileKeys := make([]store.UsageKey, 0, len(files)) @@ -125,12 +115,16 @@ func (integration codexIntegration) Sync( invalidLines := parsed.InvalidLines unsupportedLines := parsed.UnsupportedLines if hasCursor { - if fullParse && !codexCheckpointPrefixMatches(parsed.Events, cursor) { - result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationHistoryChanged)) - observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) - continue - } if fullParse { + prefixMatches, err := codexCheckpointPrefixMatches(ctx, db, parsed.Events, cursor) + if err != nil { + return SyncOutcome{}, err + } + if !prefixMatches { + result.Errors = append(result.Errors, codexObservationError(file, store.UsageImportObservationHistoryChanged)) + observations = append(observations, newUsageObservation(source.ID, file, store.UsageImportObservationHistoryChanged)) + continue + } eventsToStore = parsed.Events[cursor.ImportedFacts:] importedFacts = int64(len(parsed.Events)) } else { @@ -214,6 +208,26 @@ func (integration codexIntegration) Sync( return SyncOutcome{Result: result, Performed: true}, nil } +func (integration codexIntegration) loadSyncSnapshot( + ctx context.Context, + db *store.Store, + sourceID int64, +) ([]SourceFile, map[store.UsageKey]store.CodexUsageImportFile, map[store.UsageKey]store.UsageImportObservation, error) { + files, err := ListCodexSessionFilesContext(ctx, integration.codexDir) + if err != nil { + return nil, nil, nil, apperror.Wrap(apperror.UsageImportFailed, "failed to list Codex session files", err) + } + cursorRows, err := db.ListCodexUsageImportFiles(ctx, sourceID) + if err != nil { + return nil, nil, nil, err + } + observationRows, err := db.ListUsageImportObservations(ctx, sourceID) + if err != nil { + return nil, nil, nil, err + } + return files, codexCursorMap(cursorRows), observationMap(observationRows), nil +} + func (integration codexIntegration) preflight( ctx context.Context, stores store.Factory, @@ -377,15 +391,27 @@ func parseCodexUsageChange( return parsed, true, err } -func codexCheckpointPrefixMatches(events []Event, cursor store.CodexUsageImportFile) bool { +func codexCheckpointPrefixMatches( + ctx context.Context, + db *store.Store, + events []Event, + cursor store.CodexUsageImportFile, +) (bool, error) { if cursor.ImportedFacts < 0 || cursor.ImportedFacts > int64(len(events)) || cursor.IdentityRevision != CodexUsageIdentityRevision { - return false + return false, nil } + var digestMatches bool if cursor.CheckpointRevision == usageCheckpointRevision { - return checkpointEventDigest(ProviderCodex, events, cursor.ImportedFacts) == cursor.CheckpointEventDigest + digestMatches = checkpointEventDigest(ProviderCodex, events, cursor.ImportedFacts) == cursor.CheckpointEventDigest + } else { + digestMatches = cursor.CheckpointRevision == 0 && EventDigest(events, cursor.ImportedFacts) == cursor.EventDigest + } + if !digestMatches { + return false, nil } - return cursor.CheckpointRevision == 0 && EventDigest(events, cursor.ImportedFacts) == cursor.EventDigest + facts := usageEventsToFactParams(cursor.SourceID, events[:cursor.ImportedFacts]) + return db.CodexUsageFactPrefixMatches(ctx, cursor.SourceID, facts) } func codexCursorMatchesFile(cursor store.CodexUsageImportFile, file SourceFile) bool { diff --git a/internal/usage/grok_build_sync.go b/internal/usage/grok_build_sync.go index 5a4cd5c..4c1f92a 100644 --- a/internal/usage/grok_build_sync.go +++ b/internal/usage/grok_build_sync.go @@ -59,7 +59,7 @@ func (integration grokBuildIntegration) Sync( stores store.Factory, options SyncOptions, ) (SyncOutcome, error) { - files, cursors, observed, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) + _, _, _, priceBackfill, noWorkResult, work, err := integration.preflight(ctx, stores, options) if err != nil { return SyncOutcome{}, err } @@ -78,19 +78,9 @@ func (integration grokBuildIntegration) Sync( if err != nil { return SyncOutcome{}, err } - if cursors == nil { - cursorRows, err := db.ListGrokBuildUsageImportFiles(ctx, source.ID) - if err != nil { - return SyncOutcome{}, err - } - cursors = grokBuildCursorMap(cursorRows) - } - if observed == nil { - observationRows, err := db.ListUsageImportObservations(ctx, source.ID) - if err != nil { - return SyncOutcome{}, err - } - observed = observationMap(observationRows) + files, cursors, observed, err := integration.loadSyncSnapshot(ctx, db, source.ID) + if err != nil { + return SyncOutcome{}, err } result := UsageSyncResult{ ProviderID: grokconfig.ProviderID, @@ -254,6 +244,26 @@ func (integration grokBuildIntegration) Sync( return SyncOutcome{Result: result, Performed: true}, nil } +func (integration grokBuildIntegration) loadSyncSnapshot( + ctx context.Context, + db *store.Store, + sourceID int64, +) ([]SourceFile, map[store.UsageKey]store.GrokBuildUsageImportFile, map[store.UsageKey]store.UsageImportObservation, error) { + files, err := ListGrokBuildSessionFilesContext(ctx, integration.grokHome) + if err != nil { + return nil, nil, nil, apperror.Wrap(apperror.UsageImportFailed, "failed to list Grok Build session files", err) + } + cursorRows, err := db.ListGrokBuildUsageImportFiles(ctx, sourceID) + if err != nil { + return nil, nil, nil, err + } + observationRows, err := db.ListUsageImportObservations(ctx, sourceID) + if err != nil { + return nil, nil, nil, err + } + return files, grokBuildCursorMap(cursorRows), observationMap(observationRows), nil +} + func (integration grokBuildIntegration) preflight( ctx context.Context, stores store.Factory, diff --git a/internal/usage/grok_build_sync_test.go b/internal/usage/grok_build_sync_test.go index 5186484..3c6a9cb 100644 --- a/internal/usage/grok_build_sync_test.go +++ b/internal/usage/grok_build_sync_test.go @@ -170,6 +170,80 @@ func TestBackgroundGrokBuildSyncNoopAndBoundedAppend(t *testing.T) { } } +func TestBackgroundGrokBuildSyncKeepsCursorCreatedAfterPreflight(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + grokHome := t.TempDir() + oldPath := writeGrokBuildUsageFixture( + t, + grokHome, + "workspace-old", + "session-old", + syntheticGrokBuildUsageLine( + "session-old", + "prompt-old", + "grok-build-latest", + 1_750_000_000, + TokenCounts{InputTokens: 10, OutputTokens: 2, TotalTokens: 12}, + ), + ) + environment := newGrokBuildUsageTestEnvironment(t, configDir, grokHome) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncGrokBuild(ctx); err != nil { + t.Fatalf("initial Grok Build sync: %v", err) + } + if err := os.Remove(oldPath); err != nil { + t.Fatalf("remove old Grok Build fixture: %v", err) + } + + var second BackgroundSyncOutcome + var secondErr error + newPath := filepath.Join(grokHome, "sessions", "workspace-new", "session-new", "updates.jsonl") + first, err := environment.service.SyncProviderBackground(ctx, grokconfig.ProviderID, func() { + writeAppUsageFile(t, newPath, syntheticGrokBuildUsageLine( + "session-new", + "prompt-new", + "grok-build-latest", + 1_750_000_001, + TokenCounts{InputTokens: 20, OutputTokens: 4, TotalTokens: 24}, + )) + other := newGrokBuildUsageTestEnvironment(t, configDir, grokHome) + second, secondErr = other.service.SyncProviderBackground(ctx, grokconfig.ProviderID, nil) + }) + if err != nil { + t.Fatalf("first background Grok Build sync: %v", err) + } + if secondErr != nil || !second.Performed || second.Result.ImportedEvents != 1 { + t.Fatalf("second background Grok Build sync = %#v, err = %v", second, secondErr) + } + if !first.Performed { + t.Fatalf("first background Grok Build sync was not performed: %#v", first) + } + + fileKey, err := SourceKey(newPath) + if err != nil { + t.Fatalf("derive new Grok Build file key: %v", err) + } + db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("open Store: %v", err) + } + defer db.Close() + source, err := db.GetUsageSource(ctx, grokconfig.ProviderID, SourceGrokBuildSessionJSONL) + if err != nil { + t.Fatalf("read Grok Build usage source: %v", err) + } + if _, err := db.GetGrokBuildUsageImportFile(ctx, source.ID, fileKey); err != nil { + t.Fatalf("new Grok Build cursor was removed by stale finalization: %v", err) + } + summary, err := environment.service.Summary(ctx, UsageSummaryRequest{ProviderID: grokconfig.ProviderID}) + if err != nil || summary.EventCount != 2 || summary.TotalTokens != 36 { + t.Fatalf("Grok Build summary after stale-snapshot race = %#v, err = %v", summary, err) + } +} + func TestUsageSyncGrokBuildBackfillsNewlyRecognizedUnknownModels(t *testing.T) { ctx := context.Background() environment := newGrokBuildUsageTestEnvironment(t, t.TempDir(), t.TempDir()) diff --git a/internal/usage/service_test.go b/internal/usage/service_test.go index dd9ec27..2f0638c 100644 --- a/internal/usage/service_test.go +++ b/internal/usage/service_test.go @@ -856,6 +856,166 @@ func TestUsageSyncCodexRejectsTruncatedAndRewrittenHistory(t *testing.T) { } } +func TestUsageSyncCodexRejectsTimestampAndModelOnlyPrefixRewrites(t *testing.T) { + for _, test := range []struct { + name string + rewrite func(string) string + }{ + { + name: "timestamp-only rewrite", + rewrite: func(original string) string { + oldLine := `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}` + newLine := `{"type":"event_msg","timestamp":"2026-07-01T00:00:00Z","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}` + return strings.Replace(original, oldLine, newLine, 1) + }, + }, + { + name: "stored-model-only rewrite", + rewrite: func(original string) string { + return strings.Replace(original, `"model":"gpt-5.3-codex"`, `"model":"gpt-5.3-codex-2026-07-01"`, 1) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + environment := newUsageTestEnvironment(t, configDir, codexDir) + _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx) + if err != nil { + t.Fatalf("initialize runtime: %v", err) + } + original := strings.Join([]string{ + `{"type":"session_meta","session_id":"session-prefix-rewrite"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":20,"output_tokens":4}}}}`, + }, "\n") + path := writeAppUsageFixture(t, codexDir, original) + if result, err := environment.service.SyncCodex(ctx); err != nil || result.ImportedEvents != 2 { + t.Fatalf("initial sync result=%#v err=%v", result, err) + } + fileKey, err := SourceKey(path) + if err != nil { + t.Fatalf("derive file key: %v", err) + } + db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("open database: %v", err) + } + source, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if err != nil { + _ = db.Close() + t.Fatalf("read usage source: %v", err) + } + before, err := db.GetCodexUsageImportFile(ctx, source.ID, fileKey) + if closeErr := db.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + t.Fatalf("read initial cursor: %v", err) + } + + rewritten := test.rewrite(original) + if rewritten == original { + t.Fatal("rewrite did not change the fixture") + } + writeAppUsageFile(t, path, rewritten) + future := time.Now().Add(2 * time.Second) + if err := os.Chtimes(path, future, future); err != nil { + t.Fatalf("advance rewritten fixture timestamp: %v", err) + } + result, err := environment.service.SyncCodex(ctx) + if err != nil || result.ImportedEvents != 0 || len(result.Errors) != 1 || result.Errors[0].Message != codexHistoryChangedMessage { + t.Fatalf("changed history sync result=%#v err=%v", result, err) + } + + db, err = environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("reopen database: %v", err) + } + after, err := db.GetCodexUsageImportFile(ctx, source.ID, fileKey) + if closeErr := db.Close(); err == nil && closeErr != nil { + err = closeErr + } + if err != nil { + t.Fatalf("read unchanged cursor: %v", err) + } + if after != before { + t.Fatalf("history rewrite advanced cursor: before=%#v after=%#v", before, after) + } + summary, err := environment.service.Summary(ctx, UsageSummaryRequest{ProviderID: ProviderCodex}) + if err != nil || summary.EventCount != 2 || summary.TotalTokens != 24 { + t.Fatalf("changed history altered facts: summary=%#v err=%v", summary, err) + } + }) + } +} + +func TestBackgroundCodexSyncKeepsCursorCreatedAfterPreflight(t *testing.T) { + ctx := context.Background() + configDir := t.TempDir() + codexDir := t.TempDir() + oldPath := writeAppUsageFixture(t, codexDir, strings.Join([]string{ + `{"type":"session_meta","session_id":"session-old"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":10,"output_tokens":2}}}}`, + }, "\n")) + environment := newUsageTestEnvironment(t, configDir, codexDir) + if _, err := bootstrap.NewService(environment.runtime, nil, nil).Initialize(ctx); err != nil { + t.Fatalf("initialize runtime: %v", err) + } + if _, err := environment.service.SyncCodex(ctx); err != nil { + t.Fatalf("initial Codex sync: %v", err) + } + if err := os.Remove(oldPath); err != nil { + t.Fatalf("remove old Codex fixture: %v", err) + } + + newPath := filepath.Join(codexDir, "sessions", "2026", "07", "07", "session-new.jsonl") + var second BackgroundSyncOutcome + var secondErr error + first, err := environment.service.SyncCodexBackground(ctx, func() { + writeAppUsageFile(t, newPath, strings.Join([]string{ + `{"type":"session_meta","session_id":"session-new"}`, + `{"type":"turn_context","model":"gpt-5.3-codex"}`, + `{"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":20,"output_tokens":4}}}}`, + }, "\n")) + other := newUsageTestEnvironment(t, configDir, codexDir) + second, secondErr = other.service.SyncCodexBackground(ctx, nil) + }) + if err != nil { + t.Fatalf("first background Codex sync: %v", err) + } + if secondErr != nil || !second.Performed || second.Result.ImportedEvents != 1 { + t.Fatalf("second background Codex sync = %#v, err = %v", second, secondErr) + } + if !first.Performed { + t.Fatalf("first background Codex sync was not performed: %#v", first) + } + + fileKey, err := SourceKey(newPath) + if err != nil { + t.Fatalf("derive new Codex file key: %v", err) + } + db, err := environment.runtime.StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("open Store: %v", err) + } + defer db.Close() + source, err := db.GetUsageSource(ctx, ProviderCodex, SourceCodexSessionJSONL) + if err != nil { + t.Fatalf("read Codex usage source: %v", err) + } + if _, err := db.GetCodexUsageImportFile(ctx, source.ID, fileKey); err != nil { + t.Fatalf("new Codex cursor was removed by stale finalization: %v", err) + } + summary, err := environment.service.Summary(ctx, UsageSummaryRequest{ProviderID: ProviderCodex}) + if err != nil || summary.EventCount != 2 || summary.TotalTokens != 36 { + t.Fatalf("Codex summary after stale-snapshot race = %#v, err = %v", summary, err) + } +} + func TestUsageSyncCodexRevalidatesCompatibleParserRevision(t *testing.T) { ctx := context.Background() configDir := t.TempDir() From b2cdad5885710ee6b6b7de17a89795cbd6d8da3a Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Sat, 12 Sep 2026 10:47:52 +0800 Subject: [PATCH 4/4] fix(desktop): widen usage report update intervals --- desktop/backend/services_test.go | 8 ++-- desktop/backend/usage_auto_sync_test.go | 2 +- .../features/settings/CodexSettings.svelte | 3 +- .../settings/GrokBuildSettings.svelte | 4 +- .../settings/UsageSyncSettings.svelte | 11 +++-- .../features/settings/usage-sync-settings.ts | 11 ++++- desktop/frontend/src/lib/i18n.ts | 2 + docs/codex/usage-cost.md | 2 +- docs/grok-build/usage-cost.md | 2 +- docs/zh/codex/usage-cost.md | 2 +- docs/zh/grok-build/usage-cost.md | 2 +- internal/codex/automation/policy_test.go | 5 ++- internal/grokbuild/service_settings_test.go | 45 +++++++++++++++++++ internal/usage/settings.go | 7 ++- internal/usage/settings_test.go | 36 +++++++++++++++ 15 files changed, 122 insertions(+), 20 deletions(-) create mode 100644 internal/usage/settings_test.go diff --git a/desktop/backend/services_test.go b/desktop/backend/services_test.go index 2fa7bcd..aeb6e95 100644 --- a/desktop/backend/services_test.go +++ b/desktop/backend/services_test.go @@ -443,9 +443,9 @@ func TestCodexSettingsServiceKeepsConcurrentUsageIntervalUpdatesConsistent(t *te } start := make(chan struct{}) - errorsByUpdate := make(chan error, 4) + errorsByUpdate := make(chan error, 5) var wg sync.WaitGroup - for _, interval := range []int{5, 15, 30, 60} { + for _, interval := range []int{15, 30, 60, 120, 300} { interval := interval wg.Add(1) go func() { @@ -493,9 +493,9 @@ func TestGrokBuildSettingsKeepProviderIntervalsIndependentUnderConcurrency(t *te codexBefore := services.codexUsageSync.Status() start := make(chan struct{}) - errorsByUpdate := make(chan error, 4) + errorsByUpdate := make(chan error, 5) var wg sync.WaitGroup - for _, interval := range []int{5, 15, 30, 60} { + for _, interval := range []int{15, 30, 60, 120, 300} { interval := interval wg.Add(1) go func() { diff --git a/desktop/backend/usage_auto_sync_test.go b/desktop/backend/usage_auto_sync_test.go index d0e08f5..0e02162 100644 --- a/desktop/backend/usage_auto_sync_test.go +++ b/desktop/backend/usage_auto_sync_test.go @@ -227,7 +227,7 @@ func TestUsageAutoSyncStatusIsProviderScoped(t *testing.T) { codex := newUsageAutoSyncRuntime( "codex", func(context.Context) (usage.ProviderSyncSettings, error) { - return usage.ProviderSyncSettings{UsageSyncIntervalSeconds: 5}, nil + return usage.ProviderSyncSettings{UsageSyncIntervalSeconds: 15}, nil }, performedUsageSync(func(context.Context) (usage.UsageSyncResult, error) { return usage.UsageSyncResult{ProviderID: "codex"}, nil diff --git a/desktop/frontend/src/features/settings/CodexSettings.svelte b/desktop/frontend/src/features/settings/CodexSettings.svelte index 1928822..bde1304 100644 --- a/desktop/frontend/src/features/settings/CodexSettings.svelte +++ b/desktop/frontend/src/features/settings/CodexSettings.svelte @@ -12,6 +12,7 @@ import ProfileAutomationSettings from "./ProfileAutomationSettings.svelte"; import UsageSyncSettings from "./UsageSyncSettings.svelte"; + import { usageSyncIntervalDefault } from "./usage-sync-settings"; import { useCodexRuntime } from "./codex-runtime.svelte.js"; const controller = useCodexRuntime(); @@ -36,7 +37,7 @@ void; } = $props(); + + const selected = $derived(usageIntervalLabel(interval)); @@ -30,15 +32,16 @@ {#snippet control()} {#if busy}{/if} - - {$_("usageSettings.seconds", { values: { count: interval } })} + + {$_(selected.key, { values: { count: selected.count } })} {#each usageIntervals as seconds (seconds)} + {@const option = usageIntervalLabel(seconds)} {/each} diff --git a/desktop/frontend/src/features/settings/usage-sync-settings.ts b/desktop/frontend/src/features/settings/usage-sync-settings.ts index 0cdb5ee..5cb7a78 100644 --- a/desktop/frontend/src/features/settings/usage-sync-settings.ts +++ b/desktop/frontend/src/features/settings/usage-sync-settings.ts @@ -1 +1,10 @@ -export const usageIntervals = [5, 15, 30, 60] as const; +export const usageIntervals = [15, 30, 60, 120, 300] as const; + +export const usageSyncIntervalDefault = 60; + +export function usageIntervalLabel(seconds: number): { key: "usageSettings.seconds" | "usageSettings.minutes"; count: number } { + if (seconds >= 60 && seconds % 60 === 0) { + return { key: "usageSettings.minutes", count: seconds / 60 }; + } + return { key: "usageSettings.seconds", count: seconds }; +} diff --git a/desktop/frontend/src/lib/i18n.ts b/desktop/frontend/src/lib/i18n.ts index b94ab6f..7bca62d 100644 --- a/desktop/frontend/src/lib/i18n.ts +++ b/desktop/frontend/src/lib/i18n.ts @@ -797,6 +797,7 @@ const messages = { label: "Update frequency", description: "Keep Usage reports up to date from this Agent's local activity.", seconds: "{count} seconds", + minutes: "{count, plural, one {# minute} other {# minutes}}", }, settings: { title: "Settings", @@ -1921,6 +1922,7 @@ const messages = { label: "更新频率", description: "根据此 Agent 的本地活动更新用量报告。", seconds: "{count} 秒", + minutes: "{count} 分钟", }, settings: { title: "设置", diff --git a/docs/codex/usage-cost.md b/docs/codex/usage-cost.md index c92f1b2..74735a7 100644 --- a/docs/codex/usage-cost.md +++ b/docs/codex/usage-cost.md @@ -6,7 +6,7 @@ ProfileDeck reads local Codex session data to show token usage, activity, and es The Desktop app syncs after startup and continues while ProfileDeck is open or in the menu bar. -To change the interval, open **Codex → Settings → Usage reports → Update frequency** and choose 5, 15, 30, or 60 seconds. The default is 15 seconds. The Usage page shows the latest sync result and reports files it could not read. +To change the interval, open **Codex → Settings → Usage reports → Update frequency** and choose 15 seconds, 30 seconds, 1 minute, 2 minutes, or 5 minutes. The default is 1 minute. The Usage page shows the latest sync result and reports files it could not read. When session files have not changed, background sync checks their metadata without reading their contents. Normal appends read only a small integrity boundary and the new part of each file. If an existing file is truncated or ProfileDeck detects a change to earlier usage, it keeps the previously imported history and skips that file version. It checks the file again after it changes or when you run a CLI sync. diff --git a/docs/grok-build/usage-cost.md b/docs/grok-build/usage-cost.md index 9a405d8..773e239 100644 --- a/docs/grok-build/usage-cost.md +++ b/docs/grok-build/usage-cost.md @@ -6,7 +6,7 @@ ProfileDeck reads local Grok Build session records to show token usage, activity The Desktop app syncs after startup and continues while ProfileDeck is open or in the menu bar. -To change the interval, open **Grok Build → Settings → Usage reports → Update frequency** and choose 5, 15, 30, or 60 seconds. The default is 15 seconds. Codex and Grok Build use separate intervals and sync status. +To change the interval, open **Grok Build → Settings → Usage reports → Update frequency** and choose 15 seconds, 30 seconds, 1 minute, 2 minutes, or 5 minutes. The default is 1 minute. Codex and Grok Build use separate intervals and sync status. When session files have not changed, background sync checks their metadata without reading their contents. Normal appends read only a small integrity boundary and the new part of each file. diff --git a/docs/zh/codex/usage-cost.md b/docs/zh/codex/usage-cost.md index 233a9bb..3360c8b 100644 --- a/docs/zh/codex/usage-cost.md +++ b/docs/zh/codex/usage-cost.md @@ -6,7 +6,7 @@ ProfileDeck 读取本地 Codex 会话数据，展示令牌用量、活动情况 桌面端会在启动后同步，并在 ProfileDeck 窗口打开或驻留菜单栏时继续同步。 -如需调整间隔，请打开 **Codex → 设置 → 用量报告 → 更新频率**，选择 5、15、30 或 60 秒；默认值为 15 秒。用量页面会显示最近一次同步结果，并报告无法读取的文件。 +如需调整间隔，请打开 **Codex → 设置 → 用量报告 → 更新频率**，选择 15 秒、30 秒、1 分钟、2 分钟或 5 分钟；默认值为 1 分钟。用量页面会显示最近一次同步结果，并报告无法读取的文件。 会话文件没有变化时，后台同步只检查文件信息，不会读取文件内容。文件正常追加后，只读取一小段完整性校验边界和新增部分。如果现有文件被截短，或 ProfileDeck 检测到之前的用量记录发生变化，它会保留已导入的历史并跳过该文件版本；文件再次变化或手动运行 CLI 同步后会重新检查。 diff --git a/docs/zh/grok-build/usage-cost.md b/docs/zh/grok-build/usage-cost.md index ca7f166..cef1f13 100644 --- a/docs/zh/grok-build/usage-cost.md +++ b/docs/zh/grok-build/usage-cost.md @@ -6,7 +6,7 @@ ProfileDeck 读取本地 Grok Build 会话记录，展示令牌用量、活动 桌面端会在启动后同步，并在 ProfileDeck 窗口打开或驻留菜单栏时继续同步。 -如需调整间隔，请打开 **Grok Build → 设置 → 用量报告 → 更新频率**，选择 5、15、30 或 60 秒；默认值为 15 秒。Codex 与 Grok Build 分别使用独立的间隔和同步状态。 +如需调整间隔，请打开 **Grok Build → 设置 → 用量报告 → 更新频率**，选择 15 秒、30 秒、1 分钟、2 分钟或 5 分钟；默认值为 1 分钟。Codex 与 Grok Build 分别使用独立的间隔和同步状态。 会话文件没有变化时，后台同步只检查文件信息，不会读取文件内容。文件正常追加后，只读取一小段完整性校验边界和新增部分。 diff --git a/internal/codex/automation/policy_test.go b/internal/codex/automation/policy_test.go index 07307ae..3dcdbfe 100644 --- a/internal/codex/automation/policy_test.go +++ b/internal/codex/automation/policy_test.go @@ -3,7 +3,10 @@ package automation import "testing" func TestProfileSettingsPolicy(t *testing.T) { - if _, err := NormalizeUsageSyncInterval(15); err != nil { + if got, err := NormalizeUsageSyncInterval(5); err != nil || got != 15 { + t.Fatalf("retired usage interval = %d, err = %v", got, err) + } + if _, err := NormalizeUsageSyncInterval(300); err != nil { t.Fatal(err) } if _, err := NormalizeQuotaRefreshInterval(600); err != nil { diff --git a/internal/grokbuild/service_settings_test.go b/internal/grokbuild/service_settings_test.go index 53c9e97..eb16323 100644 --- a/internal/grokbuild/service_settings_test.go +++ b/internal/grokbuild/service_settings_test.go @@ -82,3 +82,48 @@ func TestGrokBuildUsageSyncSettingsShareStrictProviderPolicy(t *testing.T) { } } } + +func TestGrokBuildUsageSyncSettingsMapRetiredFiveSecondInterval(t *testing.T) { + ctx := context.Background() + application := newApplication(t, t.TempDir()) + if _, err := application.Usage().SyncGrokBuild(ctx); err != nil { + t.Fatalf("explicit usage sync did not provision Provider: %v", err) + } + db, err := application.Runtime().StoreFactory().OpenHealthy(ctx, false) + if err != nil { + t.Fatalf("open Store: %v", err) + } + if _, err := db.UpsertProviderSetting(ctx, store.UpsertProviderSettingParams{ + ProviderID: grokconfig.ProviderID, SchemaVersion: store.ProviderSettingsSchemaVersion, + SettingsJSON: `{"usage_sync_interval_seconds":5}`, + }); err != nil { + _ = db.Close() + t.Fatalf("write retired interval: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close Store: %v", err) + } + + loaded, err := application.GrokBuild().GetSettings(ctx) + if err != nil || loaded.UsageSyncIntervalSeconds != 15 { + t.Fatalf("loaded retired interval = %#v, err = %v", loaded, err) + } + + retired := 5 + saved, err := application.GrokBuild().UpdateSettings(ctx, grokbuild.UpdateSettingsRequest{ + UsageSyncIntervalSeconds: &retired, + }) + if err != nil || saved.UsageSyncIntervalSeconds != 15 { + t.Fatalf("saved retired interval = %#v, err = %v", saved, err) + } + + db, err = application.Runtime().StoreFactory().OpenHealthy(ctx, true) + if err != nil { + t.Fatalf("reopen Store: %v", err) + } + defer db.Close() + setting, err := db.GetProviderSetting(ctx, grokconfig.ProviderID) + if err != nil || setting.SettingsJSON != `{"usage_sync_interval_seconds":15}` { + t.Fatalf("persisted retired interval = %#v, err = %v", setting, err) + } +} diff --git a/internal/usage/settings.go b/internal/usage/settings.go index 9321202..60b91ac 100644 --- a/internal/usage/settings.go +++ b/internal/usage/settings.go @@ -13,7 +13,7 @@ import ( "github.com/strahe/profiledeck/internal/store" ) -const UsageSyncIntervalDefault = 15 +const UsageSyncIntervalDefault = 60 type ProviderSyncSettings struct { UsageSyncIntervalSeconds int `json:"usage_sync_interval_seconds"` @@ -21,7 +21,10 @@ type ProviderSyncSettings struct { func NormalizeUsageSyncInterval(value int) (int, *apperror.Error) { switch value { - case 5, 15, 30, 60: + case 5: + // Retired cadence: keep existing Provider settings loadable. + return 15, nil + case 15, 30, 60, 120, 300: return value, nil default: return 0, apperror.New( diff --git a/internal/usage/settings_test.go b/internal/usage/settings_test.go new file mode 100644 index 0000000..4bf59eb --- /dev/null +++ b/internal/usage/settings_test.go @@ -0,0 +1,36 @@ +package usage + +import ( + "testing" + + "github.com/strahe/profiledeck/internal/apperror" +) + +func TestNormalizeUsageSyncInterval(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + input int + want int + }{ + {name: "retired five seconds", input: 5, want: 15}, + {name: "fifteen seconds", input: 15, want: 15}, + {name: "thirty seconds", input: 30, want: 30}, + {name: "one minute", input: 60, want: 60}, + {name: "two minutes", input: 120, want: 120}, + {name: "five minutes", input: 300, want: 300}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := NormalizeUsageSyncInterval(test.input) + if err != nil || got != test.want { + t.Fatalf("NormalizeUsageSyncInterval(%d) = %d, %v, want %d", test.input, got, err, test.want) + } + }) + } + + if _, err := NormalizeUsageSyncInterval(10); err == nil { + t.Fatal("unsupported interval was accepted") + } else if err.Code != apperror.SettingInvalid { + t.Fatalf("unsupported interval error = %v", err) + } +}