Skip to content

Commit 46efc1e

Browse files
ndycodendycode
committed
feat(auth): add retention and recovery doctor
1 parent d262177 commit 46efc1e

6 files changed

Lines changed: 815 additions & 106 deletions

File tree

‎lib/codex-manager.ts‎

Lines changed: 134 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ import {
6666
getActionableNamedBackupRestores,
6767
getRedactedFilesystemErrorLabel,
6868
getNamedBackupsDirectoryPath,
69+
listAccountSnapshots,
6970
listNamedBackups,
7071
listRotatingBackups,
7172
NAMED_BACKUP_LIST_CONCURRENCY,
@@ -89,6 +90,7 @@ import {
8990
getCodexCliConfigPath,
9091
loadCodexCliState,
9192
} from "./codex-cli/state.js";
93+
import { getLatestCodexCliSyncRollbackPlan } from "./codex-cli/sync.js";
9294
import { setCodexCliActiveSelection } from "./codex-cli/writer.js";
9395
import { ANSI } from "./ui/ansi.js";
9496
import { confirm } from "./ui/confirm.js";
@@ -3348,21 +3350,24 @@ async function runDoctor(args: string[]): Promise<number> {
33483350

33493351
setStoragePath(null);
33503352
const storagePath = getStoragePath();
3353+
const walPath = `${storagePath}.wal`;
3354+
const storageExists = existsSync(storagePath);
3355+
const walExists = existsSync(walPath);
33513356
const checks: DoctorCheck[] = [];
33523357
const addCheck = (check: DoctorCheck): void => {
33533358
checks.push(check);
33543359
};
33553360

33563361
addCheck({
33573362
key: "storage-file",
3358-
severity: existsSync(storagePath) ? "ok" : "warn",
3359-
message: existsSync(storagePath)
3363+
severity: storageExists ? "ok" : "warn",
3364+
message: storageExists
33603365
? "Account storage file found"
33613366
: "Account storage file does not exist yet (first login pending)",
33623367
details: storagePath,
33633368
});
33643369

3365-
if (existsSync(storagePath)) {
3370+
if (storageExists) {
33663371
try {
33673372
const stat = await fs.stat(storagePath);
33683373
addCheck({
@@ -3381,6 +3386,82 @@ async function runDoctor(args: string[]): Promise<number> {
33813386
}
33823387
}
33833388

3389+
addCheck({
3390+
key: "storage-journal",
3391+
severity: walExists ? "ok" : "warn",
3392+
message: walExists
3393+
? "Write-ahead journal found"
3394+
: "Write-ahead journal missing; recovery will rely on backups",
3395+
details: walPath,
3396+
});
3397+
3398+
const rotatingBackups = await listRotatingBackups();
3399+
const validRotatingBackups = rotatingBackups.filter((backup) => backup.valid);
3400+
const invalidRotatingBackups = rotatingBackups.filter(
3401+
(backup) => !backup.valid,
3402+
);
3403+
addCheck({
3404+
key: "rotating-backups",
3405+
severity:
3406+
validRotatingBackups.length > 0
3407+
? "ok"
3408+
: rotatingBackups.length > 0
3409+
? "error"
3410+
: "warn",
3411+
message:
3412+
validRotatingBackups.length > 0
3413+
? `${validRotatingBackups.length} rotating backup(s) available`
3414+
: rotatingBackups.length > 0
3415+
? "Rotating backups are unreadable"
3416+
: "No rotating backups found yet",
3417+
details:
3418+
invalidRotatingBackups.length > 0
3419+
? `${invalidRotatingBackups.length} invalid backup(s); recreate by saving accounts`
3420+
: dirname(storagePath),
3421+
});
3422+
3423+
const snapshotBackups = await listAccountSnapshots();
3424+
const validSnapshots = snapshotBackups.filter((snapshot) => snapshot.valid);
3425+
const invalidSnapshots = snapshotBackups.filter((snapshot) => !snapshot.valid);
3426+
addCheck({
3427+
key: "snapshot-backups",
3428+
severity:
3429+
validSnapshots.length > 0
3430+
? "ok"
3431+
: snapshotBackups.length > 0
3432+
? "error"
3433+
: "warn",
3434+
message:
3435+
validSnapshots.length > 0
3436+
? `${validSnapshots.length} recovery snapshot(s) available`
3437+
: snapshotBackups.length > 0
3438+
? "Snapshot backups are unreadable"
3439+
: "No recovery snapshots found",
3440+
details:
3441+
invalidSnapshots.length > 0
3442+
? `${invalidSnapshots.length} invalid snapshot(s); create a fresh snapshot before destructive actions`
3443+
: getNamedBackupsDirectoryPath(),
3444+
});
3445+
3446+
addCheck({
3447+
key: "recovery-chain",
3448+
severity:
3449+
storageExists ||
3450+
walExists ||
3451+
validRotatingBackups.length > 0 ||
3452+
validSnapshots.length > 0
3453+
? "ok"
3454+
: "warn",
3455+
message:
3456+
storageExists ||
3457+
walExists ||
3458+
validRotatingBackups.length > 0 ||
3459+
validSnapshots.length > 0
3460+
? "Recovery artifacts present"
3461+
: "No recovery artifacts found; create a snapshot or backup before destructive actions",
3462+
details: `storage=${storageExists}, wal=${walExists}, rotating=${validRotatingBackups.length}, snapshots=${validSnapshots.length}`,
3463+
});
3464+
33843465
const codexAuthPath = getCodexCliAuthPath();
33853466
const codexConfigPath = getCodexCliConfigPath();
33863467
let codexAuthEmail: string | undefined;
@@ -3485,6 +3566,56 @@ async function runDoctor(args: string[]): Promise<number> {
34853566
});
34863567

34873568
const storage = await loadAccounts();
3569+
const rollbackPlan = await getLatestCodexCliSyncRollbackPlan();
3570+
if (rollbackPlan.status === "ready") {
3571+
const accountCount =
3572+
rollbackPlan.accountCount ?? rollbackPlan.storage?.accounts.length;
3573+
addCheck({
3574+
key: "codex-cli-rollback-checkpoint",
3575+
severity: "ok",
3576+
message: `Latest manual Codex CLI rollback checkpoint ready (${accountCount ?? "?"} account${accountCount === 1 ? "" : "s"})`,
3577+
details: rollbackPlan.snapshot?.path,
3578+
});
3579+
} else {
3580+
const isBlocked = Boolean(rollbackPlan.snapshot);
3581+
addCheck({
3582+
key: "codex-cli-rollback-checkpoint",
3583+
severity: isBlocked ? "error" : "warn",
3584+
message: isBlocked
3585+
? "Latest manual Codex CLI rollback checkpoint cannot be restored"
3586+
: "No manual Codex CLI rollback checkpoint has been recorded yet",
3587+
details: [
3588+
rollbackPlan.snapshot?.path ?? rollbackPlan.snapshot?.name,
3589+
rollbackPlan.reason,
3590+
isBlocked
3591+
? "Action: Recreate the rollback checkpoint with a fresh manual Codex CLI sync before attempting rollback."
3592+
: "Action: Run a manual Codex CLI sync with backups enabled to capture a rollback checkpoint before applying changes.",
3593+
]
3594+
.filter(Boolean)
3595+
.join(" | "),
3596+
});
3597+
}
3598+
3599+
const actionableNamedBackupRestores = await getActionableNamedBackupRestores({
3600+
currentStorage: storage,
3601+
});
3602+
const actionableBackupCount = actionableNamedBackupRestores.assessments.length;
3603+
addCheck({
3604+
key: "named-backup-restores",
3605+
severity: actionableBackupCount > 0 ? "ok" : "warn",
3606+
message:
3607+
actionableBackupCount > 0
3608+
? `Found ${actionableBackupCount} actionable named backup restore${actionableBackupCount === 1 ? "" : "s"}`
3609+
: "No actionable named backup restores available",
3610+
details: [
3611+
`total backups: ${actionableNamedBackupRestores.totalBackups}`,
3612+
actionableBackupCount > 0
3613+
? undefined
3614+
: `Action: Add or copy a named backup into ${getNamedBackupsDirectoryPath()} before attempting recovery.`,
3615+
]
3616+
.filter(Boolean)
3617+
.join(" | "),
3618+
});
34883619
let fixChanged = false;
34893620
let fixActions: DoctorFixAction[] = [];
34903621
if (options.fix && storage && storage.accounts.length > 0) {

‎lib/storage.ts‎

Lines changed: 165 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ const ACCOUNTS_BACKUP_HISTORY_DEPTH = 3;
4848
const BACKUP_COPY_MAX_ATTEMPTS = 5;
4949
const BACKUP_COPY_BASE_DELAY_MS = 10;
5050
export const NAMED_BACKUP_LIST_CONCURRENCY = 8;
51+
export const ACCOUNT_SNAPSHOT_RETENTION_PER_REASON = 3;
52+
const AUTO_SNAPSHOT_NAME_PATTERN =
53+
/^accounts-(?<reason>[a-z0-9-]+)-snapshot-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}_\d{3}$/i;
5154
const RESET_MARKER_SUFFIX = ".reset-intent";
5255
let storageBackupEnabled = true;
5356
let lastAccountsSaveTimestamp = 0;
@@ -1989,6 +1992,11 @@ export async function listNamedBackups(): Promise<NamedBackupMetadata[]> {
19891992
return scanResult.backups.map((entry) => entry.backup);
19901993
}
19911994

1995+
export async function listAccountSnapshots(): Promise<NamedBackupMetadata[]> {
1996+
const backups = await listNamedBackups();
1997+
return backups.filter((backup) => isAccountSnapshotName(backup.name));
1998+
}
1999+
19922000
export async function listRotatingBackups(): Promise<RotatingBackupMetadata[]> {
19932001
let storagePath: string | null = null;
19942002
try {
@@ -2192,6 +2200,149 @@ function buildAccountSnapshotName(
21922200
return `accounts-${reason}-snapshot-${formatTimestampForSnapshot(timestamp)}`;
21932201
}
21942202

2203+
export function isAccountSnapshotName(name: string): boolean {
2204+
return AUTO_SNAPSHOT_NAME_PATTERN.test(name);
2205+
}
2206+
2207+
function getAccountSnapshotReason(name: string): string | null {
2208+
const match = name.match(AUTO_SNAPSHOT_NAME_PATTERN);
2209+
return match?.groups?.reason?.toLowerCase() ?? null;
2210+
}
2211+
2212+
type AutoSnapshotDetails = {
2213+
backup: NamedBackupMetadata;
2214+
name: string;
2215+
reason: string;
2216+
sortTimestamp: number;
2217+
};
2218+
2219+
function parseAutoSnapshot(
2220+
backup: NamedBackupMetadata,
2221+
): AutoSnapshotDetails | null {
2222+
const reason = getAccountSnapshotReason(backup.name);
2223+
if (!reason) {
2224+
return null;
2225+
}
2226+
return {
2227+
backup,
2228+
name: backup.name,
2229+
reason,
2230+
sortTimestamp: backup.updatedAt ?? backup.createdAt ?? 0,
2231+
};
2232+
}
2233+
2234+
async function getLatestManualCodexCliRollbackSnapshotNames(): Promise<
2235+
Set<string>
2236+
> {
2237+
try {
2238+
const syncHistoryModule = await import("./sync-history.js");
2239+
if (typeof syncHistoryModule.readSyncHistory !== "function") {
2240+
return new Set();
2241+
}
2242+
const history = await syncHistoryModule.readSyncHistory({
2243+
kind: "codex-cli-sync",
2244+
});
2245+
for (let index = history.length - 1; index >= 0; index -= 1) {
2246+
const entry = history[index];
2247+
if (!entry || entry.kind !== "codex-cli-sync") {
2248+
continue;
2249+
}
2250+
const run = entry.run;
2251+
if (run?.trigger !== "manual" || run.outcome !== "changed") {
2252+
continue;
2253+
}
2254+
const snapshotName = run.rollbackSnapshot?.name?.trim();
2255+
if (!snapshotName) {
2256+
break;
2257+
}
2258+
return new Set([snapshotName]);
2259+
}
2260+
} catch (error) {
2261+
log.debug("Failed to load rollback snapshot names for retention", {
2262+
error: String(error),
2263+
});
2264+
}
2265+
return new Set();
2266+
}
2267+
2268+
export interface AutoSnapshotPruneOptions {
2269+
backups?: NamedBackupMetadata[];
2270+
preserveNames?: Iterable<string>;
2271+
keepLatestPerReason?: number;
2272+
}
2273+
2274+
export interface AutoSnapshotPruneResult {
2275+
pruned: NamedBackupMetadata[];
2276+
kept: NamedBackupMetadata[];
2277+
}
2278+
2279+
export async function pruneAutoGeneratedSnapshots(
2280+
options: AutoSnapshotPruneOptions = {},
2281+
): Promise<AutoSnapshotPruneResult> {
2282+
const backups = options.backups ?? (await listNamedBackups());
2283+
const keepLatestPerReason = Math.max(
2284+
1,
2285+
options.keepLatestPerReason ?? 1,
2286+
);
2287+
const preserveNames = new Set(options.preserveNames ?? []);
2288+
for (const rollbackName of await getLatestManualCodexCliRollbackSnapshotNames()) {
2289+
preserveNames.add(rollbackName);
2290+
}
2291+
2292+
const autoSnapshots = backups
2293+
.map((backup) => parseAutoSnapshot(backup))
2294+
.filter((snapshot): snapshot is AutoSnapshotDetails => snapshot !== null);
2295+
if (autoSnapshots.length === 0) {
2296+
return { pruned: [], kept: [] };
2297+
}
2298+
2299+
const keepSet = new Set<string>(preserveNames);
2300+
const snapshotsByReason = new Map<string, AutoSnapshotDetails[]>();
2301+
for (const snapshot of autoSnapshots) {
2302+
const bucket = snapshotsByReason.get(snapshot.reason) ?? [];
2303+
bucket.push(snapshot);
2304+
snapshotsByReason.set(snapshot.reason, bucket);
2305+
}
2306+
2307+
for (const snapshots of snapshotsByReason.values()) {
2308+
snapshots.sort((left, right) => right.sortTimestamp - left.sortTimestamp);
2309+
for (const snapshot of snapshots.slice(0, keepLatestPerReason)) {
2310+
keepSet.add(snapshot.name);
2311+
}
2312+
}
2313+
2314+
const keptNames = new Set<string>(keepSet);
2315+
const pruned: NamedBackupMetadata[] = [];
2316+
for (const snapshot of autoSnapshots) {
2317+
if (keepSet.has(snapshot.name)) {
2318+
continue;
2319+
}
2320+
try {
2321+
await unlinkWithRetry(snapshot.backup.path);
2322+
pruned.push(snapshot.backup);
2323+
} catch (error) {
2324+
keptNames.add(snapshot.name);
2325+
log.warn("Failed to prune auto-generated snapshot", {
2326+
name: snapshot.name,
2327+
path: snapshot.backup.path,
2328+
error: String(error),
2329+
});
2330+
}
2331+
}
2332+
2333+
const kept = autoSnapshots
2334+
.filter((snapshot) => keptNames.has(snapshot.name))
2335+
.map((snapshot) => snapshot.backup);
2336+
2337+
return { pruned, kept };
2338+
}
2339+
2340+
async function enforceSnapshotRetention(): Promise<void> {
2341+
await pruneAutoGeneratedSnapshots({
2342+
keepLatestPerReason: ACCOUNT_SNAPSHOT_RETENTION_PER_REASON,
2343+
});
2344+
}
2345+
21952346
function extractPathTail(pathValue: string): string {
21962347
const segments = pathValue.split(/[\\/]+/).filter(Boolean);
21972348
return segments.at(-1) ?? pathValue;
@@ -2234,8 +2385,9 @@ export async function snapshotAccountStorage(
22342385
}
22352386

22362387
const backupName = buildAccountSnapshotName(reason, now);
2388+
let snapshot: NamedBackupMetadata;
22372389
try {
2238-
return await createBackup(backupName, { force });
2390+
snapshot = await createBackup(backupName, { force });
22392391
} catch (error) {
22402392
if (failurePolicy === "error") {
22412393
throw error;
@@ -2247,6 +2399,18 @@ export async function snapshotAccountStorage(
22472399
});
22482400
return null;
22492401
}
2402+
2403+
try {
2404+
await enforceSnapshotRetention();
2405+
} catch (error) {
2406+
log.warn("Failed to enforce account snapshot retention", {
2407+
reason,
2408+
backupName,
2409+
error: formatSnapshotErrorForLog(error),
2410+
});
2411+
}
2412+
2413+
return snapshot;
22502414
}
22512415

22522416
export async function assessNamedBackupRestore(

0 commit comments

Comments
 (0)