Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions apps/server/src/git/GitWorkflowService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ import { GitManager, type GitRunStackedActionOptions } from "./GitManager.ts";
import {
GitVcsDriver,
type GitRemoteStatusOptions,
type GitWorktreeBaseRef,
type GitWorktreeEntry,
} from "../vcs/GitVcsDriver.ts";
import { VcsDriverRegistry, type VcsDriverHandle } from "../vcs/VcsDriverRegistry.ts";
Expand Down Expand Up @@ -144,6 +145,11 @@ export interface GitWorkflowServiceShape {
readonly createWorktree: (
input: VcsCreateWorktreeInput,
) => Effect.Effect<VcsCreateWorktreeResult, GitCommandError>;
/** See GitVcsDriverShape.resolveFreshWorktreeBase. */
readonly resolveFreshWorktreeBase: (input: {
readonly cwd: string;
readonly branch: string;
}) => Effect.Effect<GitWorktreeBaseRef, GitCommandError>;
readonly removeWorktree: (input: VcsRemoveWorktreeInput) => Effect.Effect<void, GitCommandError>;
readonly createRef: (
input: VcsCreateRefInput,
Expand Down Expand Up @@ -520,6 +526,10 @@ export const make = Effect.fn("makeGitWorkflowService")(function* () {
ensureGitCommand("GitWorkflowService.createWorktree", input.cwd).pipe(
Effect.andThen(git.createWorktree(input)),
),
resolveFreshWorktreeBase: (input) =>
ensureGitCommand("GitWorkflowService.resolveFreshWorktreeBase", input.cwd).pipe(
Effect.andThen(git.resolveFreshWorktreeBase(input)),
),
removeWorktree: (input) =>
ensureGitCommand("GitWorkflowService.removeWorktree", input.cwd).pipe(
Effect.andThen(git.removeWorktree(input)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ import {
ProviderCommandReactorLive,
resolveForkTurnBoundary,
} from "./ProviderCommandReactor.ts";
import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";
import {
OrchestrationEngineService,
type OrchestrationEngineShape,
} from "../Services/OrchestrationEngine.ts";
import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts";
import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts";
import * as NodeServices from "@effect/platform-node/NodeServices";
Expand Down Expand Up @@ -169,6 +172,9 @@ describe("ProviderCommandReactor", () => {
/** Mirror the provider lifecycle's projected `starting` state while a
* replacement session is being bound. */
readonly projectStartingDuringRestart?: boolean;
/** State left behind by a previous server process, seeded before the
* reactor starts. */
readonly beforeStart?: (engine: OrchestrationEngineShape) => Effect.Effect<void, unknown>;
}) {
const now = "2026-01-01T00:00:00.000Z";
const baseDir =
Expand Down Expand Up @@ -542,6 +548,9 @@ describe("ProviderCommandReactor", () => {
const snapshotQuery = await runtime.runPromise(Effect.service(ProjectionSnapshotQuery));
const reactor = await runtime.runPromise(Effect.service(ProviderCommandReactor));
scope = await Effect.runPromise(Scope.make("sequential"));
if (input?.beforeStart) {
await Effect.runPromise(input.beforeStart(engine).pipe(Effect.orDie));
}
await Effect.runPromise(reactor.start().pipe(Scope.provide(scope)));
const drain = () => Effect.runPromise(reactor.drain);

Expand Down Expand Up @@ -1139,6 +1148,79 @@ describe("ProviderCommandReactor", () => {
});
});

// A server restart mid-turn (crash, update, dev reload) leaves the session
// row saying "running" with nothing behind it; without this the thread shows
// "Preparing turn" forever and the only way out is deleting it.
it("settles sessions left in flight by the previous server process on start", async () => {
const now = "2026-01-01T00:00:00.000Z";
const staleThreadId = ThreadId.make("thread-stale-restart");
const staleTurnId = asTurnId("turn-stale-restart");
const harness = await createHarness({
beforeStart: (engine) =>
Effect.gen(function* () {
yield* engine.dispatch({
type: "project.create",
commandId: CommandId.make("cmd-project-create-stale"),
projectId: asProjectId("project-stale"),
title: "Stale Project",
workspaceRoot: PROJECT_ROOT,
defaultModelSelection: null,
createdAt: now,
});
yield* engine.dispatch({
type: "thread.create",
commandId: CommandId.make("cmd-thread-create-stale"),
threadId: staleThreadId,
projectId: asProjectId("project-stale"),
title: "Stale Thread",
modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex" },
interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
runtimeMode: "approval-required",
branch: null,
worktreePath: null,
createdAt: now,
});
yield* engine.dispatch({
type: "thread.session.set",
commandId: CommandId.make("cmd-session-set-stale"),
threadId: staleThreadId,
session: {
threadId: staleThreadId,
status: "running",
providerName: "codex",
providerInstanceId: ProviderInstanceId.make("codex"),
runtimeMode: "approval-required",
activeTurnId: staleTurnId,
pendingBackgroundTaskCount: 0,
lastError: null,
updatedAt: now,
},
createdAt: now,
});
}),
});

await waitFor(async () => {
const readModel = await harness.readModel();
const thread = readModel.threads.find((entry) => entry.id === staleThreadId);
return (
thread?.session?.status === "interrupted" &&
thread.activities.some((entry) => entry.kind === "provider.session.restart-interrupted")
);
});

const readModel = await harness.readModel();
const thread = readModel.threads.find((entry) => entry.id === staleThreadId);
expect(thread?.session).toMatchObject({ status: "interrupted", activeTurnId: null });
expect(thread?.latestTurn).toMatchObject({ turnId: staleTurnId, state: "interrupted" });
expect(
thread?.activities.find((entry) => entry.kind === "provider.session.restart-interrupted"),
).toMatchObject({ turnId: staleTurnId });
// The thread the harness created after start is untouched.
const freshThread = readModel.threads.find((entry) => entry.id === ThreadId.make("thread-1"));
expect(freshThread?.session ?? null).toBeNull();
});

it("settles a stale running session when the provider reports no active turn to steer", async () => {
const harness = await createHarness();
const turnId = asTurnId("turn-stale");
Expand Down
55 changes: 55 additions & 0 deletions apps/server/src/orchestration/Layers/ProviderCommandReactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ const make = Effect.gen(function* () {
| "provider.approval.respond.failed"
| "provider.user-input.respond.failed"
| "provider.session.stop.failed"
| "provider.session.restart-interrupted"
| "provider.goal.failed";
readonly summary: string;
readonly detail: string;
Expand Down Expand Up @@ -2470,7 +2471,61 @@ const make = Effect.gen(function* () {
}
});

// Provider processes do not outlive the server, so a session still recorded
// as starting or running when the server comes up belongs to the previous
// process and no command can reach it. Settle each one as interrupted, with
// a note in the thread, instead of leaving the thread on "Preparing turn"
// until the user gives up and deletes it.
const restartInterruptedStatuses: ReadonlySet<OrchestrationSession["status"]> = new Set([
"starting",
"running",
]);
const settleSessionsFromPreviousProcess = Effect.fn("settleSessionsFromPreviousProcess")(
function* () {
const snapshot = yield* projectionSnapshotQuery.getShellSnapshot();
for (const thread of snapshot.threads) {
const session = thread.session;
if (!session || !restartInterruptedStatuses.has(session.status)) {
continue;
}
const createdAt = yield* nowIso;
yield* setThreadSession({
threadId: thread.id,
session: {
...session,
status: "interrupted",
activeTurnId: null,
pendingBackgroundTaskCount: 0,
lastError: null,
updatedAt: createdAt,
},
createdAt,
});
yield* appendProviderFailureActivity({
threadId: thread.id,
kind: "provider.session.restart-interrupted",
summary: "Turn interrupted by a server restart",
detail:
"Threadlines restarted while this turn was in progress, so its agent session is gone. Send a message to continue.",
turnId: session.activeTurnId,
createdAt,
});
}
},
);

const start: ProviderCommandReactorShape["start"] = Effect.fn("start")(function* () {
yield* settleSessionsFromPreviousProcess().pipe(
Effect.catchCause((cause) =>
Effect.logWarning(
"provider command reactor failed to settle sessions left by the previous process",
{
cause: Cause.pretty(cause),
},
),
),
);

const processEvent = Effect.fn("processEvent")(function* (event: OrchestrationEvent) {
if (
event.type === "thread.runtime-mode-set" ||
Expand Down
10 changes: 10 additions & 0 deletions apps/server/src/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3189,6 +3189,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
Effect.succeed({
worktree: { path: "/tmp/wt", refName: "feature/demo" },
}),
resolveFreshWorktreeBase: (input) =>
Effect.succeed({ refName: input.branch, isRemote: false }),
workingTreeDiff: () => Effect.succeed({ diff: "" }),
discardChanges: (input) => Effect.succeed({ discardedPaths: [...input.filePaths] }),
stageChanges: (input) => Effect.succeed({ stagedPaths: [...input.filePaths] }),
Expand Down Expand Up @@ -4462,6 +4464,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
layers: {
gitVcsDriver: {
createWorktree,
resolveFreshWorktreeBase: (input) =>
Effect.succeed({ refName: input.branch, isRemote: false }),
},
vcsStatusBroadcaster: {
refreshStatus,
Expand Down Expand Up @@ -4582,6 +4586,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
layers: {
gitVcsDriver: {
createWorktree,
resolveFreshWorktreeBase: (input) =>
Effect.succeed({ refName: input.branch, isRemote: false }),
},
orchestrationEngine: {
dispatch: (command) =>
Expand Down Expand Up @@ -4683,6 +4689,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
layers: {
gitVcsDriver: {
createWorktree,
resolveFreshWorktreeBase: (input) =>
Effect.succeed({ refName: input.branch, isRemote: false }),
},
orchestrationEngine: {
dispatch: (command) => {
Expand Down Expand Up @@ -4786,6 +4794,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
layers: {
gitVcsDriver: {
createWorktree,
resolveFreshWorktreeBase: (input) =>
Effect.succeed({ refName: input.branch, isRemote: false }),
},
orchestrationEngine: {
dispatch: (command) =>
Expand Down
13 changes: 13 additions & 0 deletions apps/server/src/terminal/Layers/NodePTY.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,19 @@ export const layer = Layer.effect(

const nodePty = yield* Effect.promise(() => import("node-pty"));

// node-pty's Windows kill path forks a helper to list the shell's console
// processes and takes the first IPC message back as the answer. Under
// `node --watch` the server carries WATCH_REPORT_DEPENDENCIES, which the
// forked helper inherits; its module loader then reports its own requires
// over that same channel, node-pty reads `consoleProcessList` off the
// wrong message, and the resulting unhandled rejection takes the server
// down. The fork is deferred until the pty reports ready, so the variable
// has to stay unset rather than be hidden around the kill call. The server
// loads its module graph eagerly, so watch mode loses nothing here.
if (globalThis.process.platform === "win32") {
delete globalThis.process.env.WATCH_REPORT_DEPENDENCIES;
}

const ensureNodePtySpawnHelperExecutableCached = yield* Effect.cached(
ensureNodePtySpawnHelperExecutable().pipe(
Effect.provideService(FileSystem.FileSystem, fs),
Expand Down
17 changes: 17 additions & 0 deletions apps/server/src/vcs/GitVcsDriver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,12 @@ export interface GitRemoteStatusDetails {
aheadOfDefaultCount: number;
}

/** Start point for a new worktree: a local branch or a remote-tracking ref. */
export interface GitWorktreeBaseRef {
readonly refName: string;
readonly isRemote: boolean;
}

export interface GitRemoteStatusOptions {
readonly forceRefresh?: boolean;
}
Expand Down Expand Up @@ -304,6 +310,17 @@ export interface GitVcsDriverShape {
readonly createWorktree: (
input: VcsCreateWorktreeInput,
) => Effect.Effect<VcsCreateWorktreeResult, GitCommandError>;
/**
* Where a worktree cut "from `branch`" should start. Fetches the branch's
* upstream (best effort, bounded) and answers with the upstream ref when it
* is strictly ahead of the local branch, so a thread started from `main`
* begins at the latest `main` rather than at a stale local copy. The local
* branch wins whenever it has commits of its own or has no upstream.
*/
readonly resolveFreshWorktreeBase: (input: {
readonly cwd: string;
readonly branch: string;
}) => Effect.Effect<GitWorktreeBaseRef, GitCommandError>;
readonly fetchPullRequestBranch: (
input: GitFetchPullRequestBranchInput,
) => Effect.Effect<void, GitCommandError>;
Expand Down
Loading
Loading