Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/development/onboarding-kickoff-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ chats and the adjacent campaign quickstart handoff.
guardrail, and wakes the agent from visible task text. The campaign skill is
not server-materialized: the kickoff message instructs the trial agent to
clone the campaign's skill repo and run the named skill on the connected repo.
- Campaign quickstart may carry an anonymous `{ attemptId, variant }`
attribution pair. The pair is stored only on the trial chat's JSON metadata
and included in the internal campaign export; it does not change trial
idempotency, quota, or runtime behavior and requires no schema migration.
- A `/me/onboarding/kickoff` request carrying `campaign` is a stale quickstart
request. It must not create an onboarding kickoff chat or campaign idempotency
key; it returns `410 campaign_kickoff_moved` when landing campaigns are enabled
Expand Down Expand Up @@ -53,6 +57,10 @@ chats and the adjacent campaign quickstart handoff.
legacy input that normalizes to `{ campaign: "production-scan", repoSlug }`;
requests must not send both fields. An onboarding action still stamps
completion like any onboarding kickoff.
- A successfully created campaign action chat is best-effort recorded on the
caller's matching trial-chat metadata. This keeps conversion measurement
inside the existing trial-export authorization boundary; ordinary action
chat content is never added to the cross-tenant export.
- Campaign action fields belong only to the signed-in Web DTO
(`CreateWebTaskChat`). The agent SDK's `CreateTaskChat` type and
`/api/v1/agent/chats` contract do not expose them; a raw agent request that
Expand Down
14 changes: 10 additions & 4 deletions packages/server/src/__tests__/landing-campaigns-start.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
type LandingCampaignAttribution,
MESSAGE_FORMATS,
parseLandingCampaignTrialAgentMetadata,
parseLandingCampaignTrialChatMetadata,
Expand Down Expand Up @@ -110,8 +111,9 @@ async function startProductionScan(
app: ReturnType<ReturnType<typeof useTestApp>>,
admin: Awaited<ReturnType<typeof createTestAdmin>>,
repoUrl = "https://github.com/acme/backend",
attribution?: LandingCampaignAttribution,
) {
return startCampaign(app, admin, "production-scan", repoUrl);
return startCampaign(app, admin, "production-scan", repoUrl, attribution);
}

async function startProductionScanInOrg(
Expand All @@ -128,8 +130,9 @@ async function startCampaign(
admin: Awaited<ReturnType<typeof createTestAdmin>>,
campaign: string,
repoUrl = "https://github.com/acme/backend",
attribution?: LandingCampaignAttribution,
) {
return startCampaignInOrg(app, admin, admin.organizationId, campaign, repoUrl);
return startCampaignInOrg(app, admin, admin.organizationId, campaign, repoUrl, attribution);
}

async function startCampaignInOrg(
Expand All @@ -138,12 +141,13 @@ async function startCampaignInOrg(
organizationId: string,
campaign: string,
repoUrl = "https://github.com/acme/backend",
attribution?: LandingCampaignAttribution,
) {
return app.inject({
method: "POST",
url: START_URL,
headers: { authorization: `Bearer ${admin.accessToken}` },
payload: { organizationId, campaign, repoUrl },
payload: { organizationId, campaign, repoUrl, ...(attribution ? { attribution } : {}) },
});
}

Expand Down Expand Up @@ -526,7 +530,8 @@ describe("POST /me/landing-campaigns/start", () => {
const admin = await createTestAdmin(app);
await seedOfficialRuntime(app, admin.organizationId);

const res = await startProductionScan(app, admin);
const attribution = { attemptId: crypto.randomUUID(), variant: "control" };
const res = await startProductionScan(app, admin, "https://github.com/acme/backend", attribution);

expect(res.statusCode).toBe(200);
const body = res.json<{ chatId: string; agentUuid: string; campaign: string; repoCanonicalKey: string }>();
Expand Down Expand Up @@ -622,6 +627,7 @@ describe("POST /me/landing-campaigns/start", () => {
campaign: "production-scan",
agentId: body.agentUuid,
repo: { canonicalKey: "github.com/acme/backend" },
attribution,
state: "running",
inputLocked: false,
maxAgentTurns: 1,
Expand Down
13 changes: 11 additions & 2 deletions packages/server/src/__tests__/scan-campaign-export.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ describe("scan campaign export routes", () => {
const chatCreatedAt = new Date("2026-07-10T10:00:00.000Z");
const humanMessageAt = new Date("2026-07-10T10:01:30.000Z");
const agentMessageAt = new Date("2026-07-10T10:05:00.000Z");
const attemptId = crypto.randomUUID();

await app.db
.update(agents)
Expand All @@ -134,6 +135,8 @@ describe("scan campaign export routes", () => {
skillSetId: "production-scan",
skillSetVersion: "test",
repo: { url: "https://github.com/acme/api", canonicalKey: "github.com/acme/api" },
attribution: { attemptId, variant: "control" },
actionConversion: { chatId: "action-chat-1", recordedAt: "2026-07-10T10:06:00.000Z" },
state: "completed",
inputLocked: true,
maxAgentTurns: 2,
Expand Down Expand Up @@ -238,11 +241,11 @@ describe("scan campaign export routes", () => {
},
]);

return { app, admin, agent, agentName, chatId };
return { app, admin, agent, agentName, chatId, attemptId };
}

it("exports trial, summary, and redacted message NDJSON for a client campaign", async () => {
const { app, admin, agent, agentName, chatId } = await seedTrialHistory();
const { app, admin, agent, agentName, chatId, attemptId } = await seedTrialHistory();

const create = await app.inject({
method: "POST",
Expand All @@ -260,6 +263,7 @@ describe("scan campaign export routes", () => {

expect(create.statusCode).toBe(200);
const body = create.json();
expect(body.manifest.schemaVersion).toBe(2);
expect(body.manifest.counts).toEqual({ trials: 1, chats: 1, messages: 2, exportedMessages: 2 });
const trials = body.files["trials.ndjson"].trim().split("\n").map(JSON.parse);
const summaries = body.files["summaries.ndjson"].trim().split("\n").map(JSON.parse);
Expand All @@ -280,6 +284,11 @@ describe("scan campaign export routes", () => {
firstHumanResponseSeconds: 90,
durationSeconds: 300,
hasLikelyReportLink: true,
attemptId,
variant: "control",
actionConverted: true,
actionChatId: "action-chat-1",
actionRecordedAt: "2026-07-10T10:06:00.000Z",
});
expect(exportedMessages[0]).toMatchObject({
chatId,
Expand Down
45 changes: 45 additions & 0 deletions packages/server/src/__tests__/scan-fix-idempotency.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import crypto from "node:crypto";
import { parseLandingCampaignTrialChatMetadata } from "@first-tree/shared";
import { eq } from "drizzle-orm";
import { describe, expect, it } from "vitest";
import { chats } from "../db/schema/chats.js";
import { clients } from "../db/schema/clients.js";
import { messages } from "../db/schema/messages.js";
import { createAgent } from "../services/agent.js";
import { buildLandingCampaignChatMetadata } from "../services/landing-campaigns/metadata.js";
import { createMeChat } from "../services/me-chat.js";
import { createTestAdmin, useTestApp } from "./helpers.js";

/**
Expand Down Expand Up @@ -66,13 +69,43 @@ function directFixPayload(agentUuid: string) {
};
}

async function seedAttributableTrial(
app: ReturnType<ReturnType<typeof useTestApp>>,
admin: Awaited<ReturnType<typeof createTestAdmin>>,
agentUuid: string,
): Promise<{ chatId: string; attemptId: string }> {
const { chatId } = await createMeChat(app.db, admin.humanAgentUuid, admin.organizationId, {
participantIds: [agentUuid],
topic: "Production scan trial",
});
const attemptId = crypto.randomUUID();
await app.db
.update(chats)
.set({
metadata: buildLandingCampaignChatMetadata({
campaign: "production-scan",
agentId: agentUuid,
skillSetId: "production-scan",
skillSetVersion: "test",
repo: { url: `https://github.com/${REPO_SLUG}`, canonicalKey: `github.com/${REPO_SLUG}` },
attribution: { attemptId, variant: "control" },
state: "completed",
inputLocked: true,
maxAgentTurns: 1,
}),
})
.where(eq(chats.id, chatId));
return { chatId, attemptId };
}

describe("production-scan fix conversion — cross-path idempotency", () => {
const getApp = useTestApp({ growthLandingPagesEnabled: true });

it("dedups the fix launcher across the onboarding path and the direct path", async () => {
const app = getApp();
const admin = await createTestAdmin(app);
const agent = await createOrgAgent(app, admin);
const trial = await seedAttributableTrial(app, admin, agent.uuid);

// PATH 1 — not-yet-onboarded: onboarding kickoff creates the fix launcher,
// keyed `<human>:scan-fix:<repoSlug>`.
Expand Down Expand Up @@ -107,12 +140,18 @@ describe("production-scan fix conversion — cross-path idempotency", () => {
// appending a duplicate bootstrap message.
const msgs = await app.db.select().from(messages).where(eq(messages.chatId, chat1));
expect(msgs).toHaveLength(1);
const [trialRow] = await app.db.select({ metadata: chats.metadata }).from(chats).where(eq(chats.id, trial.chatId));
expect(parseLandingCampaignTrialChatMetadata(trialRow?.metadata)).toMatchObject({
attribution: { attemptId: trial.attemptId, variant: "control" },
actionConversion: { chatId: chat1, recordedAt: expect.any(String) },
});
});

it("the direct path alone is idempotent on re-entry", async () => {
const app = getApp();
const admin = await createTestAdmin(app);
const agent = await createOrgAgent(app, admin);
const trial = await seedAttributableTrial(app, admin, agent.uuid);
const url = `/api/v1/orgs/${encodeURIComponent(admin.organizationId)}/chats`;
const headers = { authorization: `Bearer ${admin.accessToken}` };

Expand All @@ -121,6 +160,12 @@ describe("production-scan fix conversion — cross-path idempotency", () => {
expect(a.json<{ chatId: string }>().chatId).toBe(b.json<{ chatId: string }>().chatId);
const fixChats = await app.db.select().from(chats).where(eq(chats.topic, FIX_TOPIC));
expect(fixChats).toHaveLength(1);
const actionChatId = a.json<{ chatId: string }>().chatId;
const [trialRow] = await app.db.select({ metadata: chats.metadata }).from(chats).where(eq(chats.id, trial.chatId));
expect(parseLandingCampaignTrialChatMetadata(trialRow?.metadata)?.actionConversion).toMatchObject({
chatId: actionChatId,
recordedAt: expect.any(String),
});
});

it("the onboarding path alone is idempotent on re-entry", async () => {
Expand Down
7 changes: 6 additions & 1 deletion packages/server/src/api/internal/scan-campaign-exports.ts
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,11 @@ async function buildScanCampaignExportFromDb(db: Database, input: ScanCampaignEx
maxEstimatedTokens: trial?.maxEstimatedTokens ?? null,
limitReason: trial?.limitReason ?? null,
repo: trial?.repo ?? null,
attemptId: trial?.attribution?.attemptId ?? null,
variant: trial?.attribution?.variant ?? null,
actionConverted: trial?.actionConversion !== undefined,
actionChatId: trial?.actionConversion?.chatId ?? null,
actionRecordedAt: trial?.actionConversion?.recordedAt ?? null,
outcome: inferOutcome(trial?.state ?? null, counted.humanMessageCount),
humanMessageCount: counted.humanMessageCount,
agentMessageCount: counted.agentMessageCount,
Expand Down Expand Up @@ -377,7 +382,7 @@ async function buildScanCampaignExportFromDb(db: Database, input: ScanCampaignEx
const manifest = {
exportId,
createdAt,
schemaVersion: 1,
schemaVersion: 2,
request: {
clientId: input.clientId,
agentName: input.agentName,
Expand Down
16 changes: 16 additions & 0 deletions packages/server/src/api/me.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import {
campaignActionKickoffKey,
hasTreeSetupKickoffMessage,
kickoffOnboarding,
recordCampaignActionConversion,
resolveCampaignActionContext,
} from "../services/onboarding-kickoff.js";
import { getOrgContextTreeWithMeta } from "../services/org-settings.js";
Expand Down Expand Up @@ -374,6 +375,21 @@ export async function meRoutes(app: FastifyInstance): Promise<void> {
// `complete` keep their exact previous semantics.
stamp: body.stamp ?? ((body.complete ?? true) ? "completed" : "none"),
});
if (campaignAction) {
try {
await recordCampaignActionConversion(app.db, {
humanAgentId,
organizationId,
action: campaignAction,
actionChatId: result.chatId,
});
} catch (err) {
app.log.warn(
{ err, campaign: campaignAction.campaign, actionChatId: result.chatId },
"landing campaign action conversion could not be recorded",
);
}
}
if (result.sent) {
notifyRecipients(app.notifier, result.sent.recipients, result.sent.messageId);
app.log.info({ event: "onboarding.kickoff", userId, chatId: result.chatId }, "onboarding funnel: kickoff");
Expand Down
21 changes: 20 additions & 1 deletion packages/server/src/api/orgs/chats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@ import { createChat, listChatsForMember, resolveAgentIdsByNameInOrg } from "../.
import { assertNoLandingCampaignTrialAgents } from "../../services/landing-campaigns/guards.js";
import { createMeChat, listMeChatSourceCounts, listMeChats } from "../../services/me-chat.js";
import { notifyRecipients } from "../../services/notifier.js";
import { campaignActionKickoffKey, resolveCampaignActionContext } from "../../services/onboarding-kickoff.js";
import {
campaignActionKickoffKey,
recordCampaignActionConversion,
resolveCampaignActionContext,
} from "../../services/onboarding-kickoff.js";

/**
* Class B — org-scoped chat collection routes. Mounted at
Expand Down Expand Up @@ -149,6 +153,21 @@ export async function orgChatRoutes(app: FastifyInstance): Promise<void> {
? { onboardingKickoffKey: campaignActionKickoffKey(scope.humanAgentId, campaignAction) }
: {}),
});
if (campaignAction) {
try {
await recordCampaignActionConversion(app.db, {
humanAgentId: scope.humanAgentId,
organizationId: scope.organizationId,
action: campaignAction,
actionChatId: result.chat.id,
});
} catch (err) {
app.log.warn(
{ err, campaign: campaignAction.campaign, actionChatId: result.chat.id },
"landing campaign action conversion could not be recorded",
);
}
}
notifyRecipients(app.notifier, result.recipients, result.message.id);
return reply.status(201).send({
chatId: result.chat.id,
Expand Down
6 changes: 6 additions & 0 deletions packages/server/src/services/landing-campaigns/metadata.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {
type LandingCampaignActionConversion,
type LandingCampaignAttribution,
type LandingCampaignRepoMetadata,
type LandingCampaignTrialAwaitingUserKind,
type LandingCampaignTrialChatState,
Expand Down Expand Up @@ -42,6 +44,8 @@ export function buildLandingCampaignChatMetadata(input: {
skillSetId: string;
skillSetVersion: string;
repo: LandingCampaignRepoMetadata;
attribution?: LandingCampaignAttribution;
actionConversion?: LandingCampaignActionConversion;
state: LandingCampaignTrialChatState;
inputLocked: boolean;
awaitingUserKind?: LandingCampaignTrialAwaitingUserKind;
Expand All @@ -62,6 +66,8 @@ export function buildLandingCampaignChatMetadata(input: {
skillSetId: input.skillSetId,
skillSetVersion: input.skillSetVersion,
repo: input.repo,
...(input.attribution ? { attribution: input.attribution } : {}),
...(input.actionConversion ? { actionConversion: input.actionConversion } : {}),
state: input.state,
inputLocked: input.inputLocked,
...(input.awaitingUserKind ? { awaitingUserKind: input.awaitingUserKind } : {}),
Expand Down
3 changes: 3 additions & 0 deletions packages/server/src/services/landing-campaigns/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ async function ensureTrialChatAndBootstrap(
campaign: string;
repo: LandingCampaignRepoMetadata;
skillSet: LandingCampaignSkillSet;
attribution?: LandingCampaignStartRequest["attribution"];
},
): Promise<{ chatId: string; sent?: { recipients: string[]; messageId: string } }> {
const kickoffKey = [
Expand All @@ -387,6 +388,7 @@ async function ensureTrialChatAndBootstrap(
skillSetId: input.skillSet.id,
skillSetVersion: input.skillSet.version,
repo: input.repo,
...(input.attribution ? { attribution: input.attribution } : {}),
state: "running",
inputLocked: false,
maxAgentTurns: app.config.growth.landingCampaignMaxAgentTurns,
Expand Down Expand Up @@ -547,6 +549,7 @@ export async function startLandingCampaignTrial(
campaign: body.campaign,
repo,
skillSet,
...(body.attribution ? { attribution: body.attribution } : {}),
});
if (result.sent) notifyRecipients(app.notifier, result.sent.recipients, result.sent.messageId);

Expand Down
Loading
Loading