From 61854f6259cd9dfdfb68e0badb62ba5a3ad9fa68 Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:12:13 -0400 Subject: [PATCH 01/10] Import X data through Birdclaw --- docs/integration-policy.md | 6 + native/helpers/birdclaw-go/.gitignore | 1 + native/helpers/birdclaw-go/LICENSE.openclaw | 21 ++ native/helpers/birdclaw-go/go.mod | 3 + native/helpers/birdclaw-go/main.go | 267 ++++++++++++++++++ native/helpers/birdclaw-go/main_test.go | 39 +++ .../Interface/Components/PlatformIcon.swift | 4 + .../Interface/Root/RootDisplayHelpers.swift | 6 + scripts/build-cued-daemon-app.sh | 6 + skills/cued/SKILL.md | 2 +- src/core/types/provider.ts | 1 + src/platforms/core/proofs.ts | 8 + src/platforms/core/registry.ts | 6 + .../core/state/integration-state.test.ts | 8 +- src/platforms/core/state/local.ts | 23 ++ src/platforms/core/state/status.ts | 1 + src/platforms/core/types.ts | 25 ++ src/platforms/x/e2e.test.ts | 115 ++++++++ src/platforms/x/helper/binary.ts | 78 +++++ src/platforms/x/sync/bundle.test.ts | 109 +++++++ src/platforms/x/sync/bundle.ts | 261 +++++++++++++++++ src/platforms/x/sync/worker.ts | 20 ++ src/runtime/daemon/server.test.ts | 27 ++ src/runtime/daemon/server.ts | 15 +- src/runtime/onboarding.test.ts | 2 + src/runtime/projection/projector.ts | 4 +- 26 files changed, 1048 insertions(+), 10 deletions(-) create mode 100644 native/helpers/birdclaw-go/.gitignore create mode 100644 native/helpers/birdclaw-go/LICENSE.openclaw create mode 100644 native/helpers/birdclaw-go/go.mod create mode 100644 native/helpers/birdclaw-go/main.go create mode 100644 native/helpers/birdclaw-go/main_test.go create mode 100644 src/platforms/x/e2e.test.ts create mode 100644 src/platforms/x/helper/binary.ts create mode 100644 src/platforms/x/sync/bundle.test.ts create mode 100644 src/platforms/x/sync/bundle.ts create mode 100644 src/platforms/x/sync/worker.ts diff --git a/docs/integration-policy.md b/docs/integration-policy.md index b0cf3698..38f0ab1f 100644 --- a/docs/integration-policy.md +++ b/docs/integration-policy.md @@ -35,6 +35,12 @@ QR or device-linking flows are acceptable when they are the platform's normal lo Shared or managed app credentials must be injected at release time or supplied through local environment/config overrides for source builds. They must not be committed. +## X + +X reuses [Birdclaw](https://github.com/steipete/birdclaw) as the collector. Birdclaw imports an X archive and can refresh supported data through `xurl`; Cued's bundled Go helper reads `~/.birdclaw/birdclaw.sqlite` without owning X credentials or a second X API client. + +Run `birdclaw import archive ~/Downloads/twitter-archive.zip --json` first. Cued then imports DM contacts, conversations, messages, profiles, and current follower/following edges into `~/.cued/local.db`; mutual status and profile metrics live in `contact_sources.metadata_json`. Set `CUED_BIRDCLAW_DB_PATH` only when Birdclaw uses a non-default data directory. X sync remains manual by default because Birdclaw's live transports can use metered provider APIs; run `cued sync run x` explicitly. + ## Telegram Telegram is deferred for the public OSS launch. diff --git a/native/helpers/birdclaw-go/.gitignore b/native/helpers/birdclaw-go/.gitignore new file mode 100644 index 00000000..30bcfa4e --- /dev/null +++ b/native/helpers/birdclaw-go/.gitignore @@ -0,0 +1 @@ +.build/ diff --git a/native/helpers/birdclaw-go/LICENSE.openclaw b/native/helpers/birdclaw-go/LICENSE.openclaw new file mode 100644 index 00000000..c6d2a30e --- /dev/null +++ b/native/helpers/birdclaw-go/LICENSE.openclaw @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 openclaw + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/native/helpers/birdclaw-go/go.mod b/native/helpers/birdclaw-go/go.mod new file mode 100644 index 00000000..2690d6d2 --- /dev/null +++ b/native/helpers/birdclaw-go/go.mod @@ -0,0 +1,3 @@ +module cued-birdclaw-helper + +go 1.26.0 diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go new file mode 100644 index 00000000..6da4a505 --- /dev/null +++ b/native/helpers/birdclaw-go/main.go @@ -0,0 +1,267 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +const ( + helperVersion = "0.1.0" + protocolVersion = 1 + defaultLimit = 5000 +) + +type accountRow struct { + ID string `json:"id"` + Name string `json:"name"` + Handle string `json:"handle"` + ExternalUserID string `json:"external_user_id"` +} + +type profileRow struct { + ID string `json:"id"` + Handle string `json:"handle"` + DisplayName string `json:"display_name"` + Bio string `json:"bio"` + FollowersCount int `json:"followers_count"` + FollowingCount int `json:"following_count"` + AvatarURL string `json:"avatar_url"` + Location string `json:"location"` + URL string `json:"url"` + VerifiedType string `json:"verified_type"` +} + +type conversationRow struct { + ID string `json:"id"` + ParticipantProfileID string `json:"participant_profile_id"` + Title string `json:"title"` + LastMessageAt string `json:"last_message_at"` +} + +type messageRow struct { + ID string `json:"id"` + ConversationID string `json:"conversation_id"` + SenderProfileID string `json:"sender_profile_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + Direction string `json:"direction"` +} + +type followEdgeRow struct { + Direction string `json:"direction"` + ProfileID string `json:"profile_id"` + ExternalUserID string `json:"external_user_id"` + Source string `json:"source"` + FirstSeenAt string `json:"first_seen_at"` + LastSeenAt string `json:"last_seen_at"` +} + +type syncResult struct { + HelperVersion string `json:"helperVersion"` + ProtocolVersion int `json:"protocolVersion"` + DatabasePath string `json:"databasePath"` + Account accountRow `json:"account"` + Profiles []profileRow `json:"profiles"` + Conversations []conversationRow `json:"conversations"` + Messages []messageRow `json:"messages"` + FollowEdges []followEdgeRow `json:"followEdges"` + HasMore bool `json:"hasMore"` + NextCreatedAt string `json:"nextCreatedAt,omitempty"` + NextMessageID string `json:"nextMessageId,omitempty"` +} + +func main() { + if len(os.Args) < 2 { + fail(errors.New("usage: cued-birdclaw-helper ")) + } + switch os.Args[1] { + case "version": + writeJSON(map[string]any{"version": helperVersion, "protocolVersion": protocolVersion}) + case "status": + runStatus(os.Args[2:]) + case "sync": + runSync(os.Args[2:]) + default: + fail(fmt.Errorf("unknown command %q", os.Args[1])) + } +} + +func runStatus(args []string) { + flags := flag.NewFlagSet("status", flag.ContinueOnError) + dbPath := flags.String("db", "", "Birdclaw SQLite database path") + if err := flags.Parse(args); err != nil { + fail(err) + } + resolved, err := resolveDBPath(*dbPath) + if err != nil { + fail(err) + } + if _, err := os.Stat(resolved); err != nil { + fail(fmt.Errorf("birdclaw database: %w", err)) + } + var rows []map[string]any + if err := sqliteJSON(context.Background(), resolved, "SELECT COUNT(*) AS accounts FROM accounts", &rows); err != nil { + fail(err) + } + writeJSON(map[string]any{ + "helperVersion": helperVersion, "protocolVersion": protocolVersion, + "databasePath": resolved, "available": true, + }) +} + +func runSync(args []string) { + flags := flag.NewFlagSet("sync", flag.ContinueOnError) + dbPath := flags.String("db", "", "Birdclaw SQLite database path") + account := flags.String("account", "default", "Birdclaw account id or handle") + afterCreatedAt := flags.String("after-created-at", "", "exclusive message timestamp cursor") + afterMessageID := flags.String("after-message-id", "", "exclusive message id cursor") + limit := flags.Int("limit", defaultLimit, "maximum messages per page") + if err := flags.Parse(args); err != nil { + fail(err) + } + if *limit < 1 || *limit > 25000 { + fail(errors.New("limit must be between 1 and 25000")) + } + resolved, err := resolveDBPath(*dbPath) + if err != nil { + fail(err) + } + ctx := context.Background() + selected, err := selectAccount(ctx, resolved, *account) + if err != nil { + fail(err) + } + accountID := quoteSQL(selected.ID) + var result syncResult + result.HelperVersion = helperVersion + result.ProtocolVersion = protocolVersion + result.DatabasePath = resolved + result.Account = selected + + conversationFilter := "SELECT id FROM dm_conversations WHERE account_id = " + accountID + profileQuery := `SELECT p.id, COALESCE(p.handle, '') AS handle, COALESCE(p.display_name, '') AS display_name, + COALESCE(p.bio, '') AS bio, COALESCE(p.followers_count, 0) AS followers_count, + COALESCE(p.following_count, 0) AS following_count, COALESCE(p.avatar_url, '') AS avatar_url, + COALESCE(p.location, '') AS location, COALESCE(p.url, '') AS url, + COALESCE(p.verified_type, '') AS verified_type + FROM profiles p WHERE p.id IN ( + SELECT participant_profile_id FROM dm_conversations WHERE account_id = ` + accountID + ` + UNION SELECT sender_profile_id FROM dm_messages WHERE conversation_id IN (` + conversationFilter + `) + UNION SELECT profile_id FROM follow_edges WHERE account_id = ` + accountID + ` AND current = 1 + ) ORDER BY lower(p.handle), p.id` + if err := sqliteJSON(ctx, resolved, profileQuery, &result.Profiles); err != nil { + fail(err) + } + if err := sqliteJSON(ctx, resolved, `SELECT id, participant_profile_id, COALESCE(title, '') AS title, + COALESCE(last_message_at, '') AS last_message_at FROM dm_conversations + WHERE account_id = `+accountID+` ORDER BY last_message_at, id`, &result.Conversations); err != nil { + fail(err) + } + cursorFilter := "" + if *afterCreatedAt != "" { + cursorFilter = " AND (m.created_at > " + quoteSQL(*afterCreatedAt) + + " OR (m.created_at = " + quoteSQL(*afterCreatedAt) + " AND m.id > " + quoteSQL(*afterMessageID) + "))" + } + messageQuery := `SELECT m.id, m.conversation_id, COALESCE(m.sender_profile_id, '') AS sender_profile_id, + COALESCE(m.text, '') AS text, m.created_at, COALESCE(m.direction, '') AS direction + FROM dm_messages m WHERE m.conversation_id IN (` + conversationFilter + `)` + cursorFilter + + " ORDER BY m.created_at, m.id LIMIT " + strconv.Itoa(*limit+1) + if err := sqliteJSON(ctx, resolved, messageQuery, &result.Messages); err != nil { + fail(err) + } + if len(result.Messages) > *limit { + result.Messages = result.Messages[:*limit] + result.HasMore = true + } + if len(result.Messages) > 0 { + last := result.Messages[len(result.Messages)-1] + result.NextCreatedAt = last.CreatedAt + result.NextMessageID = last.ID + } + if err := sqliteJSON(ctx, resolved, `SELECT direction, profile_id, + COALESCE(external_user_id, '') AS external_user_id, COALESCE(source, '') AS source, + COALESCE(first_seen_at, '') AS first_seen_at, COALESCE(last_seen_at, '') AS last_seen_at + FROM follow_edges WHERE account_id = `+accountID+` AND current = 1 ORDER BY profile_id, direction`, &result.FollowEdges); err != nil { + fail(err) + } + writeJSON(result) +} + +func selectAccount(ctx context.Context, dbPath, requested string) (accountRow, error) { + where := "" + if strings.TrimSpace(requested) != "" && requested != "default" { + value := strings.TrimPrefix(strings.TrimSpace(requested), "@") + where = " WHERE id = " + quoteSQL(value) + " OR handle = " + quoteSQL(value) + } + var rows []accountRow + err := sqliteJSON(ctx, dbPath, `SELECT id, COALESCE(name, '') AS name, COALESCE(handle, '') AS handle, + COALESCE(external_user_id, '') AS external_user_id FROM accounts`+where+` ORDER BY is_default DESC, created_at LIMIT 1`, &rows) + if err != nil { + return accountRow{}, err + } + if len(rows) == 0 { + return accountRow{}, fmt.Errorf("birdclaw account %q not found", requested) + } + return rows[0], nil +} + +func sqliteJSON(ctx context.Context, dbPath, query string, target any) error { + binary := strings.TrimSpace(os.Getenv("CUED_BIRDCLAW_SQLITE_BINARY")) + if binary == "" { + binary = "sqlite3" + } + cmd := exec.CommandContext(ctx, binary, "-json", dbPath, query) // #nosec G204 -- no shell is used. + raw, err := cmd.Output() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("birdclaw sqlite query: %s", strings.TrimSpace(string(exitErr.Stderr))) + } + return err + } + if strings.TrimSpace(string(raw)) == "" { + raw = []byte("[]") + } + return json.Unmarshal(raw, target) +} + +func resolveDBPath(value string) (string, error) { + path := strings.TrimSpace(value) + if path == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + path = filepath.Join(home, ".birdclaw", "birdclaw.sqlite") + } else if strings.HasPrefix(path, "~/") { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + path = filepath.Join(home, strings.TrimPrefix(path, "~/")) + } + return filepath.Abs(path) +} + +func quoteSQL(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func writeJSON(value any) { + if err := json.NewEncoder(os.Stdout).Encode(value); err != nil { + fail(err) + } +} + +func fail(err error) { + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"ok": false, "error": err.Error()}) + os.Exit(1) +} diff --git a/native/helpers/birdclaw-go/main_test.go b/native/helpers/birdclaw-go/main_test.go new file mode 100644 index 00000000..ac6a92ff --- /dev/null +++ b/native/helpers/birdclaw-go/main_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSelectAccountUsesConfiguredSQLiteBinary(t *testing.T) { + bin := filepath.Join(t.TempDir(), "sqlite3") + script := `#!/bin/sh +case "$*" in + *"FROM accounts"*) + printf '%s' '[{"id":"acct-1","name":"Sam","handle":"sam","external_user_id":"10"}]' + ;; + *) + echo "unexpected query" >&2 + exit 2 + ;; +esac +` + if err := os.WriteFile(bin, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("CUED_BIRDCLAW_SQLITE_BINARY", bin) + account, err := selectAccount(t.Context(), filepath.Join(t.TempDir(), "birdclaw.sqlite"), "@sam") + if err != nil { + t.Fatal(err) + } + if account.ID != "acct-1" || account.ExternalUserID != "10" { + t.Fatalf("account = %#v", account) + } +} + +func TestQuoteSQL(t *testing.T) { + if got := quoteSQL("sam'o"); got != "'sam''o'" { + t.Fatalf("quoteSQL = %q", got) + } +} diff --git a/native/macos/CuedNative/Sources/Interface/Components/PlatformIcon.swift b/native/macos/CuedNative/Sources/Interface/Components/PlatformIcon.swift index 310650e0..768799b7 100644 --- a/native/macos/CuedNative/Sources/Interface/Components/PlatformIcon.swift +++ b/native/macos/CuedNative/Sources/Interface/Components/PlatformIcon.swift @@ -103,6 +103,10 @@ public struct PlatformIcon: View { .font(.system(size: 14, weight: .black, design: .rounded)) .foregroundStyle(.white) .offset(y: 0.5) + case "x": + Text("X") + .font(.system(size: 14, weight: .bold, design: .rounded)) + .foregroundStyle(.white) case "signal": ZStack { ZStack { diff --git a/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift b/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift index 17864672..f6b6200d 100644 --- a/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift +++ b/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift @@ -120,6 +120,8 @@ func platformAccentColor(for platform: String) -> Color { return Color(red: 0.24, green: 0.56, blue: 0.98) case "whatsapp": return Color(red: 0.13, green: 0.74, blue: 0.38) + case "x": + return Color(red: 0.08, green: 0.08, blue: 0.09) default: return .accentColor } @@ -239,6 +241,8 @@ func platformDescription(for configuration: PlatformConfig) -> String { return "Syncs WhatsApp messages after you link your phone." case "signal": return "Syncs Signal messages after you link your device." + case "x": + return "Imports X direct messages, profiles, and follow relationships from local Birdclaw data." default: return "Syncs this source on this Mac." } @@ -331,6 +335,8 @@ func platformTitle(_ platform: String, fallback: String?) -> String { return "Discord" case "whatsapp": return "WhatsApp" + case "x": + return "X" default: if let fallback = fallback?.trimmingCharacters(in: .whitespacesAndNewlines), !fallback.isEmpty { return fallback diff --git a/scripts/build-cued-daemon-app.sh b/scripts/build-cued-daemon-app.sh index ee8e6636..606f12b6 100644 --- a/scripts/build-cued-daemon-app.sh +++ b/scripts/build-cued-daemon-app.sh @@ -29,6 +29,7 @@ APP_PERMISSIONS_ENTITLEMENTS="$ROOT_DIR/scripts/packaging/app-permissions.entitl SIGNAL_HELPER_SOURCE_DIR="$ROOT_DIR/native/helpers/signal-cli/.build/cued-signal-cli" SLACK_HELPER_SOURCE="$ROOT_DIR/native/helpers/slack-go/.build/cued-slack-helper" WHATSAPP_HELPER_SOURCE="$ROOT_DIR/native/helpers/whatsapp-go/.build/cued-whatsapp-helper" +BIRDCLAW_HELPER_SOURCE="$ROOT_DIR/native/helpers/birdclaw-go/.build/cued-birdclaw-helper" PERMISSIONS_SCRIPT_SOURCE="$ROOT_DIR/scripts/request-macos-access.sh" APP_ICON_SOURCE="$ROOT_DIR/native/macos/CuedNative/Resources/AppIcon.icns" TRAY_ICON_SOURCE="$ROOT_DIR/native/macos/CuedNative/Resources/trayIconTemplate.png" @@ -240,6 +241,8 @@ mkdir -p "$(dirname "$SLACK_HELPER_SOURCE")" (cd "$ROOT_DIR/native/helpers/slack-go" && GOWORK=off go build -o "$SLACK_HELPER_SOURCE" .) >/dev/null mkdir -p "$(dirname "$WHATSAPP_HELPER_SOURCE")" (cd "$ROOT_DIR/native/helpers/whatsapp-go" && GOWORK=off go build -o "$WHATSAPP_HELPER_SOURCE" .) >/dev/null +mkdir -p "$(dirname "$BIRDCLAW_HELPER_SOURCE")" +(cd "$ROOT_DIR/native/helpers/birdclaw-go" && GOWORK=off go build -o "$BIRDCLAW_HELPER_SOURCE" .) >/dev/null npm_config_ignore_scripts=true pnpm --dir "$ROOT_DIR" --filter . deploy --legacy --prod "$DEPLOY_STAGING_DIR" >/dev/null copy_better_sqlite3_binary @@ -270,6 +273,8 @@ cp "$SLACK_HELPER_SOURCE" "$HELPERS_DIR/cued-slack-helper" chmod +x "$HELPERS_DIR/cued-slack-helper" cp "$WHATSAPP_HELPER_SOURCE" "$HELPERS_DIR/cued-whatsapp-helper" chmod +x "$HELPERS_DIR/cued-whatsapp-helper" +cp "$BIRDCLAW_HELPER_SOURCE" "$HELPERS_DIR/cued-birdclaw-helper" +chmod +x "$HELPERS_DIR/cued-birdclaw-helper" if [[ -z "$BETTER_SQLITE3_BINDING_SOURCE" ]]; then echo "better-sqlite3-multiple-ciphers native binding not found in node_modules" >&2 @@ -418,6 +423,7 @@ if [[ -f "\$SCRIPT_DIR/oauth/google-oauth-client.json" ]]; then fi export CUED_SLACK_HELPER_BINARY="\${CUED_SLACK_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-slack-helper}" export CUED_WHATSAPP_HELPER_BINARY="\${CUED_WHATSAPP_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-whatsapp-helper}" +export CUED_BIRDCLAW_HELPER_BINARY="\${CUED_BIRDCLAW_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-birdclaw-helper}" export CUED_APP_VERSION="$APP_VERSION" export CUED_RELEASE_CHANNEL="$RELEASE_CHANNEL" exec "\$NODE_BIN" "\$RUNTIME_ROOT/dist/cli.js" "\$@" diff --git a/skills/cued/SKILL.md b/skills/cued/SKILL.md index 0758d445..7fb431e9 100644 --- a/skills/cued/SKILL.md +++ b/skills/cued/SKILL.md @@ -1,6 +1,6 @@ --- name: cued -description: Queries Cued through the local `cued` CLI for the user's real contacts, conversations, and messages synced from iMessage, Slack, WhatsApp, LinkedIn, Gmail, and Signal. ALWAYS use this skill when the user asks anything about their contacts, messages, texts, conversations, or communication - even short queries like "who texted me", "check my messages", "what's John's email", or "what did we talk about on Slack". Covers finding contacts, looking up phone numbers and email addresses, reading message history, follow-up detection, ghosting detection, dormant relationships, network search, unread triage, cross-platform conversation lookup, contact deduplication, attachment lookup/fetch, and relationship analysis. The database has real data - do not tell the user you lack access to their messages or contacts. +description: Queries Cued through the local `cued` CLI for the user's real contacts, conversations, and messages synced from iMessage, Slack, WhatsApp, LinkedIn, Gmail, Signal, and X. ALWAYS use this skill when the user asks anything about their contacts, messages, texts, conversations, or communication - even short queries like "who texted me", "check my messages", "what's John's email", or "what did we talk about on Slack". Covers finding contacts, looking up phone numbers and email addresses, reading message history, follow-up detection, ghosting detection, dormant relationships, network search, unread triage, cross-platform conversation lookup, contact deduplication, attachment lookup/fetch, and relationship analysis. The database has real data - do not tell the user you lack access to their messages or contacts. --- # Cued diff --git a/src/core/types/provider.ts b/src/core/types/provider.ts index 1eb9082b..b6cc2d07 100644 --- a/src/core/types/provider.ts +++ b/src/core/types/provider.ts @@ -23,6 +23,7 @@ export interface ContactObservationPayload { fields: ContactFields; handles: ContactHandleInput[]; sourceProfileUrl?: string | null; + sourceMetadata?: Record | null; } export interface ConversationParticipantInput { diff --git a/src/platforms/core/proofs.ts b/src/platforms/core/proofs.ts index 9ace7214..886933b9 100644 --- a/src/platforms/core/proofs.ts +++ b/src/platforms/core/proofs.ts @@ -109,6 +109,14 @@ const PROOF_KIND_CONTRACTS: SyncProofKindContract[] = [ invalidatedBy: "A later helper resync reports additional pages or a newer message range.", resumeCursorMeans: "The WhatsApp helper resync cursor and selected since timestamp.", }, + { + platform: "x", + proofKind: "messages", + scopeKind: "account", + completeMeans: "All X DM messages currently present in the local Birdclaw database were read.", + invalidatedBy: "A later Birdclaw import or live refresh adds messages to its local database.", + resumeCursorMeans: "The Birdclaw DM created-at and message-id boundary for the account.", + }, ]; export function listSyncProofKindContracts(platform?: AdapterPlatform): SyncProofKindContract[] { diff --git a/src/platforms/core/registry.ts b/src/platforms/core/registry.ts index 81b7a7a5..7b508ace 100644 --- a/src/platforms/core/registry.ts +++ b/src/platforms/core/registry.ts @@ -66,6 +66,12 @@ const ADAPTER_DEFINITIONS: Record = { autoSync: true, workerTimeoutMs: 60_000, }, + x: { + platform: "x", + workerEntrypoint: join(MODULE_DIRNAME, "../x/sync/worker.js"), + autoSync: false, + workerTimeoutMs: 120_000, + }, }; export function listAdapterPlatforms(): AdapterPlatform[] { diff --git a/src/platforms/core/state/integration-state.test.ts b/src/platforms/core/state/integration-state.test.ts index 82c9b1f4..5508e4f0 100644 --- a/src/platforms/core/state/integration-state.test.ts +++ b/src/platforms/core/state/integration-state.test.ts @@ -204,7 +204,7 @@ process.exit(44); const db = createDb(); const refreshed = await refreshManagedIntegrationStates(db); - expect(refreshed.refreshed).toBe(4); + expect(refreshed.refreshed).toBe(5); expect(listIntegrationStates(db)).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -310,6 +310,7 @@ process.exit(44); "linkedin", "whatsapp", "signal", + "x", ]); db.close(); }); @@ -330,6 +331,7 @@ process.exit(44); "linkedin", "whatsapp", "signal", + "x", ]); db.close(); @@ -736,7 +738,7 @@ process.exit(44); const refreshed = refreshLocalIntegrationStates(db); - expect(refreshed.refreshed).toBe(2); + expect(refreshed.refreshed).toBe(3); expect(listIntegrationStates(db)).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -1668,7 +1670,7 @@ process.exit(44); const refreshed = await refreshManagedIntegrationStates(db); - expect(refreshed.refreshed).toBe(10); + expect(refreshed.refreshed).toBe(11); expect(listIntegrationStates(db)).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/src/platforms/core/state/local.ts b/src/platforms/core/state/local.ts index bb2500b1..52abe887 100644 --- a/src/platforms/core/state/local.ts +++ b/src/platforms/core/state/local.ts @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { existsSync } from "node:fs"; import { resolveMacOSNativeBinary } from "../../../runtime/native-binary.js"; import { DEFAULT_CHAT_DB_PATH, IMessageReader } from "../../imessage/reader.js"; +import { inspectBirdclawHelper } from "../../x/helper/binary.js"; import { type IntegrationAuthState, parseIntegrationAuthState } from "../types.js"; import type { ManagedIntegrationState } from "./types.js"; @@ -82,6 +83,7 @@ export function buildLocalIntegrationStates(): ManagedIntegrationState[] { const chatDbPath = process.env.CUED_IMESSAGE_DB_PATH ?? DEFAULT_CHAT_DB_PATH; const contactsAuthState = getContactsAuthState(); const imessageAuthState = getIMessageAuthState(); + const birdclaw = inspectBirdclawHelper(); return [ { platform: "contacts", @@ -110,5 +112,26 @@ export function buildLocalIntegrationStates(): ManagedIntegrationState[] { importedFrom: "local-system", artifactPaths: existsSync(chatDbPath) ? [chatDbPath] : [], }, + { + platform: "x", + accountKey: "default", + displayName: "X via Birdclaw", + authState: birdclaw.available + ? "authorized" + : birdclaw.helperPath + ? "missing" + : "native_helper_missing", + enabled: true, + connectionKind: "local-cli", + runtimeKind: "native", + syncCapable: birdclaw.available, + importedFrom: "birdclaw", + artifactPaths: existsSync(birdclaw.databasePath) ? [birdclaw.databasePath] : [], + metadata: { + helperPath: birdclaw.helperPath, + databasePath: birdclaw.databasePath, + reason: birdclaw.reason, + }, + }, ]; } diff --git a/src/platforms/core/state/status.ts b/src/platforms/core/state/status.ts index f54668ff..c1cc86ce 100644 --- a/src/platforms/core/state/status.ts +++ b/src/platforms/core/state/status.ts @@ -704,6 +704,7 @@ function buildSetupIntegrations( "linkedin", "whatsapp", "signal", + "x", ]; const byPlatform = new Map(); for (const integration of existingIntegrations ?? listIntegrationStates(db, options)) { diff --git a/src/platforms/core/types.ts b/src/platforms/core/types.ts index fd30efc2..2e702139 100644 --- a/src/platforms/core/types.ts +++ b/src/platforms/core/types.ts @@ -7,6 +7,7 @@ export const PLATFORM_VALUES = [ "signal", "slack", "whatsapp", + "x", ] as const; export type Platform = (typeof PLATFORM_VALUES)[number]; @@ -17,6 +18,7 @@ export const PLATFORM_PERMISSION_REQUIREMENT_VALUES = ["contacts", "full_disk_ac export type PlatformPermissionRequirement = (typeof PLATFORM_PERMISSION_REQUIREMENT_VALUES)[number]; export const PLATFORM_HELPER_REQUIREMENT_VALUES = [ + "birdclaw_helper", "signal_cli", "slack_helper", "whatsapp_helper", @@ -141,6 +143,17 @@ export const PLATFORM_DEFINITIONS = { permissionRequirements: [], helperRequirements: ["whatsapp_helper"], }, + x: { + adapter: true, + defaultAccountKey: "default", + supportsMultipleAccounts: false, + requestableIntegration: false, + requestableOrder: 7, + supportedHostOs: ["macos", "windows", "linux"], + onboardingVisible: true, + permissionRequirements: [], + helperRequirements: ["birdclaw_helper"], + }, } as const satisfies Record; export const PLATFORM_FEATURE_MATRIX = { @@ -240,6 +253,18 @@ export const PLATFORM_FEATURE_MATRIX = { attachments: "yes", contact_sync: "partial", }, + x: { + receive: "yes", + realtime_ingest: "no", + full_history_sync: "yes", + message_edits: "no", + deletes: "no", + reactions: "no", + threads_replies: "no", + read_receipts: "no", + attachments: "no", + contact_sync: "yes", + }, } as const satisfies Record>; type PlatformCapabilityFlag = "adapter" | "requestableIntegration"; diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts new file mode 100644 index 00000000..c2a96442 --- /dev/null +++ b/src/platforms/x/e2e.test.ts @@ -0,0 +1,115 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import Database from "better-sqlite3-multiple-ciphers"; +import { afterEach, describe, expect, it } from "vitest"; +import { CuedDatabase } from "../../db/database.js"; +import { projectPendingRawEvents } from "../../runtime/projection/projector.js"; +import { buildXSyncBundle } from "./sync/bundle.js"; + +describe("X Birdclaw end to end", () => { + const tempDirs: string[] = []; + + afterEach(() => { + while (tempDirs.length > 0) { + const dir = tempDirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } + }); + + it("reads Birdclaw SQLite through the Go helper and projects searchable Cued data", async () => { + const temp = mkdtempSync(join(tmpdir(), "cued-birdclaw-e2e-")); + tempDirs.push(temp); + const birdclawPath = join(temp, "birdclaw.sqlite"); + const cuedPath = join(temp, "cued.sqlite"); + const helperPath = join(temp, "cued-birdclaw-helper"); + const helperSource = resolve("native/helpers/birdclaw-go"); + execFileSync("go", ["build", "-o", helperPath, "."], { + cwd: helperSource, + env: { ...process.env, GOWORK: "off" }, + }); + + const birdclaw = new Database(birdclawPath); + birdclaw.exec(` + CREATE TABLE accounts ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, handle TEXT NOT NULL, + external_user_id TEXT, is_default INTEGER NOT NULL, created_at TEXT NOT NULL + ); + CREATE TABLE profiles ( + id TEXT PRIMARY KEY, handle TEXT NOT NULL, display_name TEXT NOT NULL, bio TEXT NOT NULL, + followers_count INTEGER NOT NULL, following_count INTEGER NOT NULL, avatar_url TEXT, + location TEXT, url TEXT, verified_type TEXT + ); + CREATE TABLE dm_conversations ( + id TEXT PRIMARY KEY, account_id TEXT NOT NULL, participant_profile_id TEXT NOT NULL, + title TEXT NOT NULL, last_message_at TEXT NOT NULL + ); + CREATE TABLE dm_messages ( + id TEXT PRIMARY KEY, conversation_id TEXT NOT NULL, sender_profile_id TEXT NOT NULL, + text TEXT NOT NULL, created_at TEXT NOT NULL, direction TEXT NOT NULL + ); + CREATE TABLE follow_edges ( + account_id TEXT NOT NULL, direction TEXT NOT NULL, profile_id TEXT NOT NULL, + external_user_id TEXT NOT NULL, source TEXT NOT NULL, current INTEGER NOT NULL, + first_seen_at TEXT NOT NULL, last_seen_at TEXT NOT NULL + ); + INSERT INTO accounts VALUES ('acct-1', 'Sam', 'sam', '10', 1, '2026-01-01'); + INSERT INTO profiles VALUES ( + 'profile-20', 'ava', 'Ava Chen', 'Robotics', 100, 80, + 'https://pbs.twimg.com/ava.jpg', 'New York', 'https://ava.example', 'blue' + ); + INSERT INTO dm_conversations VALUES ( + '10-20', 'acct-1', 'profile-20', 'Ava Chen', '2026-08-03T12:00:00.000Z' + ); + INSERT INTO dm_messages VALUES ( + 'message-1', '10-20', 'profile-20', 'Hello from real SQLite', + '2026-08-03T12:00:00.000Z', 'inbound' + ); + INSERT INTO follow_edges VALUES + ('acct-1', 'followers', 'profile-20', '20', 'archive', 1, '2026-01-01', '2026-08-01'), + ('acct-1', 'following', 'profile-20', '20', 'archive', 1, '2026-01-01', '2026-08-01'); + `); + birdclaw.close(); + + const previousHelper = process.env.CUED_BIRDCLAW_HELPER_BINARY; + const previousDatabase = process.env.CUED_BIRDCLAW_DB_PATH; + process.env.CUED_BIRDCLAW_HELPER_BINARY = helperPath; + process.env.CUED_BIRDCLAW_DB_PATH = birdclawPath; + const bundle = await buildXSyncBundle(); + if (previousHelper === undefined) delete process.env.CUED_BIRDCLAW_HELPER_BINARY; + else process.env.CUED_BIRDCLAW_HELPER_BINARY = previousHelper; + if (previousDatabase === undefined) delete process.env.CUED_BIRDCLAW_DB_PATH; + else process.env.CUED_BIRDCLAW_DB_PATH = previousDatabase; + const cued = new CuedDatabase(cuedPath); + try { + cued.migrate(); + cued.upsertSourceAccounts(bundle.sourceAccounts ?? []); + cued.insertRawEvents(bundle.rawEvents); + const projection = projectPendingRawEvents(cued); + expect(projection.appliedRawEvents).toBe(3); + + const sqlite = ( + cued as unknown as { + sqlite: { + prepare: (sql: string) => { get: (...params: unknown[]) => Record }; + }; + } + ).sqlite; + const contact = sqlite + .prepare(` + SELECT c.name, cs.metadata_json + FROM contacts c JOIN contact_sources cs ON cs.contact_id = c.id + WHERE cs.platform = 'x' AND cs.source_entity_key = 'x:user:20' + `) + .get() as { name: string; metadata_json: string }; + expect(contact.name).toBe("Ava Chen"); + expect(JSON.parse(contact.metadata_json)).toMatchObject({ mutual: true, followsYou: true }); + expect( + sqlite.prepare("SELECT content FROM messages WHERE platform = 'x'").get(), + ).toMatchObject({ content: "Hello from real SQLite" }); + } finally { + cued.close(); + } + }, 30_000); +}); diff --git a/src/platforms/x/helper/binary.ts b/src/platforms/x/helper/binary.ts new file mode 100644 index 00000000..10b445a3 --- /dev/null +++ b/src/platforms/x/helper/binary.ts @@ -0,0 +1,78 @@ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const BINARY_NAME = "cued-birdclaw-helper"; +const SUPPORTED_PROTOCOL_VERSION = 1; + +function repoRoot(): string { + return resolve(dirname(fileURLToPath(import.meta.url)), "../../../.."); +} + +export function resolveBirdclawDatabasePath(env: NodeJS.ProcessEnv = process.env): string { + return env.CUED_BIRDCLAW_DB_PATH?.trim() || join(homedir(), ".birdclaw", "birdclaw.sqlite"); +} + +export function birdclawHelperCandidates(root = repoRoot()): string[] { + return [ + process.env.CUED_APP_PATH?.trim() + ? join(process.env.CUED_APP_PATH.trim(), "Contents", "Resources", "helpers", BINARY_NAME) + : null, + resolve(dirname(fileURLToPath(import.meta.url)), "../../../../../helpers", BINARY_NAME), + join(root, "native", "helpers", "birdclaw-go", ".build", BINARY_NAME), + join(root, "native", "helpers", "birdclaw-go", BINARY_NAME), + ].filter((value): value is string => Boolean(value)); +} + +export function resolveBirdclawHelperBinary( + envValue = process.env.CUED_BIRDCLAW_HELPER_BINARY, +): string | null { + return ( + envValue?.trim() || + birdclawHelperCandidates().find((candidate) => existsSync(candidate)) || + null + ); +} + +export function inspectBirdclawHelper(): { + helperPath: string | null; + databasePath: string; + available: boolean; + reason: string | null; +} { + const helperPath = resolveBirdclawHelperBinary(); + const databasePath = resolveBirdclawDatabasePath(); + if (!helperPath) { + return { + helperPath: null, + databasePath, + available: false, + reason: "Birdclaw helper is missing", + }; + } + if (!existsSync(databasePath)) { + return { helperPath, databasePath, available: false, reason: "Birdclaw database is missing" }; + } + try { + const output = execFileSync(helperPath, ["version"], { encoding: "utf8" }); + const parsed = JSON.parse(output) as { protocolVersion?: unknown }; + if (parsed.protocolVersion !== SUPPORTED_PROTOCOL_VERSION) { + return { + helperPath, + databasePath, + available: false, + reason: `Unsupported Birdclaw helper protocol: ${String(parsed.protocolVersion)}`, + }; + } + return { helperPath, databasePath, available: true, reason: null }; + } catch (error) { + return { + helperPath, + databasePath, + available: false, + reason: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/src/platforms/x/sync/bundle.test.ts b/src/platforms/x/sync/bundle.test.ts new file mode 100644 index 00000000..a73da247 --- /dev/null +++ b/src/platforms/x/sync/bundle.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { type BirdclawSnapshot, buildXSyncBundle } from "./bundle.js"; + +const snapshot: BirdclawSnapshot = { + account: { id: "acct-1", name: "Sam", handle: "sam", external_user_id: "10" }, + profiles: [ + { + id: "profile-20", + handle: "ava", + display_name: "Ava", + bio: "Robotics", + followers_count: 100, + following_count: 80, + avatar_url: "https://pbs.twimg.com/ava.jpg", + location: "New York", + url: "https://ava.example", + verified_type: "blue", + }, + ], + conversations: [ + { + id: "10-20", + participant_profile_id: "profile-20", + title: "Ava", + last_message_at: "2026-08-03T12:00:00.000Z", + }, + ], + messages: [ + { + id: "message-1", + conversation_id: "10-20", + sender_profile_id: "profile-20", + text: "Hello from Birdclaw", + created_at: "2026-08-03T12:00:00.000Z", + direction: "inbound", + }, + ], + followEdges: [ + { + direction: "followers", + profile_id: "profile-20", + external_user_id: "20", + source: "archive", + first_seen_at: "2026-01-01T00:00:00.000Z", + last_seen_at: "2026-08-01T00:00:00.000Z", + }, + { + direction: "following", + profile_id: "profile-20", + external_user_id: "20", + source: "archive", + first_seen_at: "2026-01-01T00:00:00.000Z", + last_seen_at: "2026-08-01T00:00:00.000Z", + }, + ], + hasMore: false, + nextCreatedAt: "2026-08-03T12:00:00.000Z", + nextMessageId: "message-1", +}; + +describe("Birdclaw X sync bundle", () => { + it("normalizes DMs, deterministic X identities, and mutual graph metadata", async () => { + const bundle = await buildXSyncBundle({ accountKey: "default" }, { snapshot }); + + expect(bundle.sourceAccounts).toEqual([ + { platform: "x", accountKey: "acct-1", displayName: "@sam" }, + ]); + expect(bundle.rawEvents.map((event) => event.entityKind)).toEqual([ + "contact", + "conversation", + "message", + ]); + expect(bundle.rawEvents[0]?.payload).toMatchObject({ + sourceEntityKey: "x:user:20", + handles: expect.arrayContaining([ + { type: "x_user_id", value: "20", deterministic: true }, + { type: "x_handle", value: "ava", deterministic: true }, + ]), + sourceMetadata: { + followsYou: true, + youFollow: true, + mutual: true, + followersCount: 100, + }, + }); + expect(bundle.rawEvents.at(-1)?.payload).toMatchObject({ + content: "Hello from Birdclaw", + senderSourceKey: "x:user:20", + isFromMe: false, + }); + expect(bundle.proofs?.[0]?.coverage).toMatchObject({ + source: "birdclaw.sqlite", + relationshipEdges: 2, + }); + }); + + it("continues from the helper cursor when another DM page remains", async () => { + const bundle = await buildXSyncBundle( + {}, + { snapshot: { ...snapshot, hasMore: true }, sourceCursor: { createdAt: null } }, + ); + expect(bundle.hasMore).toBe(true); + expect(bundle.sourceCursor).toEqual({ + createdAt: "2026-08-03T12:00:00.000Z", + messageId: "message-1", + }); + expect(bundle.proofs?.[0]?.status).toBe("running"); + }); +}); diff --git a/src/platforms/x/sync/bundle.ts b/src/platforms/x/sync/bundle.ts new file mode 100644 index 00000000..68d4ef36 --- /dev/null +++ b/src/platforms/x/sync/bundle.ts @@ -0,0 +1,261 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promisify } from "node:util"; +import type { + ContactObservationPayload, + ConversationObservationPayload, + MessagePayload, + ProviderRawEventInput, +} from "../../../core/types/provider.js"; +import type { SyncBundle } from "../../core/sync.js"; +import { resolveBirdclawDatabasePath, resolveBirdclawHelperBinary } from "../helper/binary.js"; + +const execFileAsync = promisify(execFile); + +export interface BirdclawSnapshot { + account: { id: string; name: string; handle: string; external_user_id: string }; + profiles: Array<{ + id: string; + handle: string; + display_name: string; + bio: string; + followers_count: number; + following_count: number; + avatar_url: string; + location: string; + url: string; + verified_type: string; + }>; + conversations: Array<{ + id: string; + participant_profile_id: string; + title: string; + last_message_at: string; + }>; + messages: Array<{ + id: string; + conversation_id: string; + sender_profile_id: string; + text: string; + created_at: string; + direction: string; + }>; + followEdges: Array<{ + direction: string; + profile_id: string; + external_user_id: string; + source: string; + first_seen_at: string; + last_seen_at: string; + }>; + hasMore: boolean; + nextCreatedAt?: string; + nextMessageId?: string; +} + +type BirdclawCursor = { createdAt: string | null; messageId: string | null }; + +function stableId(seed: string): string { + return createHash("sha256").update(seed).digest("hex"); +} + +function parseCursor(value: unknown): BirdclawCursor { + const cursor = value && typeof value === "object" ? (value as Record) : {}; + return { + createdAt: typeof cursor.createdAt === "string" ? cursor.createdAt : null, + messageId: typeof cursor.messageId === "string" ? cursor.messageId : null, + }; +} + +async function readBirdclawSnapshot( + accountKey: string, + cursor: BirdclawCursor, +): Promise { + const helper = resolveBirdclawHelperBinary(); + if (!helper) throw new Error("Birdclaw helper is not installed"); + const args = ["sync", "--db", resolveBirdclawDatabasePath(), "--account", accountKey]; + if (cursor.createdAt) { + args.push("--after-created-at", cursor.createdAt, "--after-message-id", cursor.messageId ?? ""); + } + const { stdout } = await execFileAsync(helper, args, { + timeout: 120_000, + maxBuffer: 256 * 1024 * 1024, + }); + return JSON.parse(stdout) as BirdclawSnapshot; +} + +function sourceKey(id: string): string { + return `x:user:${id}`; +} + +export function buildBirdclawRawEvents(input: { + accountKey: string; + snapshot: BirdclawSnapshot; + observedAt: number; +}): SyncBundle["rawEvents"] { + const { snapshot } = input; + const relationshipByProfile = new Map>(); + const externalIdByProfile = new Map(); + for (const edge of snapshot.followEdges) { + const directions = relationshipByProfile.get(edge.profile_id) ?? new Set(); + directions.add(edge.direction); + relationshipByProfile.set(edge.profile_id, directions); + if (edge.external_user_id) externalIdByProfile.set(edge.profile_id, edge.external_user_id); + } + const profileSourceKeys = new Map(); + const rawEvents: SyncBundle["rawEvents"] = []; + for (const profile of snapshot.profiles) { + const externalId = externalIdByProfile.get(profile.id) || profile.id; + const contactSourceKey = sourceKey(externalId); + profileSourceKeys.set(profile.id, contactSourceKey); + const directions = relationshipByProfile.get(profile.id) ?? new Set(); + const id = stableId(`x:contact:${input.accountKey}:${externalId}`); + rawEvents.push({ + id, + platform: "x", + accountKey: input.accountKey, + entityKind: "contact", + eventKind: "observed", + externalEntityId: externalId, + observedAt: input.observedAt, + dedupeKey: id, + payload: { + sourceEntityKey: contactSourceKey, + fields: { + display_name: + profile.display_name || (profile.handle ? `@${profile.handle}` : profile.id), + photo_url: profile.avatar_url || null, + }, + handles: [ + ...(externalId !== profile.id + ? [{ type: "x_user_id", value: externalId, deterministic: true }] + : []), + ...(profile.handle + ? [ + { type: "x_handle", value: profile.handle, deterministic: true }, + { + type: "x_profile_url", + value: `https://x.com/${profile.handle}`, + deterministic: true, + }, + ] + : []), + ], + sourceProfileUrl: profile.handle ? `https://x.com/${profile.handle}` : null, + sourceMetadata: { + birdclawProfileId: profile.id, + bio: profile.bio, + location: profile.location, + url: profile.url, + verifiedType: profile.verified_type, + followersCount: profile.followers_count, + followingCount: profile.following_count, + followsYou: directions.has("followers"), + youFollow: directions.has("following"), + mutual: directions.has("followers") && directions.has("following"), + }, + } satisfies ContactObservationPayload, + } satisfies ProviderRawEventInput); + } + + const selfKey = sourceKey(snapshot.account.external_user_id || snapshot.account.id); + for (const conversation of snapshot.conversations) { + const conversationId = stableId(`x:conversation:${input.accountKey}:${conversation.id}`); + const remoteKey = profileSourceKeys.get(conversation.participant_profile_id); + rawEvents.push({ + id: conversationId, + platform: "x", + accountKey: input.accountKey, + entityKind: "conversation", + eventKind: "observed", + conversationExternalId: conversation.id, + observedAt: input.observedAt, + dedupeKey: conversationId, + payload: { + sourceConversationKey: `x:conversation:${conversation.id}`, + conversationType: "dm", + displayName: conversation.title || "X conversation", + participants: [ + { sourceEntityKey: selfKey, isSelf: true }, + ...(remoteKey ? [{ sourceEntityKey: remoteKey }] : []), + ], + } satisfies ConversationObservationPayload, + }); + } + for (const message of snapshot.messages) { + const sentAt = Date.parse(message.created_at); + const id = stableId(`x:message:${input.accountKey}:${message.id}`); + rawEvents.push({ + id, + platform: "x", + accountKey: input.accountKey, + entityKind: "message", + eventKind: "created", + externalEntityId: message.id, + conversationExternalId: message.conversation_id, + occurredAt: Number.isFinite(sentAt) ? sentAt : input.observedAt, + observedAt: input.observedAt, + dedupeKey: id, + payload: { + sourceMessageKey: `x:message:${message.id}`, + sourceConversationKey: `x:conversation:${message.conversation_id}`, + senderSourceKey: + message.direction === "outbound" + ? selfKey + : (profileSourceKeys.get(message.sender_profile_id) ?? null), + sentAt: Number.isFinite(sentAt) ? sentAt : input.observedAt, + content: message.text, + isFromMe: message.direction === "outbound", + } satisfies MessagePayload, + }); + } + return rawEvents; +} + +export async function buildXSyncBundle( + input: { accountKey?: string } = {}, + options: { snapshot?: BirdclawSnapshot; sourceCursor?: unknown } = {}, +): Promise { + const requestedAccountKey = input.accountKey ?? process.env.CUED_ACCOUNT_KEY ?? "default"; + const cursor = parseCursor(options.sourceCursor); + const snapshot = options.snapshot ?? (await readBirdclawSnapshot(requestedAccountKey, cursor)); + const accountKey = snapshot.account.id; + const observedAt = Date.now(); + const rawEvents = buildBirdclawRawEvents({ accountKey, snapshot, observedAt }); + const sourceCursor: BirdclawCursor = { + createdAt: snapshot.nextCreatedAt ?? cursor.createdAt, + messageId: snapshot.nextMessageId ?? cursor.messageId, + }; + return { + sourceAccounts: [ + { + platform: "x", + accountKey, + displayName: snapshot.account.handle + ? `@${snapshot.account.handle}` + : snapshot.account.name, + }, + ], + rawEvents, + sourceCursor, + syncMode: cursor.createdAt ? "incremental" : "full", + hasMore: snapshot.hasMore, + continuation: snapshot.hasMore + ? { reason: "account_pagination", detail: "Birdclaw DM page remains" } + : undefined, + proofs: [ + { + scope: { kind: "account", key: "birdclaw_local_archive" }, + proofKind: "messages", + status: snapshot.hasMore ? "running" : "complete", + observedAt, + resumeCursor: snapshot.hasMore ? sourceCursor : null, + coverage: { + source: "birdclaw.sqlite", + relationshipEdges: snapshot.followEdges.length, + }, + stats: { messageCount: snapshot.messages.length, rawEventCount: rawEvents.length }, + }, + ], + }; +} diff --git a/src/platforms/x/sync/worker.ts b/src/platforms/x/sync/worker.ts new file mode 100644 index 00000000..34b6cad8 --- /dev/null +++ b/src/platforms/x/sync/worker.ts @@ -0,0 +1,20 @@ +import { readAdapterInvocationEnv } from "../../core/invocation.js"; +import { buildXSyncBundle } from "./bundle.js"; + +async function main(): Promise { + try { + const invocation = readAdapterInvocationEnv("x"); + const bundle = await buildXSyncBundle( + { accountKey: process.env.CUED_ACCOUNT_KEY }, + { sourceCursor: invocation.sourceCursor }, + ); + process.stdout.write(JSON.stringify({ ok: true, bundle })); + } catch (error) { + process.stdout.write( + JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) }), + ); + process.exitCode = 1; + } +} + +void main(); diff --git a/src/runtime/daemon/server.test.ts b/src/runtime/daemon/server.test.ts index bc723db6..0882097e 100644 --- a/src/runtime/daemon/server.test.ts +++ b/src/runtime/daemon/server.test.ts @@ -348,6 +348,33 @@ describe("interactive auth sessions", () => { }); describe("sync resume targets", () => { + it("does not poll metered X reads unless autosync is explicitly configured", () => { + const previous = process.env.CUED_AUTOSYNC_PLATFORMS; + delete process.env.CUED_AUTOSYNC_PLATFORMS; + try { + expect( + getAutoSyncTargets({ + listEnabledSyncTargets: () => [ + { platform: "x", account_key: "default" }, + { platform: "slack", account_key: "workspace" }, + ], + listIntegrationStates: () => [], + }), + ).toEqual([{ platform: "slack", accountKey: "workspace" }]); + + process.env.CUED_AUTOSYNC_PLATFORMS = "x"; + expect( + getAutoSyncTargets({ + listEnabledSyncTargets: () => [], + listIntegrationStates: () => [], + }), + ).toEqual([{ platform: "x", accountKey: "default" }]); + } finally { + if (previous == null) delete process.env.CUED_AUTOSYNC_PLATFORMS; + else process.env.CUED_AUTOSYNC_PLATFORMS = previous; + } + }); + it("allows autosync to be explicitly disabled", () => { const previous = process.env.CUED_AUTOSYNC_PLATFORMS; process.env.CUED_AUTOSYNC_PLATFORMS = "none"; diff --git a/src/runtime/daemon/server.ts b/src/runtime/daemon/server.ts index a0a35fe0..c4c57d60 100644 --- a/src/runtime/daemon/server.ts +++ b/src/runtime/daemon/server.ts @@ -28,7 +28,11 @@ import { buildAdapterInvocationEnv, selectAdapterInvocationProofs, } from "../../platforms/core/invocation.js"; -import { isAdapterPlatform, listAutoSyncPlatforms } from "../../platforms/core/registry.js"; +import { + getAdapterDefinition, + isAdapterPlatform, + listAutoSyncPlatforms, +} from "../../platforms/core/registry.js"; import { isAdapterWorkerError, runAdapter } from "../../platforms/core/runner.js"; import { loadIntegrationSecret } from "../../platforms/core/secrets/keychain.js"; import { refreshLocalIntegrationStates } from "../../platforms/core/state/local-refresh.js"; @@ -407,8 +411,10 @@ export function getAutoSyncTargets( const enabled = db .listEnabledSyncTargets() - .filter((target): target is { platform: AdapterPlatform; account_key: string } => - isAdapterPlatform(target.platform), + .filter( + (target): target is { platform: AdapterPlatform; account_key: string } => + isAdapterPlatform(target.platform) && + getAdapterDefinition(target.platform)?.autoSync === true, ) .map((target) => ({ platform: target.platform, @@ -5372,7 +5378,8 @@ async function dispatchRequest( wakeIngest: schedulers.wakeIngest, onRuntimeStateChanged: onAuthRuntimeStateChanged, shouldQueueAuthenticatedSync: (platform) => - platform !== "whatsapp" || shouldRunRealtimePlatform("whatsapp"), + getAdapterDefinition(platform)?.autoSync === true && + (platform !== "whatsapp" || shouldRunRealtimePlatform("whatsapp")), emitAuthenticatedHook: async (platform, accountKey) => { await emitAuthenticatedHook(db, platform, accountKey); }, diff --git a/src/runtime/onboarding.test.ts b/src/runtime/onboarding.test.ts index 4ba9ca17..179a2bb3 100644 --- a/src/runtime/onboarding.test.ts +++ b/src/runtime/onboarding.test.ts @@ -123,6 +123,7 @@ describe("onboarding snapshot", () => { "linkedin", "whatsapp", "signal", + "x", ]); db.close(); @@ -157,6 +158,7 @@ describe("onboarding snapshot", () => { "linkedin", "whatsapp", "signal", + "x", ]); db.close(); diff --git a/src/runtime/projection/projector.ts b/src/runtime/projection/projector.ts index 3b0e1a80..bc6e9ece 100644 --- a/src/runtime/projection/projector.ts +++ b/src/runtime/projection/projector.ts @@ -1209,7 +1209,7 @@ function projectContactObservation( accountKey: event.account_key, sourceEntityKey: payload.sourceEntityKey, profileUrl: normalizeText(payload.sourceProfileUrl ?? null), - metadataJson: null, + metadataJson: payload.sourceMetadata ? JSON.stringify(payload.sourceMetadata) : null, firstSeenAt: event.observed_at, lastSeenAt: event.observed_at, }) @@ -1218,7 +1218,7 @@ function projectContactObservation( set: { contactId, profileUrl: normalizeText(payload.sourceProfileUrl ?? null), - metadataJson: null, + metadataJson: payload.sourceMetadata ? JSON.stringify(payload.sourceMetadata) : null, lastSeenAt: event.observed_at, }, }) From 05d97cb8422c99243c496682a114e3cac0b5f53d Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:26:57 -0400 Subject: [PATCH 02/10] Own X archive ingestion in Cued --- docs/integration-policy.md | 4 +- native/helpers/birdclaw-go/LICENSE.openclaw | 24 + native/helpers/birdclaw-go/main.go | 500 ++++++++++++++---- native/helpers/birdclaw-go/main_test.go | 66 ++- .../Interface/Root/RootDisplayHelpers.swift | 2 +- scripts/build-cued-daemon-app.sh | 2 +- src/platforms/core/proofs.ts | 6 +- src/platforms/core/state/local.ts | 9 +- src/platforms/core/types.ts | 6 +- src/platforms/x/e2e.test.ts | 161 +++--- src/platforms/x/helper/binary.ts | 46 +- src/platforms/x/sync/bundle.test.ts | 76 ++- src/platforms/x/sync/bundle.ts | 138 +++-- 13 files changed, 758 insertions(+), 282 deletions(-) diff --git a/docs/integration-policy.md b/docs/integration-policy.md index 38f0ab1f..c2b3a5d8 100644 --- a/docs/integration-policy.md +++ b/docs/integration-policy.md @@ -37,9 +37,9 @@ Shared or managed app credentials must be injected at release time or supplied t ## X -X reuses [Birdclaw](https://github.com/steipete/birdclaw) as the collector. Birdclaw imports an X archive and can refresh supported data through `xurl`; Cued's bundled Go helper reads `~/.birdclaw/birdclaw.sqlite` without owning X credentials or a second X API client. +X archive support ports only the useful normalization behavior from [Birdclaw](https://github.com/steipete/birdclaw) and the small Go-adapter pattern from [Clawdex](https://github.com/openclaw/clawdex). The bundled helper reads an X archive zip directly; Cued does not install, invoke, or read the database of either project. -Run `birdclaw import archive ~/Downloads/twitter-archive.zip --json` first. Cued then imports DM contacts, conversations, messages, profiles, and current follower/following edges into `~/.cued/local.db`; mutual status and profile metrics live in `contact_sources.metadata_json`. Set `CUED_BIRDCLAW_DB_PATH` only when Birdclaw uses a non-default data directory. X sync remains manual by default because Birdclaw's live transports can use metered provider APIs; run `cued sync run x` explicitly. +Put the exported archive in `~/Downloads` with `twitter` or `x` and `archive` in the zip filename, or set `CUED_X_ARCHIVE_PATH`. Cued imports DM contacts, conversations, messages, and follower/following relationships into `~/.cued/local.db`; current mutual state and its archive generation live in `contact_sources.metadata_json`. Sync is manual because the archive is a point-in-time export: run `cued sync run x` after replacing it. ## Telegram diff --git a/native/helpers/birdclaw-go/LICENSE.openclaw b/native/helpers/birdclaw-go/LICENSE.openclaw index c6d2a30e..bde8f2fb 100644 --- a/native/helpers/birdclaw-go/LICENSE.openclaw +++ b/native/helpers/birdclaw-go/LICENSE.openclaw @@ -19,3 +19,27 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +MIT License + +Copyright (c) 2026 Peter Steinberger + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index 6da4a505..cf2c594c 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -1,24 +1,34 @@ package main import ( - "context" + "archive/zip" + "bufio" "encoding/json" "errors" "flag" "fmt" "os" - "os/exec" "path/filepath" + "regexp" + "sort" "strconv" "strings" + "time" ) const ( - helperVersion = "0.1.0" + helperVersion = "0.2.0" protocolVersion = 1 defaultLimit = 5000 ) +var ( + dmEntryPattern = regexp.MustCompile(`(?i)(?:^|/)data/direct-messages(?:-group)?(?:-part\d+)?\.js$`) + tweetEntryPattern = regexp.MustCompile(`(?i)(?:^|/)data/tweets(?:-part\d+)?\.js$`) + followerEntryPattern = regexp.MustCompile(`(?i)(?:^|/)data/follower(?:-part\d+)?\.js$`) + followingEntryPattern = regexp.MustCompile(`(?i)(?:^|/)data/following(?:-part\d+)?\.js$`) +) + type accountRow struct { ID string `json:"id"` Name string `json:"name"` @@ -40,10 +50,11 @@ type profileRow struct { } type conversationRow struct { - ID string `json:"id"` - ParticipantProfileID string `json:"participant_profile_id"` - Title string `json:"title"` - LastMessageAt string `json:"last_message_at"` + ID string `json:"id"` + ParticipantIDs []string `json:"participant_ids"` + Title string `json:"title"` + LastMessageAt string `json:"last_message_at"` + IsGroup bool `json:"is_group"` } type messageRow struct { @@ -67,7 +78,10 @@ type followEdgeRow struct { type syncResult struct { HelperVersion string `json:"helperVersion"` ProtocolVersion int `json:"protocolVersion"` - DatabasePath string `json:"databasePath"` + ArchivePath string `json:"archivePath"` + Generation string `json:"generation"` + NextPhase string `json:"nextPhase,omitempty"` + NextEdgeOffset int `json:"nextEdgeOffset,omitempty"` Account accountRow `json:"account"` Profiles []profileRow `json:"profiles"` Conversations []conversationRow `json:"conversations"` @@ -78,6 +92,14 @@ type syncResult struct { NextMessageID string `json:"nextMessageId,omitempty"` } +type archiveData struct { + account accountRow + profiles map[string]profileRow + conversations map[string]conversationRow + messages []messageRow + followEdges []followEdgeRow +} + func main() { if len(os.Args) < 2 { fail(errors.New("usage: cued-birdclaw-helper ")) @@ -96,33 +118,36 @@ func main() { func runStatus(args []string) { flags := flag.NewFlagSet("status", flag.ContinueOnError) - dbPath := flags.String("db", "", "Birdclaw SQLite database path") + archivePath := flags.String("archive", "", "X archive zip path") if err := flags.Parse(args); err != nil { fail(err) } - resolved, err := resolveDBPath(*dbPath) + resolved, err := resolveArchivePath(*archivePath) if err != nil { fail(err) } - if _, err := os.Stat(resolved); err != nil { - fail(fmt.Errorf("birdclaw database: %w", err)) + reader, err := zip.OpenReader(resolved) + if err != nil { + fail(fmt.Errorf("open X archive: %w", err)) } - var rows []map[string]any - if err := sqliteJSON(context.Background(), resolved, "SELECT COUNT(*) AS accounts FROM accounts", &rows); err != nil { - fail(err) + defer reader.Close() + if findEntry(reader.File, `data/account.js`) == nil { + fail(errors.New("X archive is missing data/account.js")) } writeJSON(map[string]any{ "helperVersion": helperVersion, "protocolVersion": protocolVersion, - "databasePath": resolved, "available": true, + "archivePath": resolved, "available": true, }) } func runSync(args []string) { flags := flag.NewFlagSet("sync", flag.ContinueOnError) - dbPath := flags.String("db", "", "Birdclaw SQLite database path") - account := flags.String("account", "default", "Birdclaw account id or handle") + archivePath := flags.String("archive", "", "X archive zip path") afterCreatedAt := flags.String("after-created-at", "", "exclusive message timestamp cursor") afterMessageID := flags.String("after-message-id", "", "exclusive message id cursor") + expectedGeneration := flags.String("generation", "", "archive generation for the cursor") + phase := flags.String("phase", "messages", "messages or relationships") + edgeOffset := flags.Int("edge-offset", 0, "relationship cursor offset") limit := flags.Int("limit", defaultLimit, "maximum messages per page") if err := flags.Parse(args); err != nil { fail(err) @@ -130,118 +155,339 @@ func runSync(args []string) { if *limit < 1 || *limit > 25000 { fail(errors.New("limit must be between 1 and 25000")) } - resolved, err := resolveDBPath(*dbPath) + resolved, err := resolveArchivePath(*archivePath) if err != nil { fail(err) } - ctx := context.Background() - selected, err := selectAccount(ctx, resolved, *account) + generation, err := archiveGeneration(resolved) if err != nil { fail(err) } - accountID := quoteSQL(selected.ID) - var result syncResult - result.HelperVersion = helperVersion - result.ProtocolVersion = protocolVersion - result.DatabasePath = resolved - result.Account = selected - - conversationFilter := "SELECT id FROM dm_conversations WHERE account_id = " + accountID - profileQuery := `SELECT p.id, COALESCE(p.handle, '') AS handle, COALESCE(p.display_name, '') AS display_name, - COALESCE(p.bio, '') AS bio, COALESCE(p.followers_count, 0) AS followers_count, - COALESCE(p.following_count, 0) AS following_count, COALESCE(p.avatar_url, '') AS avatar_url, - COALESCE(p.location, '') AS location, COALESCE(p.url, '') AS url, - COALESCE(p.verified_type, '') AS verified_type - FROM profiles p WHERE p.id IN ( - SELECT participant_profile_id FROM dm_conversations WHERE account_id = ` + accountID + ` - UNION SELECT sender_profile_id FROM dm_messages WHERE conversation_id IN (` + conversationFilter + `) - UNION SELECT profile_id FROM follow_edges WHERE account_id = ` + accountID + ` AND current = 1 - ) ORDER BY lower(p.handle), p.id` - if err := sqliteJSON(ctx, resolved, profileQuery, &result.Profiles); err != nil { - fail(err) + if *expectedGeneration != "" && *expectedGeneration != generation { + *afterCreatedAt, *afterMessageID, *phase, *edgeOffset = "", "", "messages", 0 } - if err := sqliteJSON(ctx, resolved, `SELECT id, participant_profile_id, COALESCE(title, '') AS title, - COALESCE(last_message_at, '') AS last_message_at FROM dm_conversations - WHERE account_id = `+accountID+` ORDER BY last_message_at, id`, &result.Conversations); err != nil { + data, err := readArchive(resolved) + if err != nil { fail(err) } - cursorFilter := "" - if *afterCreatedAt != "" { - cursorFilter = " AND (m.created_at > " + quoteSQL(*afterCreatedAt) + - " OR (m.created_at = " + quoteSQL(*afterCreatedAt) + " AND m.id > " + quoteSQL(*afterMessageID) + "))" + result := syncResult{ + HelperVersion: helperVersion, ProtocolVersion: protocolVersion, ArchivePath: resolved, + Generation: generation, Account: data.account, Profiles: []profileRow{}, + Conversations: []conversationRow{}, Messages: []messageRow{}, FollowEdges: []followEdgeRow{}, } - messageQuery := `SELECT m.id, m.conversation_id, COALESCE(m.sender_profile_id, '') AS sender_profile_id, - COALESCE(m.text, '') AS text, m.created_at, COALESCE(m.direction, '') AS direction - FROM dm_messages m WHERE m.conversation_id IN (` + conversationFilter + `)` + cursorFilter + - " ORDER BY m.created_at, m.id LIMIT " + strconv.Itoa(*limit+1) - if err := sqliteJSON(ctx, resolved, messageQuery, &result.Messages); err != nil { - fail(err) + if *phase == "relationships" { + start := max(0, *edgeOffset) + if start > len(data.followEdges) { + start = len(data.followEdges) + } + end := min(start+*limit, len(data.followEdges)) + for end < len(data.followEdges) && end > start && + data.followEdges[end].ProfileID == data.followEdges[end-1].ProfileID { + end++ + } + result.FollowEdges = data.followEdges[start:end] + result.Profiles = profilesForEdges(result.FollowEdges, data.profiles) + result.HasMore = end < len(data.followEdges) + if result.HasMore { + result.NextPhase = "relationships" + result.NextEdgeOffset = end + } + } else { + filtered := make([]messageRow, 0, len(data.messages)) + for _, message := range data.messages { + if *afterCreatedAt == "" || message.CreatedAt > *afterCreatedAt || + (message.CreatedAt == *afterCreatedAt && message.ID > *afterMessageID) { + filtered = append(filtered, message) + } + } + moreMessages := len(filtered) > *limit + if moreMessages { + filtered = filtered[:*limit] + } + result.Messages = filtered + result.Conversations, result.Profiles = entitiesForMessages(filtered, data) + if len(filtered) > 0 { + last := filtered[len(filtered)-1] + result.NextCreatedAt = last.CreatedAt + result.NextMessageID = last.ID + } + result.HasMore = moreMessages || len(data.followEdges) > 0 + if moreMessages { + result.NextPhase = "messages" + } else if len(data.followEdges) > 0 { + result.NextPhase = "relationships" + } } - if len(result.Messages) > *limit { - result.Messages = result.Messages[:*limit] - result.HasMore = true + writeJSON(result) +} + +func entitiesForMessages(messages []messageRow, data archiveData) ([]conversationRow, []profileRow) { + conversationIDs := make(map[string]bool) + profileIDs := make(map[string]bool) + for _, message := range messages { + conversationIDs[message.ConversationID] = true + if message.Direction != "outbound" { + profileIDs[message.SenderProfileID] = true + } } - if len(result.Messages) > 0 { - last := result.Messages[len(result.Messages)-1] - result.NextCreatedAt = last.CreatedAt - result.NextMessageID = last.ID + conversations := make([]conversationRow, 0, len(conversationIDs)) + for id := range conversationIDs { + conversation := data.conversations[id] + conversations = append(conversations, conversation) + for _, participantID := range conversation.ParticipantIDs { + profileIDs[profileID(participantID)] = true + } } - if err := sqliteJSON(ctx, resolved, `SELECT direction, profile_id, - COALESCE(external_user_id, '') AS external_user_id, COALESCE(source, '') AS source, - COALESCE(first_seen_at, '') AS first_seen_at, COALESCE(last_seen_at, '') AS last_seen_at - FROM follow_edges WHERE account_id = `+accountID+` AND current = 1 ORDER BY profile_id, direction`, &result.FollowEdges); err != nil { - fail(err) + profiles := make([]profileRow, 0, len(profileIDs)) + for id := range profileIDs { + if profile, exists := data.profiles[id]; exists { + profiles = append(profiles, profile) + } } - writeJSON(result) + sort.Slice(conversations, func(i, j int) bool { return conversations[i].ID < conversations[j].ID }) + sort.Slice(profiles, func(i, j int) bool { return profiles[i].ID < profiles[j].ID }) + return conversations, profiles } -func selectAccount(ctx context.Context, dbPath, requested string) (accountRow, error) { - where := "" - if strings.TrimSpace(requested) != "" && requested != "default" { - value := strings.TrimPrefix(strings.TrimSpace(requested), "@") - where = " WHERE id = " + quoteSQL(value) + " OR handle = " + quoteSQL(value) +func profilesForEdges(edges []followEdgeRow, profiles map[string]profileRow) []profileRow { + seen := make(map[string]bool) + result := make([]profileRow, 0, len(edges)) + for _, edge := range edges { + if seen[edge.ProfileID] { + continue + } + seen[edge.ProfileID] = true + result = append(result, profiles[edge.ProfileID]) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + return result +} + +func archiveGeneration(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", err } - var rows []accountRow - err := sqliteJSON(ctx, dbPath, `SELECT id, COALESCE(name, '') AS name, COALESCE(handle, '') AS handle, - COALESCE(external_user_id, '') AS external_user_id FROM accounts`+where+` ORDER BY is_default DESC, created_at LIMIT 1`, &rows) + return fmt.Sprintf("%d:%d", info.Size(), info.ModTime().UnixNano()), nil +} + +func readArchive(path string) (archiveData, error) { + reader, err := zip.OpenReader(path) if err != nil { - return accountRow{}, err + return archiveData{}, fmt.Errorf("open X archive: %w", err) + } + defer reader.Close() + result := archiveData{ + profiles: make(map[string]profileRow), conversations: make(map[string]conversationRow), + } + accountEntry := findEntry(reader.File, `data/account.js`) + if accountEntry == nil { + return archiveData{}, errors.New("X archive is missing data/account.js") + } + if err := forEachRecord(accountEntry, func(record map[string]any) error { + account := asMap(record["account"]) + result.account = accountRow{ + ID: stringValue(account["accountId"]), ExternalUserID: stringValue(account["accountId"]), + Handle: stringValue(account["username"]), + Name: firstNonEmpty(stringValue(account["accountDisplayName"]), stringValue(account["name"]), stringValue(account["username"])), + } + return nil + }); err != nil { + return archiveData{}, err + } + if result.account.ID == "" { + return archiveData{}, errors.New("X archive account id is missing") } - if len(rows) == 0 { - return accountRow{}, fmt.Errorf("birdclaw account %q not found", requested) + mentionDirectory := readMentionDirectory(reader.File) + for _, file := range reader.File { + if dmEntryPattern.MatchString(normalizePath(file.Name)) { + if err := parseDMEntry(file, &result, mentionDirectory); err != nil { + return archiveData{}, err + } + } + } + for _, input := range []struct { + pattern *regexp.Regexp + direction string + key string + }{{followerEntryPattern, "followers", "follower"}, {followingEntryPattern, "following", "following"}} { + for _, file := range reader.File { + if input.pattern.MatchString(normalizePath(file.Name)) { + if err := parseFollowEntry(file, &result, mentionDirectory, input.direction, input.key); err != nil { + return archiveData{}, err + } + } + } } - return rows[0], nil + sort.Slice(result.messages, func(i, j int) bool { + if result.messages[i].CreatedAt == result.messages[j].CreatedAt { + return result.messages[i].ID < result.messages[j].ID + } + return result.messages[i].CreatedAt < result.messages[j].CreatedAt + }) + sort.Slice(result.followEdges, func(i, j int) bool { + if result.followEdges[i].ProfileID == result.followEdges[j].ProfileID { + return result.followEdges[i].Direction < result.followEdges[j].Direction + } + return result.followEdges[i].ProfileID < result.followEdges[j].ProfileID + }) + return result, nil } -func sqliteJSON(ctx context.Context, dbPath, query string, target any) error { - binary := strings.TrimSpace(os.Getenv("CUED_BIRDCLAW_SQLITE_BINARY")) - if binary == "" { - binary = "sqlite3" +func parseDMEntry(file *zip.File, result *archiveData, directory map[string]profileRow) error { + return forEachRecord(file, func(record map[string]any) error { + conversation := asMap(record["dmConversation"]) + conversationID := stringValue(conversation["conversationId"]) + if conversationID == "" { + return nil + } + participantSet := make(map[string]bool) + messages := asSlice(conversation["messages"]) + lastMessageAt := "" + for _, rawEvent := range messages { + messageCreate := asMap(asMap(rawEvent)["messageCreate"]) + if len(messageCreate) == 0 { + continue + } + senderID := stringValue(messageCreate["senderId"]) + recipientID := stringValue(messageCreate["recipientId"]) + if senderID != "" { + participantSet[senderID] = true + } + if recipientID != "" { + participantSet[recipientID] = true + } + messageID := firstNonEmpty(stringValue(messageCreate["id"]), conversationID+"-"+senderID+"-"+strconv.Itoa(len(result.messages))) + createdAt := parseTwitterDate(stringValue(messageCreate["createdAt"])) + direction := "inbound" + if senderID == result.account.ExternalUserID { + direction = "outbound" + } else if senderID != "" { + ensureProfile(result, directory, senderID) + } + result.messages = append(result.messages, messageRow{ + ID: messageID, ConversationID: conversationID, SenderProfileID: profileID(senderID), + Text: stringValue(messageCreate["text"]), CreatedAt: createdAt, Direction: direction, + }) + if createdAt > lastMessageAt { + lastMessageAt = createdAt + } + } + participants := make([]string, 0, len(participantSet)) + for userID := range participantSet { + if userID != "" && userID != result.account.ExternalUserID { + participants = append(participants, userID) + ensureProfile(result, directory, userID) + } + } + sort.Strings(participants) + if lastMessageAt != "" { + result.conversations[conversationID] = conversationRow{ + ID: conversationID, ParticipantIDs: participants, + Title: firstNonEmpty(stringValue(conversation["name"]), conversationTitle(participants, result.profiles)), + LastMessageAt: lastMessageAt, IsGroup: len(participants) > 1 || stringValue(conversation["name"]) != "", + } + } + return nil + }) +} + +func parseFollowEntry(file *zip.File, result *archiveData, directory map[string]profileRow, direction, key string) error { + return forEachRecord(file, func(record map[string]any) error { + item := asMap(record[key]) + userID := stringValue(item["accountId"]) + if userID == "" { + return nil + } + ensureProfile(result, directory, userID) + result.followEdges = append(result.followEdges, followEdgeRow{ + Direction: direction, ProfileID: profileID(userID), ExternalUserID: userID, Source: "archive", + }) + return nil + }) +} + +func readMentionDirectory(files []*zip.File) map[string]profileRow { + directory := make(map[string]profileRow) + for _, file := range files { + if !tweetEntryPattern.MatchString(normalizePath(file.Name)) { + continue + } + _ = forEachRecord(file, func(record map[string]any) error { + tweet := asMap(record["tweet"]) + entities := asMap(tweet["entities"]) + for _, rawMention := range asSlice(entities["user_mentions"]) { + mention := asMap(rawMention) + userID := firstNonEmpty(stringValue(mention["id_str"]), stringValue(mention["id"])) + if userID != "" { + handle := stringValue(mention["screen_name"]) + directory[userID] = profileRow{ + ID: profileID(userID), Handle: handle, + DisplayName: firstNonEmpty(stringValue(mention["name"]), handle, "X user "+userID), + } + } + } + return nil + }) + } + return directory +} + +func ensureProfile(result *archiveData, directory map[string]profileRow, userID string) { + id := profileID(userID) + if _, exists := result.profiles[id]; exists { + return + } + profile, exists := directory[userID] + if !exists { + profile = profileRow{ID: id, Handle: "id" + userID, DisplayName: "X user " + userID} } - cmd := exec.CommandContext(ctx, binary, "-json", dbPath, query) // #nosec G204 -- no shell is used. - raw, err := cmd.Output() + result.profiles[id] = profile +} + +func forEachRecord(file *zip.File, visit func(map[string]any) error) error { + reader, err := file.Open() if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return fmt.Errorf("birdclaw sqlite query: %s", strings.TrimSpace(string(exitErr.Stderr))) - } return err } - if strings.TrimSpace(string(raw)) == "" { - raw = []byte("[]") + defer reader.Close() + buffered := bufio.NewReader(reader) + if _, err := buffered.ReadString('='); err != nil { + return fmt.Errorf("read %s assignment: %w", file.Name, err) + } + decoder := json.NewDecoder(buffered) + token, err := decoder.Token() + if err != nil || token != json.Delim('[') { + return fmt.Errorf("read %s array: %w", file.Name, err) + } + for decoder.More() { + var record map[string]any + if err := decoder.Decode(&record); err != nil { + return fmt.Errorf("decode %s: %w", file.Name, err) + } + if err := visit(record); err != nil { + return err + } + } + return nil +} + +func findEntry(files []*zip.File, suffix string) *zip.File { + suffix = strings.ToLower(suffix) + for _, file := range files { + if strings.HasSuffix(strings.ToLower(normalizePath(file.Name)), suffix) { + return file + } } - return json.Unmarshal(raw, target) + return nil } -func resolveDBPath(value string) (string, error) { +func resolveArchivePath(value string) (string, error) { path := strings.TrimSpace(value) if path == "" { - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - path = filepath.Join(home, ".birdclaw", "birdclaw.sqlite") - } else if strings.HasPrefix(path, "~/") { + return "", errors.New("X archive path is required") + } + if strings.HasPrefix(path, "~/") { home, err := os.UserHomeDir() if err != nil { return "", err @@ -251,8 +497,60 @@ func resolveDBPath(value string) (string, error) { return filepath.Abs(path) } -func quoteSQL(value string) string { - return "'" + strings.ReplaceAll(value, "'", "''") + "'" +func parseTwitterDate(value string) string { + for _, layout := range []string{time.RFC3339Nano, "Mon Jan 02 15:04:05 -0700 2006"} { + if parsed, err := time.Parse(layout, value); err == nil { + return parsed.UTC().Format(time.RFC3339Nano) + } + } + return value +} + +func profileID(userID string) string { return "profile_user_" + userID } +func normalizePath(value string) string { return strings.ReplaceAll(value, `\`, "/") } + +func conversationTitle(participants []string, profiles map[string]profileRow) string { + names := make([]string, 0, len(participants)) + for _, userID := range participants { + names = append(names, profiles[profileID(userID)].DisplayName) + } + return strings.Join(names, ", ") +} + +func asMap(value any) map[string]any { + if result, ok := value.(map[string]any); ok { + return result + } + return map[string]any{} +} + +func asSlice(value any) []any { + if result, ok := value.([]any); ok { + return result + } + return nil +} + +func stringValue(value any) string { + switch typed := value.(type) { + case string: + return typed + case json.Number: + return typed.String() + case float64: + return strconv.FormatInt(int64(typed), 10) + default: + return "" + } +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" } func writeJSON(value any) { diff --git a/native/helpers/birdclaw-go/main_test.go b/native/helpers/birdclaw-go/main_test.go index ac6a92ff..559053da 100644 --- a/native/helpers/birdclaw-go/main_test.go +++ b/native/helpers/birdclaw-go/main_test.go @@ -1,39 +1,63 @@ package main import ( + "archive/zip" "os" "path/filepath" "testing" ) -func TestSelectAccountUsesConfiguredSQLiteBinary(t *testing.T) { - bin := filepath.Join(t.TempDir(), "sqlite3") - script := `#!/bin/sh -case "$*" in - *"FROM accounts"*) - printf '%s' '[{"id":"acct-1","name":"Sam","handle":"sam","external_user_id":"10"}]' - ;; - *) - echo "unexpected query" >&2 - exit 2 - ;; -esac -` - if err := os.WriteFile(bin, []byte(script), 0o700); err != nil { +func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "twitter-archive.zip") + file, err := os.Create(archivePath) + if err != nil { + t.Fatal(err) + } + writer := zip.NewWriter(file) + entries := map[string]string{ + "archive/data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, + "archive/data/tweets.js": `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"Ava Chen"}]}}}]`, + "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + "archive/data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]`, + "archive/data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, + } + for name, content := range entries { + entry, err := writer.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := entry.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { t.Fatal(err) } - t.Setenv("CUED_BIRDCLAW_SQLITE_BINARY", bin) - account, err := selectAccount(t.Context(), filepath.Join(t.TempDir(), "birdclaw.sqlite"), "@sam") + if err := file.Close(); err != nil { + t.Fatal(err) + } + + data, err := readArchive(archivePath) if err != nil { t.Fatal(err) } - if account.ID != "acct-1" || account.ExternalUserID != "10" { - t.Fatalf("account = %#v", account) + if data.account.ID != "10" || data.account.Handle != "sam" { + t.Fatalf("account = %#v", data.account) + } + if len(data.messages) != 1 || data.messages[0].Text != "hello" { + t.Fatalf("messages = %#v", data.messages) + } + profile := data.profiles["profile_user_20"] + if profile.Handle != "ava" || profile.DisplayName != "Ava Chen" { + t.Fatalf("profile = %#v", profile) + } + if len(data.followEdges) != 2 { + t.Fatalf("followEdges = %#v", data.followEdges) } } -func TestQuoteSQL(t *testing.T) { - if got := quoteSQL("sam'o"); got != "'sam''o'" { - t.Fatalf("quoteSQL = %q", got) +func TestParseTwitterDate(t *testing.T) { + if got := parseTwitterDate("Mon Feb 19 14:59:19 +0000 2024"); got != "2024-02-19T14:59:19Z" { + t.Fatalf("parseTwitterDate = %q", got) } } diff --git a/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift b/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift index f6b6200d..2efb8cf6 100644 --- a/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift +++ b/native/macos/CuedNative/Sources/Interface/Root/RootDisplayHelpers.swift @@ -242,7 +242,7 @@ func platformDescription(for configuration: PlatformConfig) -> String { case "signal": return "Syncs Signal messages after you link your device." case "x": - return "Imports X direct messages, profiles, and follow relationships from local Birdclaw data." + return "Imports direct messages and follow relationships from your local X archive." default: return "Syncs this source on this Mac." } diff --git a/scripts/build-cued-daemon-app.sh b/scripts/build-cued-daemon-app.sh index 606f12b6..d4871b2c 100644 --- a/scripts/build-cued-daemon-app.sh +++ b/scripts/build-cued-daemon-app.sh @@ -423,7 +423,7 @@ if [[ -f "\$SCRIPT_DIR/oauth/google-oauth-client.json" ]]; then fi export CUED_SLACK_HELPER_BINARY="\${CUED_SLACK_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-slack-helper}" export CUED_WHATSAPP_HELPER_BINARY="\${CUED_WHATSAPP_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-whatsapp-helper}" -export CUED_BIRDCLAW_HELPER_BINARY="\${CUED_BIRDCLAW_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-birdclaw-helper}" +export CUED_X_ARCHIVE_HELPER_BINARY="\${CUED_X_ARCHIVE_HELPER_BINARY:-\$SCRIPT_DIR/helpers/cued-birdclaw-helper}" export CUED_APP_VERSION="$APP_VERSION" export CUED_RELEASE_CHANNEL="$RELEASE_CHANNEL" exec "\$NODE_BIN" "\$RUNTIME_ROOT/dist/cli.js" "\$@" diff --git a/src/platforms/core/proofs.ts b/src/platforms/core/proofs.ts index 886933b9..2bcf7751 100644 --- a/src/platforms/core/proofs.ts +++ b/src/platforms/core/proofs.ts @@ -113,9 +113,9 @@ const PROOF_KIND_CONTRACTS: SyncProofKindContract[] = [ platform: "x", proofKind: "messages", scopeKind: "account", - completeMeans: "All X DM messages currently present in the local Birdclaw database were read.", - invalidatedBy: "A later Birdclaw import or live refresh adds messages to its local database.", - resumeCursorMeans: "The Birdclaw DM created-at and message-id boundary for the account.", + completeMeans: "All DMs and relationship edges in the selected X archive generation were read.", + invalidatedBy: "The selected X archive zip is replaced with a different generation.", + resumeCursorMeans: "The archive generation plus DM or relationship page boundary.", }, ]; diff --git a/src/platforms/core/state/local.ts b/src/platforms/core/state/local.ts index 52abe887..0be35352 100644 --- a/src/platforms/core/state/local.ts +++ b/src/platforms/core/state/local.ts @@ -115,7 +115,7 @@ export function buildLocalIntegrationStates(): ManagedIntegrationState[] { { platform: "x", accountKey: "default", - displayName: "X via Birdclaw", + displayName: "X Archive", authState: birdclaw.available ? "authorized" : birdclaw.helperPath @@ -125,11 +125,12 @@ export function buildLocalIntegrationStates(): ManagedIntegrationState[] { connectionKind: "local-cli", runtimeKind: "native", syncCapable: birdclaw.available, - importedFrom: "birdclaw", - artifactPaths: existsSync(birdclaw.databasePath) ? [birdclaw.databasePath] : [], + importedFrom: "x-archive", + artifactPaths: + birdclaw.archivePath && existsSync(birdclaw.archivePath) ? [birdclaw.archivePath] : [], metadata: { helperPath: birdclaw.helperPath, - databasePath: birdclaw.databasePath, + archivePath: birdclaw.archivePath, reason: birdclaw.reason, }, }, diff --git a/src/platforms/core/types.ts b/src/platforms/core/types.ts index 2e702139..f2bd7302 100644 --- a/src/platforms/core/types.ts +++ b/src/platforms/core/types.ts @@ -18,7 +18,7 @@ export const PLATFORM_PERMISSION_REQUIREMENT_VALUES = ["contacts", "full_disk_ac export type PlatformPermissionRequirement = (typeof PLATFORM_PERMISSION_REQUIREMENT_VALUES)[number]; export const PLATFORM_HELPER_REQUIREMENT_VALUES = [ - "birdclaw_helper", + "x_archive_helper", "signal_cli", "slack_helper", "whatsapp_helper", @@ -73,7 +73,7 @@ export const PLATFORM_DEFINITIONS = { supportsMultipleAccounts: true, requestableIntegration: true, requestableOrder: 2, - supportedHostOs: ["macos", "windows", "linux"], + supportedHostOs: ["macos"], onboardingVisible: true, permissionRequirements: [], helperRequirements: [], @@ -152,7 +152,7 @@ export const PLATFORM_DEFINITIONS = { supportedHostOs: ["macos", "windows", "linux"], onboardingVisible: true, permissionRequirements: [], - helperRequirements: ["birdclaw_helper"], + helperRequirements: ["x_archive_helper"], }, } as const satisfies Record; diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts index c2a96442..47c3acb8 100644 --- a/src/platforms/x/e2e.test.ts +++ b/src/platforms/x/e2e.test.ts @@ -1,14 +1,59 @@ import { execFileSync } from "node:child_process"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import Database from "better-sqlite3-multiple-ciphers"; import { afterEach, describe, expect, it } from "vitest"; import { CuedDatabase } from "../../db/database.js"; import { projectPendingRawEvents } from "../../runtime/projection/projector.js"; import { buildXSyncBundle } from "./sync/bundle.js"; -describe("X Birdclaw end to end", () => { +function writeArchive(root: string, input: { name: string; follower: boolean }): string { + const source = join(root, "archive-source"); + rmSync(source, { recursive: true, force: true }); + mkdirSync(join(source, "data"), { recursive: true }); + writeFileSync( + join(source, "data/account.js"), + `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, + ); + writeFileSync( + join(source, "data/tweets.js"), + `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"${input.name}"}]}}}]`, + ); + writeFileSync( + join(source, "data/direct-messages.js"), + `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + ); + writeFileSync( + join(source, "data/follower.js"), + input.follower + ? `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]` + : `window.YTD.follower.part0 = []`, + ); + writeFileSync( + join(source, "data/following.js"), + `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, + ); + const archivePath = join(root, "twitter-archive.zip"); + rmSync(archivePath, { force: true }); + execFileSync("zip", ["-q", "-r", archivePath, "data"], { cwd: source }); + return archivePath; +} + +async function projectArchiveCycle(db: CuedDatabase) { + let cursor: unknown = null; + let totalApplied = 0; + for (let page = 0; page < 20; page += 1) { + const bundle = await buildXSyncBundle({}, { sourceCursor: cursor }); + db.upsertSourceAccounts(bundle.sourceAccounts ?? []); + db.insertRawEvents(bundle.rawEvents); + totalApplied += projectPendingRawEvents(db).appliedRawEvents; + cursor = bundle.sourceCursor; + if (!bundle.hasMore) return { totalApplied, bundle }; + } + throw new Error("X archive pagination did not complete"); +} + +describe("X archive end to end", () => { const tempDirs: string[] = []; afterEach(() => { @@ -18,77 +63,26 @@ describe("X Birdclaw end to end", () => { } }); - it("reads Birdclaw SQLite through the Go helper and projects searchable Cued data", async () => { - const temp = mkdtempSync(join(tmpdir(), "cued-birdclaw-e2e-")); + it("imports an X archive through the compiled Go helper and refreshes contact state", async () => { + const temp = mkdtempSync(join(tmpdir(), "cued-x-archive-e2e-")); tempDirs.push(temp); - const birdclawPath = join(temp, "birdclaw.sqlite"); + const archivePath = writeArchive(temp, { name: "Ava Chen", follower: true }); const cuedPath = join(temp, "cued.sqlite"); const helperPath = join(temp, "cued-birdclaw-helper"); - const helperSource = resolve("native/helpers/birdclaw-go"); execFileSync("go", ["build", "-o", helperPath, "."], { - cwd: helperSource, + cwd: resolve("native/helpers/birdclaw-go"), env: { ...process.env, GOWORK: "off" }, }); - const birdclaw = new Database(birdclawPath); - birdclaw.exec(` - CREATE TABLE accounts ( - id TEXT PRIMARY KEY, name TEXT NOT NULL, handle TEXT NOT NULL, - external_user_id TEXT, is_default INTEGER NOT NULL, created_at TEXT NOT NULL - ); - CREATE TABLE profiles ( - id TEXT PRIMARY KEY, handle TEXT NOT NULL, display_name TEXT NOT NULL, bio TEXT NOT NULL, - followers_count INTEGER NOT NULL, following_count INTEGER NOT NULL, avatar_url TEXT, - location TEXT, url TEXT, verified_type TEXT - ); - CREATE TABLE dm_conversations ( - id TEXT PRIMARY KEY, account_id TEXT NOT NULL, participant_profile_id TEXT NOT NULL, - title TEXT NOT NULL, last_message_at TEXT NOT NULL - ); - CREATE TABLE dm_messages ( - id TEXT PRIMARY KEY, conversation_id TEXT NOT NULL, sender_profile_id TEXT NOT NULL, - text TEXT NOT NULL, created_at TEXT NOT NULL, direction TEXT NOT NULL - ); - CREATE TABLE follow_edges ( - account_id TEXT NOT NULL, direction TEXT NOT NULL, profile_id TEXT NOT NULL, - external_user_id TEXT NOT NULL, source TEXT NOT NULL, current INTEGER NOT NULL, - first_seen_at TEXT NOT NULL, last_seen_at TEXT NOT NULL - ); - INSERT INTO accounts VALUES ('acct-1', 'Sam', 'sam', '10', 1, '2026-01-01'); - INSERT INTO profiles VALUES ( - 'profile-20', 'ava', 'Ava Chen', 'Robotics', 100, 80, - 'https://pbs.twimg.com/ava.jpg', 'New York', 'https://ava.example', 'blue' - ); - INSERT INTO dm_conversations VALUES ( - '10-20', 'acct-1', 'profile-20', 'Ava Chen', '2026-08-03T12:00:00.000Z' - ); - INSERT INTO dm_messages VALUES ( - 'message-1', '10-20', 'profile-20', 'Hello from real SQLite', - '2026-08-03T12:00:00.000Z', 'inbound' - ); - INSERT INTO follow_edges VALUES - ('acct-1', 'followers', 'profile-20', '20', 'archive', 1, '2026-01-01', '2026-08-01'), - ('acct-1', 'following', 'profile-20', '20', 'archive', 1, '2026-01-01', '2026-08-01'); - `); - birdclaw.close(); - - const previousHelper = process.env.CUED_BIRDCLAW_HELPER_BINARY; - const previousDatabase = process.env.CUED_BIRDCLAW_DB_PATH; - process.env.CUED_BIRDCLAW_HELPER_BINARY = helperPath; - process.env.CUED_BIRDCLAW_DB_PATH = birdclawPath; - const bundle = await buildXSyncBundle(); - if (previousHelper === undefined) delete process.env.CUED_BIRDCLAW_HELPER_BINARY; - else process.env.CUED_BIRDCLAW_HELPER_BINARY = previousHelper; - if (previousDatabase === undefined) delete process.env.CUED_BIRDCLAW_DB_PATH; - else process.env.CUED_BIRDCLAW_DB_PATH = previousDatabase; + const previousHelper = process.env.CUED_X_ARCHIVE_HELPER_BINARY; + const previousArchive = process.env.CUED_X_ARCHIVE_PATH; + process.env.CUED_X_ARCHIVE_HELPER_BINARY = helperPath; + process.env.CUED_X_ARCHIVE_PATH = archivePath; const cued = new CuedDatabase(cuedPath); try { cued.migrate(); - cued.upsertSourceAccounts(bundle.sourceAccounts ?? []); - cued.insertRawEvents(bundle.rawEvents); - const projection = projectPendingRawEvents(cued); - expect(projection.appliedRawEvents).toBe(3); - + const first = await projectArchiveCycle(cued); + expect(first.totalApplied).toBeGreaterThanOrEqual(4); const sqlite = ( cued as unknown as { sqlite: { @@ -96,19 +90,38 @@ describe("X Birdclaw end to end", () => { }; } ).sqlite; - const contact = sqlite - .prepare(` - SELECT c.name, cs.metadata_json - FROM contacts c JOIN contact_sources cs ON cs.contact_id = c.id - WHERE cs.platform = 'x' AND cs.source_entity_key = 'x:user:20' - `) - .get() as { name: string; metadata_json: string }; - expect(contact.name).toBe("Ava Chen"); - expect(JSON.parse(contact.metadata_json)).toMatchObject({ mutual: true, followsYou: true }); + const readContact = () => + sqlite + .prepare(` + SELECT c.name, cs.metadata_json + FROM contacts c JOIN contact_sources cs ON cs.contact_id = c.id + WHERE cs.platform = 'x' AND cs.source_entity_key = 'x:user:20' + `) + .get() as { name: string; metadata_json: string }; + expect(readContact().name).toBe("Ava Chen"); + expect(JSON.parse(readContact().metadata_json)).toMatchObject({ + mutual: true, + followsYou: true, + youFollow: true, + }); expect( sqlite.prepare("SELECT content FROM messages WHERE platform = 'x'").get(), - ).toMatchObject({ content: "Hello from real SQLite" }); + ).toMatchObject({ content: "Hello from the X archive" }); + + writeArchive(temp, { name: "Ava Patel", follower: false }); + const second = await projectArchiveCycle(cued); + expect(second.totalApplied).toBeGreaterThan(0); + expect(readContact().name).toBe("Ava Patel"); + expect(JSON.parse(readContact().metadata_json)).toMatchObject({ + mutual: false, + followsYou: false, + youFollow: true, + }); } finally { + if (previousHelper === undefined) delete process.env.CUED_X_ARCHIVE_HELPER_BINARY; + else process.env.CUED_X_ARCHIVE_HELPER_BINARY = previousHelper; + if (previousArchive === undefined) delete process.env.CUED_X_ARCHIVE_PATH; + else process.env.CUED_X_ARCHIVE_PATH = previousArchive; cued.close(); } }, 30_000); diff --git a/src/platforms/x/helper/binary.ts b/src/platforms/x/helper/binary.ts index 10b445a3..7a96147f 100644 --- a/src/platforms/x/helper/binary.ts +++ b/src/platforms/x/helper/binary.ts @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, readdirSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -11,8 +11,17 @@ function repoRoot(): string { return resolve(dirname(fileURLToPath(import.meta.url)), "../../../.."); } -export function resolveBirdclawDatabasePath(env: NodeJS.ProcessEnv = process.env): string { - return env.CUED_BIRDCLAW_DB_PATH?.trim() || join(homedir(), ".birdclaw", "birdclaw.sqlite"); +export function resolveXArchivePath(env: NodeJS.ProcessEnv = process.env): string | null { + const configured = env.CUED_X_ARCHIVE_PATH?.trim(); + if (configured) return configured; + const downloads = join(homedir(), "Downloads"); + if (!existsSync(downloads)) return null; + return ( + readdirSync(downloads) + .filter((name) => /(?:twitter|x).*(?:archive).*\.zip$/i.test(name)) + .map((name) => join(downloads, name)) + .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs)[0] ?? null + ); } export function birdclawHelperCandidates(root = repoRoot()): string[] { @@ -27,7 +36,7 @@ export function birdclawHelperCandidates(root = repoRoot()): string[] { } export function resolveBirdclawHelperBinary( - envValue = process.env.CUED_BIRDCLAW_HELPER_BINARY, + envValue = process.env.CUED_X_ARCHIVE_HELPER_BINARY, ): string | null { return ( envValue?.trim() || @@ -38,39 +47,42 @@ export function resolveBirdclawHelperBinary( export function inspectBirdclawHelper(): { helperPath: string | null; - databasePath: string; + archivePath: string | null; available: boolean; reason: string | null; } { const helperPath = resolveBirdclawHelperBinary(); - const databasePath = resolveBirdclawDatabasePath(); + const archivePath = resolveXArchivePath(); if (!helperPath) { return { helperPath: null, - databasePath, + archivePath, available: false, - reason: "Birdclaw helper is missing", + reason: "X archive helper is missing", }; } - if (!existsSync(databasePath)) { - return { helperPath, databasePath, available: false, reason: "Birdclaw database is missing" }; + if (!archivePath || !existsSync(archivePath)) { + return { helperPath, archivePath, available: false, reason: "X archive is missing" }; } try { - const output = execFileSync(helperPath, ["version"], { encoding: "utf8" }); - const parsed = JSON.parse(output) as { protocolVersion?: unknown }; - if (parsed.protocolVersion !== SUPPORTED_PROTOCOL_VERSION) { + const output = execFileSync(helperPath, ["status", "--archive", archivePath], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + const parsed = JSON.parse(output) as { protocolVersion?: unknown; available?: unknown }; + if (parsed.protocolVersion !== SUPPORTED_PROTOCOL_VERSION || parsed.available !== true) { return { helperPath, - databasePath, + archivePath, available: false, - reason: `Unsupported Birdclaw helper protocol: ${String(parsed.protocolVersion)}`, + reason: `Unsupported X archive helper protocol: ${String(parsed.protocolVersion)}`, }; } - return { helperPath, databasePath, available: true, reason: null }; + return { helperPath, archivePath, available: true, reason: null }; } catch (error) { return { helperPath, - databasePath, + archivePath, available: false, reason: error instanceof Error ? error.message : String(error), }; diff --git a/src/platforms/x/sync/bundle.test.ts b/src/platforms/x/sync/bundle.test.ts index a73da247..4c188c61 100644 --- a/src/platforms/x/sync/bundle.test.ts +++ b/src/platforms/x/sync/bundle.test.ts @@ -2,10 +2,11 @@ import { describe, expect, it } from "vitest"; import { type BirdclawSnapshot, buildXSyncBundle } from "./bundle.js"; const snapshot: BirdclawSnapshot = { - account: { id: "acct-1", name: "Sam", handle: "sam", external_user_id: "10" }, + generation: "archive-v1", + account: { id: "10", name: "Sam", handle: "sam", external_user_id: "10" }, profiles: [ { - id: "profile-20", + id: "profile_user_20", handle: "ava", display_name: "Ava", bio: "Robotics", @@ -20,16 +21,17 @@ const snapshot: BirdclawSnapshot = { conversations: [ { id: "10-20", - participant_profile_id: "profile-20", + participant_ids: ["20"], title: "Ava", last_message_at: "2026-08-03T12:00:00.000Z", + is_group: false, }, ], messages: [ { id: "message-1", conversation_id: "10-20", - sender_profile_id: "profile-20", + sender_profile_id: "profile_user_20", text: "Hello from Birdclaw", created_at: "2026-08-03T12:00:00.000Z", direction: "inbound", @@ -38,7 +40,7 @@ const snapshot: BirdclawSnapshot = { followEdges: [ { direction: "followers", - profile_id: "profile-20", + profile_id: "profile_user_20", external_user_id: "20", source: "archive", first_seen_at: "2026-01-01T00:00:00.000Z", @@ -46,7 +48,7 @@ const snapshot: BirdclawSnapshot = { }, { direction: "following", - profile_id: "profile-20", + profile_id: "profile_user_20", external_user_id: "20", source: "archive", first_seen_at: "2026-01-01T00:00:00.000Z", @@ -63,7 +65,7 @@ describe("Birdclaw X sync bundle", () => { const bundle = await buildXSyncBundle({ accountKey: "default" }, { snapshot }); expect(bundle.sourceAccounts).toEqual([ - { platform: "x", accountKey: "acct-1", displayName: "@sam" }, + { platform: "x", accountKey: "10", displayName: "@sam" }, ]); expect(bundle.rawEvents.map((event) => event.entityKind)).toEqual([ "contact", @@ -77,6 +79,7 @@ describe("Birdclaw X sync bundle", () => { { type: "x_handle", value: "ava", deterministic: true }, ]), sourceMetadata: { + archiveGeneration: "archive-v1", followsYou: true, youFollow: true, mutual: true, @@ -89,7 +92,8 @@ describe("Birdclaw X sync bundle", () => { isFromMe: false, }); expect(bundle.proofs?.[0]?.coverage).toMatchObject({ - source: "birdclaw.sqlite", + source: "x_archive", + generation: "archive-v1", relationshipEdges: 2, }); }); @@ -97,13 +101,67 @@ describe("Birdclaw X sync bundle", () => { it("continues from the helper cursor when another DM page remains", async () => { const bundle = await buildXSyncBundle( {}, - { snapshot: { ...snapshot, hasMore: true }, sourceCursor: { createdAt: null } }, + { + snapshot: { ...snapshot, hasMore: true, nextPhase: "relationships", nextEdgeOffset: 10 }, + sourceCursor: { createdAt: null }, + }, ); expect(bundle.hasMore).toBe(true); expect(bundle.sourceCursor).toEqual({ + generation: "archive-v1", + phase: "relationships", createdAt: "2026-08-03T12:00:00.000Z", messageId: "message-1", + edgeOffset: 10, }); expect(bundle.proofs?.[0]?.status).toBe("running"); }); + + it("keeps outbound and group participants on real X user identities", async () => { + const bundle = await buildXSyncBundle( + {}, + { + snapshot: { + ...snapshot, + profiles: [ + ...snapshot.profiles, + { ...snapshot.profiles[0]!, id: "profile_user_30", handle: "lee", display_name: "Lee" }, + ], + conversations: [ + { + id: "group-1", + participant_ids: ["20", "30"], + title: "Builders", + last_message_at: "2026-08-03T13:00:00.000Z", + is_group: true, + }, + ], + messages: [ + { + id: "message-2", + conversation_id: "group-1", + sender_profile_id: "profile_user_10", + text: "Sent by me", + created_at: "2026-08-03T13:00:00.000Z", + direction: "outbound", + }, + ], + followEdges: [], + }, + }, + ); + const conversation = bundle.rawEvents.find((event) => event.entityKind === "conversation"); + expect(conversation?.payload).toMatchObject({ + conversationType: "group", + participants: [ + { sourceEntityKey: "x:user:10", isSelf: true }, + { sourceEntityKey: "x:user:20" }, + { sourceEntityKey: "x:user:30" }, + ], + }); + expect(bundle.rawEvents.at(-1)?.payload).toMatchObject({ + senderSourceKey: "x:user:10", + isFromMe: true, + }); + }); }); diff --git a/src/platforms/x/sync/bundle.ts b/src/platforms/x/sync/bundle.ts index 68d4ef36..beec9979 100644 --- a/src/platforms/x/sync/bundle.ts +++ b/src/platforms/x/sync/bundle.ts @@ -8,11 +8,12 @@ import type { ProviderRawEventInput, } from "../../../core/types/provider.js"; import type { SyncBundle } from "../../core/sync.js"; -import { resolveBirdclawDatabasePath, resolveBirdclawHelperBinary } from "../helper/binary.js"; +import { resolveBirdclawHelperBinary, resolveXArchivePath } from "../helper/binary.js"; const execFileAsync = promisify(execFile); export interface BirdclawSnapshot { + generation: string; account: { id: string; name: string; handle: string; external_user_id: string }; profiles: Array<{ id: string; @@ -28,9 +29,10 @@ export interface BirdclawSnapshot { }>; conversations: Array<{ id: string; - participant_profile_id: string; + participant_ids: string[]; title: string; last_message_at: string; + is_group: boolean; }>; messages: Array<{ id: string; @@ -49,11 +51,19 @@ export interface BirdclawSnapshot { last_seen_at: string; }>; hasMore: boolean; + nextPhase?: string; + nextEdgeOffset?: number; nextCreatedAt?: string; nextMessageId?: string; } -type BirdclawCursor = { createdAt: string | null; messageId: string | null }; +type BirdclawCursor = { + generation: string | null; + phase: "messages" | "relationships"; + createdAt: string | null; + messageId: string | null; + edgeOffset: number; +}; function stableId(seed: string): string { return createHash("sha256").update(seed).digest("hex"); @@ -62,18 +72,29 @@ function stableId(seed: string): string { function parseCursor(value: unknown): BirdclawCursor { const cursor = value && typeof value === "object" ? (value as Record) : {}; return { + generation: typeof cursor.generation === "string" ? cursor.generation : null, + phase: cursor.phase === "relationships" ? "relationships" : "messages", createdAt: typeof cursor.createdAt === "string" ? cursor.createdAt : null, messageId: typeof cursor.messageId === "string" ? cursor.messageId : null, + edgeOffset: typeof cursor.edgeOffset === "number" ? cursor.edgeOffset : 0, }; } -async function readBirdclawSnapshot( - accountKey: string, - cursor: BirdclawCursor, -): Promise { +async function readBirdclawSnapshot(cursor: BirdclawCursor): Promise { const helper = resolveBirdclawHelperBinary(); - if (!helper) throw new Error("Birdclaw helper is not installed"); - const args = ["sync", "--db", resolveBirdclawDatabasePath(), "--account", accountKey]; + const archivePath = resolveXArchivePath(); + if (!helper) throw new Error("X archive helper is not installed"); + if (!archivePath) throw new Error("X archive was not found; set CUED_X_ARCHIVE_PATH"); + const args = [ + "sync", + "--archive", + archivePath, + "--phase", + cursor.phase, + "--edge-offset", + String(cursor.edgeOffset), + ]; + if (cursor.generation) args.push("--generation", cursor.generation); if (cursor.createdAt) { args.push("--after-created-at", cursor.createdAt, "--after-message-id", cursor.messageId ?? ""); } @@ -88,6 +109,10 @@ function sourceKey(id: string): string { return `x:user:${id}`; } +function externalUserId(profileId: string): string | null { + return profileId.startsWith("profile_user_") ? profileId.slice("profile_user_".length) : null; +} + export function buildBirdclawRawEvents(input: { accountKey: string; snapshot: BirdclawSnapshot; @@ -95,21 +120,38 @@ export function buildBirdclawRawEvents(input: { }): SyncBundle["rawEvents"] { const { snapshot } = input; const relationshipByProfile = new Map>(); - const externalIdByProfile = new Map(); - for (const edge of snapshot.followEdges) { + for (const edge of snapshot.followEdges ?? []) { const directions = relationshipByProfile.get(edge.profile_id) ?? new Set(); directions.add(edge.direction); relationshipByProfile.set(edge.profile_id, directions); - if (edge.external_user_id) externalIdByProfile.set(edge.profile_id, edge.external_user_id); } - const profileSourceKeys = new Map(); const rawEvents: SyncBundle["rawEvents"] = []; - for (const profile of snapshot.profiles) { - const externalId = externalIdByProfile.get(profile.id) || profile.id; + for (const profile of snapshot.profiles ?? []) { + const externalId = externalUserId(profile.id); + if (!externalId) continue; const contactSourceKey = sourceKey(externalId); - profileSourceKeys.set(profile.id, contactSourceKey); const directions = relationshipByProfile.get(profile.id) ?? new Set(); - const id = stableId(`x:contact:${input.accountKey}:${externalId}`); + const sourceMetadata = { + archiveGeneration: snapshot.generation, + bio: profile.bio, + location: profile.location, + url: profile.url, + verifiedType: profile.verified_type, + followersCount: profile.followers_count, + followingCount: profile.following_count, + followsYou: directions.has("followers"), + youFollow: directions.has("following"), + mutual: directions.has("followers") && directions.has("following"), + }; + const eventVersion = stableId( + JSON.stringify({ + displayName: profile.display_name, + handle: profile.handle, + avatarUrl: profile.avatar_url, + sourceMetadata, + }), + ); + const id = stableId(`x:contact:${input.accountKey}:${externalId}:${eventVersion}`); rawEvents.push({ id, platform: "x", @@ -143,25 +185,16 @@ export function buildBirdclawRawEvents(input: { ], sourceProfileUrl: profile.handle ? `https://x.com/${profile.handle}` : null, sourceMetadata: { - birdclawProfileId: profile.id, - bio: profile.bio, - location: profile.location, - url: profile.url, - verifiedType: profile.verified_type, - followersCount: profile.followers_count, - followingCount: profile.following_count, - followsYou: directions.has("followers"), - youFollow: directions.has("following"), - mutual: directions.has("followers") && directions.has("following"), + ...sourceMetadata, }, } satisfies ContactObservationPayload, } satisfies ProviderRawEventInput); } const selfKey = sourceKey(snapshot.account.external_user_id || snapshot.account.id); - for (const conversation of snapshot.conversations) { + for (const conversation of snapshot.conversations ?? []) { const conversationId = stableId(`x:conversation:${input.accountKey}:${conversation.id}`); - const remoteKey = profileSourceKeys.get(conversation.participant_profile_id); + const remoteKeys = conversation.participant_ids.map(sourceKey); rawEvents.push({ id: conversationId, platform: "x", @@ -173,16 +206,16 @@ export function buildBirdclawRawEvents(input: { dedupeKey: conversationId, payload: { sourceConversationKey: `x:conversation:${conversation.id}`, - conversationType: "dm", + conversationType: conversation.is_group ? "group" : "dm", displayName: conversation.title || "X conversation", participants: [ { sourceEntityKey: selfKey, isSelf: true }, - ...(remoteKey ? [{ sourceEntityKey: remoteKey }] : []), + ...remoteKeys.map((participantKey) => ({ sourceEntityKey: participantKey })), ], } satisfies ConversationObservationPayload, }); } - for (const message of snapshot.messages) { + for (const message of snapshot.messages ?? []) { const sentAt = Date.parse(message.created_at); const id = stableId(`x:message:${input.accountKey}:${message.id}`); rawEvents.push({ @@ -202,7 +235,9 @@ export function buildBirdclawRawEvents(input: { senderSourceKey: message.direction === "outbound" ? selfKey - : (profileSourceKeys.get(message.sender_profile_id) ?? null), + : externalUserId(message.sender_profile_id) + ? sourceKey(externalUserId(message.sender_profile_id)!) + : null, sentAt: Number.isFinite(sentAt) ? sentAt : input.observedAt, content: message.text, isFromMe: message.direction === "outbound", @@ -213,19 +248,29 @@ export function buildBirdclawRawEvents(input: { } export async function buildXSyncBundle( - input: { accountKey?: string } = {}, + _input: { accountKey?: string } = {}, options: { snapshot?: BirdclawSnapshot; sourceCursor?: unknown } = {}, ): Promise { - const requestedAccountKey = input.accountKey ?? process.env.CUED_ACCOUNT_KEY ?? "default"; const cursor = parseCursor(options.sourceCursor); - const snapshot = options.snapshot ?? (await readBirdclawSnapshot(requestedAccountKey, cursor)); + const snapshot = options.snapshot ?? (await readBirdclawSnapshot(cursor)); const accountKey = snapshot.account.id; const observedAt = Date.now(); const rawEvents = buildBirdclawRawEvents({ accountKey, snapshot, observedAt }); - const sourceCursor: BirdclawCursor = { - createdAt: snapshot.nextCreatedAt ?? cursor.createdAt, - messageId: snapshot.nextMessageId ?? cursor.messageId, - }; + const sourceCursor: BirdclawCursor = snapshot.hasMore + ? { + generation: snapshot.generation, + phase: snapshot.nextPhase === "relationships" ? "relationships" : "messages", + createdAt: snapshot.nextCreatedAt ?? cursor.createdAt, + messageId: snapshot.nextMessageId ?? cursor.messageId, + edgeOffset: snapshot.nextEdgeOffset ?? 0, + } + : { + generation: snapshot.generation, + phase: "messages", + createdAt: null, + messageId: null, + edgeOffset: 0, + }; return { sourceAccounts: [ { @@ -238,23 +283,24 @@ export async function buildXSyncBundle( ], rawEvents, sourceCursor, - syncMode: cursor.createdAt ? "incremental" : "full", + syncMode: "full", hasMore: snapshot.hasMore, continuation: snapshot.hasMore - ? { reason: "account_pagination", detail: "Birdclaw DM page remains" } + ? { reason: "account_pagination", detail: "X archive page remains" } : undefined, proofs: [ { - scope: { kind: "account", key: "birdclaw_local_archive" }, + scope: { kind: "account", key: "x_archive" }, proofKind: "messages", status: snapshot.hasMore ? "running" : "complete", observedAt, resumeCursor: snapshot.hasMore ? sourceCursor : null, coverage: { - source: "birdclaw.sqlite", - relationshipEdges: snapshot.followEdges.length, + source: "x_archive", + generation: snapshot.generation, + relationshipEdges: snapshot.followEdges?.length ?? 0, }, - stats: { messageCount: snapshot.messages.length, rawEventCount: rawEvents.length }, + stats: { messageCount: snapshot.messages?.length ?? 0, rawEventCount: rawEvents.length }, }, ], }; From bf011728fe252de118170086a61e1e4fc6bba5be Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:29:31 -0400 Subject: [PATCH 03/10] Discover standard X archive names --- docs/integration-policy.md | 2 +- src/platforms/x/helper/binary.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integration-policy.md b/docs/integration-policy.md index c2b3a5d8..fcf12301 100644 --- a/docs/integration-policy.md +++ b/docs/integration-policy.md @@ -39,7 +39,7 @@ Shared or managed app credentials must be injected at release time or supplied t X archive support ports only the useful normalization behavior from [Birdclaw](https://github.com/steipete/birdclaw) and the small Go-adapter pattern from [Clawdex](https://github.com/openclaw/clawdex). The bundled helper reads an X archive zip directly; Cued does not install, invoke, or read the database of either project. -Put the exported archive in `~/Downloads` with `twitter` or `x` and `archive` in the zip filename, or set `CUED_X_ARCHIVE_PATH`. Cued imports DM contacts, conversations, messages, and follower/following relationships into `~/.cued/local.db`; current mutual state and its archive generation live in `contact_sources.metadata_json`. Sync is manual because the archive is a point-in-time export: run `cued sync run x` after replacing it. +Put the exported `twitter*.zip` or `x*archive*.zip` in `~/Downloads`, or set `CUED_X_ARCHIVE_PATH`. Cued imports DM contacts, conversations, messages, and follower/following relationships into `~/.cued/local.db`; current mutual state and its archive generation live in `contact_sources.metadata_json`. Sync is manual because the archive is a point-in-time export: run `cued sync run x` after replacing it. ## Telegram diff --git a/src/platforms/x/helper/binary.ts b/src/platforms/x/helper/binary.ts index 7a96147f..a7651e3b 100644 --- a/src/platforms/x/helper/binary.ts +++ b/src/platforms/x/helper/binary.ts @@ -18,7 +18,7 @@ export function resolveXArchivePath(env: NodeJS.ProcessEnv = process.env): strin if (!existsSync(downloads)) return null; return ( readdirSync(downloads) - .filter((name) => /(?:twitter|x).*(?:archive).*\.zip$/i.test(name)) + .filter((name) => /^(?:twitter.*|x.*archive).*\.zip$/i.test(name)) .map((name) => join(downloads, name)) .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs)[0] ?? null ); From b269863da7801217eab9e2d38fca5342a485cd0d Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:42:37 -0400 Subject: [PATCH 04/10] fix(x): own archive lifecycle in Cued --- native/helpers/birdclaw-go/main.go | 109 ++++++++++++++---------- native/helpers/birdclaw-go/main_test.go | 41 ++++++++- src/core/types/provider.ts | 6 ++ src/platforms/core/types.ts | 4 +- src/platforms/x/e2e.test.ts | 74 ++++++++++++++-- src/platforms/x/sync/bundle.test.ts | 42 +++++---- src/platforms/x/sync/bundle.ts | 56 ++++++++---- src/runtime/projection/events.ts | 5 +- src/runtime/projection/projector.ts | 85 +++++++++++++++++- 9 files changed, 329 insertions(+), 93 deletions(-) diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index cf2c594c..dda3b112 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -76,20 +76,21 @@ type followEdgeRow struct { } type syncResult struct { - HelperVersion string `json:"helperVersion"` - ProtocolVersion int `json:"protocolVersion"` - ArchivePath string `json:"archivePath"` - Generation string `json:"generation"` - NextPhase string `json:"nextPhase,omitempty"` - NextEdgeOffset int `json:"nextEdgeOffset,omitempty"` - Account accountRow `json:"account"` - Profiles []profileRow `json:"profiles"` - Conversations []conversationRow `json:"conversations"` - Messages []messageRow `json:"messages"` - FollowEdges []followEdgeRow `json:"followEdges"` - HasMore bool `json:"hasMore"` - NextCreatedAt string `json:"nextCreatedAt,omitempty"` - NextMessageID string `json:"nextMessageId,omitempty"` + HelperVersion string `json:"helperVersion"` + ProtocolVersion int `json:"protocolVersion"` + ArchivePath string `json:"archivePath"` + Generation string `json:"generation"` + NextPhase string `json:"nextPhase,omitempty"` + NextEdgeOffset int `json:"nextEdgeOffset,omitempty"` + NextMessageOffset int `json:"nextMessageOffset,omitempty"` + TotalMessageCount int `json:"totalMessageCount"` + TotalRelationshipCount int `json:"totalRelationshipCount"` + Account accountRow `json:"account"` + Profiles []profileRow `json:"profiles"` + Conversations []conversationRow `json:"conversations"` + Messages []messageRow `json:"messages"` + FollowEdges []followEdgeRow `json:"followEdges"` + HasMore bool `json:"hasMore"` } type archiveData struct { @@ -98,6 +99,7 @@ type archiveData struct { conversations map[string]conversationRow messages []messageRow followEdges []followEdgeRow + totalMessages int } func main() { @@ -143,8 +145,7 @@ func runStatus(args []string) { func runSync(args []string) { flags := flag.NewFlagSet("sync", flag.ContinueOnError) archivePath := flags.String("archive", "", "X archive zip path") - afterCreatedAt := flags.String("after-created-at", "", "exclusive message timestamp cursor") - afterMessageID := flags.String("after-message-id", "", "exclusive message id cursor") + messageOffset := flags.Int("message-offset", 0, "message cursor offset") expectedGeneration := flags.String("generation", "", "archive generation for the cursor") phase := flags.String("phase", "messages", "messages or relationships") edgeOffset := flags.Int("edge-offset", 0, "relationship cursor offset") @@ -164,9 +165,9 @@ func runSync(args []string) { fail(err) } if *expectedGeneration != "" && *expectedGeneration != generation { - *afterCreatedAt, *afterMessageID, *phase, *edgeOffset = "", "", "messages", 0 + *messageOffset, *phase, *edgeOffset = 0, "messages", 0 } - data, err := readArchive(resolved) + data, err := readArchivePage(resolved, *messageOffset, *limit) if err != nil { fail(err) } @@ -174,6 +175,7 @@ func runSync(args []string) { HelperVersion: helperVersion, ProtocolVersion: protocolVersion, ArchivePath: resolved, Generation: generation, Account: data.account, Profiles: []profileRow{}, Conversations: []conversationRow{}, Messages: []messageRow{}, FollowEdges: []followEdgeRow{}, + TotalMessageCount: data.totalMessages, TotalRelationshipCount: len(data.followEdges), } if *phase == "relationships" { start := max(0, *edgeOffset) @@ -193,24 +195,11 @@ func runSync(args []string) { result.NextEdgeOffset = end } } else { - filtered := make([]messageRow, 0, len(data.messages)) - for _, message := range data.messages { - if *afterCreatedAt == "" || message.CreatedAt > *afterCreatedAt || - (message.CreatedAt == *afterCreatedAt && message.ID > *afterMessageID) { - filtered = append(filtered, message) - } - } - moreMessages := len(filtered) > *limit - if moreMessages { - filtered = filtered[:*limit] - } + filtered := data.messages + moreMessages := *messageOffset+len(filtered) < data.totalMessages result.Messages = filtered result.Conversations, result.Profiles = entitiesForMessages(filtered, data) - if len(filtered) > 0 { - last := filtered[len(filtered)-1] - result.NextCreatedAt = last.CreatedAt - result.NextMessageID = last.ID - } + result.NextMessageOffset = *messageOffset + len(filtered) result.HasMore = moreMessages || len(data.followEdges) > 0 if moreMessages { result.NextPhase = "messages" @@ -272,6 +261,10 @@ func archiveGeneration(path string) (string, error) { } func readArchive(path string) (archiveData, error) { + return readArchivePage(path, 0, int(^uint(0)>>1)) +} + +func readArchivePage(path string, messageOffset, messageLimit int) (archiveData, error) { reader, err := zip.OpenReader(path) if err != nil { return archiveData{}, fmt.Errorf("open X archive: %w", err) @@ -301,7 +294,7 @@ func readArchive(path string) (archiveData, error) { mentionDirectory := readMentionDirectory(reader.File) for _, file := range reader.File { if dmEntryPattern.MatchString(normalizePath(file.Name)) { - if err := parseDMEntry(file, &result, mentionDirectory); err != nil { + if err := parseDMEntry(file, &result, mentionDirectory, messageOffset, messageLimit); err != nil { return archiveData{}, err } } @@ -319,12 +312,6 @@ func readArchive(path string) (archiveData, error) { } } } - sort.Slice(result.messages, func(i, j int) bool { - if result.messages[i].CreatedAt == result.messages[j].CreatedAt { - return result.messages[i].ID < result.messages[j].ID - } - return result.messages[i].CreatedAt < result.messages[j].CreatedAt - }) sort.Slice(result.followEdges, func(i, j int) bool { if result.followEdges[i].ProfileID == result.followEdges[j].ProfileID { return result.followEdges[i].Direction < result.followEdges[j].Direction @@ -334,7 +321,7 @@ func readArchive(path string) (archiveData, error) { return result, nil } -func parseDMEntry(file *zip.File, result *archiveData, directory map[string]profileRow) error { +func parseDMEntry(file *zip.File, result *archiveData, directory map[string]profileRow, messageOffset, messageLimit int) error { return forEachRecord(file, func(record map[string]any) error { conversation := asMap(record["dmConversation"]) conversationID := stringValue(conversation["conversationId"]) @@ -344,9 +331,14 @@ func parseDMEntry(file *zip.File, result *archiveData, directory map[string]prof participantSet := make(map[string]bool) messages := asSlice(conversation["messages"]) lastMessageAt := "" + selectedConversation := false for _, rawEvent := range messages { - messageCreate := asMap(asMap(rawEvent)["messageCreate"]) + event := asMap(rawEvent) + messageCreate := asMap(event["messageCreate"]) if len(messageCreate) == 0 { + for _, userID := range participantIDs(event) { + participantSet[userID] = true + } continue } senderID := stringValue(messageCreate["senderId"]) @@ -357,7 +349,13 @@ func parseDMEntry(file *zip.File, result *archiveData, directory map[string]prof if recipientID != "" { participantSet[recipientID] = true } - messageID := firstNonEmpty(stringValue(messageCreate["id"]), conversationID+"-"+senderID+"-"+strconv.Itoa(len(result.messages))) + messageIndex := result.totalMessages + result.totalMessages++ + if messageIndex < messageOffset || len(result.messages) >= messageLimit { + continue + } + selectedConversation = true + messageID := firstNonEmpty(stringValue(messageCreate["id"]), conversationID+"-"+senderID+"-"+strconv.Itoa(messageIndex)) createdAt := parseTwitterDate(stringValue(messageCreate["createdAt"])) direction := "inbound" if senderID == result.account.ExternalUserID { @@ -381,7 +379,7 @@ func parseDMEntry(file *zip.File, result *archiveData, directory map[string]prof } } sort.Strings(participants) - if lastMessageAt != "" { + if selectedConversation { result.conversations[conversationID] = conversationRow{ ID: conversationID, ParticipantIDs: participants, Title: firstNonEmpty(stringValue(conversation["name"]), conversationTitle(participants, result.profiles)), @@ -392,6 +390,25 @@ func parseDMEntry(file *zip.File, result *archiveData, directory map[string]prof }) } +func participantIDs(event map[string]any) []string { + result := []string{} + for _, input := range []struct { + key string + list string + }{{"joinConversation", "participantsSnapshot"}, {"participantsJoin", "userIds"}, {"participantsLeave", "userIds"}} { + payload := asMap(event[input.key]) + for _, value := range asSlice(payload[input.list]) { + if userID := stringValue(value); userID != "" { + result = append(result, userID) + } + } + if userID := stringValue(payload["initiatingUserId"]); userID != "" { + result = append(result, userID) + } + } + return result +} + func parseFollowEntry(file *zip.File, result *archiveData, directory map[string]profileRow, direction, key string) error { return forEachRecord(file, func(record map[string]any) error { item := asMap(record[key]) @@ -440,7 +457,7 @@ func ensureProfile(result *archiveData, directory map[string]profileRow, userID } profile, exists := directory[userID] if !exists { - profile = profileRow{ID: id, Handle: "id" + userID, DisplayName: "X user " + userID} + profile = profileRow{ID: id, DisplayName: "X user " + userID} } result.profiles[id] = profile } diff --git a/native/helpers/birdclaw-go/main_test.go b/native/helpers/birdclaw-go/main_test.go index 559053da..611e442b 100644 --- a/native/helpers/birdclaw-go/main_test.go +++ b/native/helpers/birdclaw-go/main_test.go @@ -17,7 +17,7 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { entries := map[string]string{ "archive/data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, "archive/data/tweets.js": `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"Ava Chen"}]}}}]`, - "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"group-1","name":"Builders","messages":[{"joinConversation":{"participantsSnapshot":["10","20","30"]}},{"participantsJoin":{"initiatingUserId":"20","userIds":["40"]}},{"participantsLeave":{"initiatingUserId":"30","userIds":["30"]}},{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, "archive/data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]`, "archive/data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, } @@ -47,6 +47,10 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { if len(data.messages) != 1 || data.messages[0].Text != "hello" { t.Fatalf("messages = %#v", data.messages) } + conversation := data.conversations["group-1"] + if len(conversation.ParticipantIDs) != 3 || conversation.ParticipantIDs[0] != "20" || conversation.ParticipantIDs[1] != "30" || conversation.ParticipantIDs[2] != "40" { + t.Fatalf("conversation participants = %#v", conversation.ParticipantIDs) + } profile := data.profiles["profile_user_20"] if profile.Handle != "ava" || profile.DisplayName != "Ava Chen" { t.Fatalf("profile = %#v", profile) @@ -56,6 +60,41 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { } } +func TestReadsOnlyRequestedMessagePage(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "twitter-archive.zip") + file, err := os.Create(archivePath) + if err != nil { + t.Fatal(err) + } + writer := zip.NewWriter(file) + entries := map[string]string{ + "data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam"}}]`, + "data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10"}},{"messageCreate":{"id":"m2","senderId":"20","recipientId":"10"}},{"messageCreate":{"id":"m3","senderId":"20","recipientId":"10"}}]}}]`, + } + for name, content := range entries { + entry, createErr := writer.Create(name) + if createErr != nil { + t.Fatal(createErr) + } + if _, writeErr := entry.Write([]byte(content)); writeErr != nil { + t.Fatal(writeErr) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + data, err := readArchivePage(archivePath, 1, 1) + if err != nil { + t.Fatal(err) + } + if data.totalMessages != 3 || len(data.messages) != 1 || data.messages[0].ID != "m2" { + t.Fatalf("page = total %d messages %#v", data.totalMessages, data.messages) + } +} + func TestParseTwitterDate(t *testing.T) { if got := parseTwitterDate("Mon Feb 19 14:59:19 +0000 2024"); got != "2024-02-19T14:59:19Z" { t.Fatalf("parseTwitterDate = %q", got) diff --git a/src/core/types/provider.ts b/src/core/types/provider.ts index b6cc2d07..e481a5db 100644 --- a/src/core/types/provider.ts +++ b/src/core/types/provider.ts @@ -26,6 +26,10 @@ export interface ContactObservationPayload { sourceMetadata?: Record | null; } +export interface ContactSnapshotCompletedPayload { + generation: string; +} + export interface ConversationParticipantInput { sourceEntityKey: string; isSelf?: boolean; @@ -40,6 +44,7 @@ export interface ConversationObservationPayload { topic?: string | null; unreadCount?: number | null; removalReason?: string | null; + participantsAreSnapshot?: boolean; participants: ConversationParticipantInput[]; } @@ -143,6 +148,7 @@ export interface CallPayload { export type RawEventPayload = | ContactObservationPayload + | ContactSnapshotCompletedPayload | ConversationObservationPayload | CallPayload | MessagePayload diff --git a/src/platforms/core/types.ts b/src/platforms/core/types.ts index f2bd7302..f5d7478c 100644 --- a/src/platforms/core/types.ts +++ b/src/platforms/core/types.ts @@ -73,7 +73,7 @@ export const PLATFORM_DEFINITIONS = { supportsMultipleAccounts: true, requestableIntegration: true, requestableOrder: 2, - supportedHostOs: ["macos"], + supportedHostOs: ["macos", "windows", "linux"], onboardingVisible: true, permissionRequirements: [], helperRequirements: [], @@ -149,7 +149,7 @@ export const PLATFORM_DEFINITIONS = { supportsMultipleAccounts: false, requestableIntegration: false, requestableOrder: 7, - supportedHostOs: ["macos", "windows", "linux"], + supportedHostOs: ["macos"], onboardingVisible: true, permissionRequirements: [], helperRequirements: ["x_archive_helper"], diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts index 47c3acb8..4f2de1e3 100644 --- a/src/platforms/x/e2e.test.ts +++ b/src/platforms/x/e2e.test.ts @@ -7,7 +7,16 @@ import { CuedDatabase } from "../../db/database.js"; import { projectPendingRawEvents } from "../../runtime/projection/projector.js"; import { buildXSyncBundle } from "./sync/bundle.js"; -function writeArchive(root: string, input: { name: string; follower: boolean }): string { +function writeArchive( + root: string, + input: { + name: string; + follower: boolean; + groupName: string; + groupMember: boolean; + graphOnly: boolean; + }, +): string { const source = join(root, "archive-source"); rmSync(source, { recursive: true, force: true }); mkdirSync(join(source, "data"), { recursive: true }); @@ -21,17 +30,17 @@ function writeArchive(root: string, input: { name: string; follower: boolean }): ); writeFileSync( join(source, "data/direct-messages.js"), - `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}},{"dmConversation":{"conversationId":"group-1","name":"${input.groupName}","messages":[{"joinConversation":{"participantsSnapshot":["10","20"${input.groupMember ? ',"30"' : ""}]}},{"messageCreate":{"id":"group-message-1","senderId":"20","text":"Hello group","createdAt":"2026-08-03T13:00:00.000Z"}}]}}]`, ); writeFileSync( join(source, "data/follower.js"), input.follower - ? `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]` + ? `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}${input.graphOnly ? ',{"follower":{"accountId":"40"}}' : ""}]` : `window.YTD.follower.part0 = []`, ); writeFileSync( join(source, "data/following.js"), - `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, + `window.YTD.following.part0 = [{"following":{"accountId":"20"}}${input.graphOnly ? ',{"following":{"accountId":"40"}}' : ""}]`, ); const archivePath = join(root, "twitter-archive.zip"); rmSync(archivePath, { force: true }); @@ -66,7 +75,13 @@ describe("X archive end to end", () => { it("imports an X archive through the compiled Go helper and refreshes contact state", async () => { const temp = mkdtempSync(join(tmpdir(), "cued-x-archive-e2e-")); tempDirs.push(temp); - const archivePath = writeArchive(temp, { name: "Ava Chen", follower: true }); + const archivePath = writeArchive(temp, { + name: "Ava Chen", + follower: true, + groupName: "Builders", + groupMember: true, + graphOnly: true, + }); const cuedPath = join(temp, "cued.sqlite"); const helperPath = join(temp, "cued-birdclaw-helper"); execFileSync("go", ["build", "-o", helperPath, "."], { @@ -105,10 +120,40 @@ describe("X archive end to end", () => { youFollow: true, }); expect( - sqlite.prepare("SELECT content FROM messages WHERE platform = 'x'").get(), + sqlite + .prepare( + "SELECT content FROM messages WHERE platform = 'x' AND content = 'Hello from the X archive'", + ) + .get(), ).toMatchObject({ content: "Hello from the X archive" }); + const readGraphOnly = () => + sqlite + .prepare( + `SELECT metadata_json FROM contact_sources WHERE platform = 'x' AND source_entity_key = 'x:user:40'`, + ) + .get() as { metadata_json: string }; + expect(JSON.parse(readGraphOnly().metadata_json)).toMatchObject({ mutual: true }); + expect( + sqlite + .prepare( + `SELECT COUNT(*) AS count FROM contact_handles WHERE type = 'x_handle' AND value = 'id40'`, + ) + .get(), + ).toMatchObject({ count: 0 }); + const groupBefore = sqlite + .prepare( + `SELECT id, name FROM conversations WHERE platform = 'x' AND source_conversation_key = 'x:conversation:group-1'`, + ) + .get() as { id: string; name: string }; + expect(groupBefore.name).toBe("Builders"); - writeArchive(temp, { name: "Ava Patel", follower: false }); + writeArchive(temp, { + name: "Ava Patel", + follower: false, + groupName: "Core team", + groupMember: false, + graphOnly: false, + }); const second = await projectArchiveCycle(cued); expect(second.totalApplied).toBeGreaterThan(0); expect(readContact().name).toBe("Ava Patel"); @@ -117,6 +162,21 @@ describe("X archive end to end", () => { followsYou: false, youFollow: true, }); + expect(JSON.parse(readGraphOnly().metadata_json)).toMatchObject({ + mutual: false, + followsYou: false, + youFollow: false, + }); + expect( + sqlite.prepare(`SELECT name FROM conversations WHERE id = ?`).get(groupBefore.id), + ).toMatchObject({ name: "Core team" }); + expect( + sqlite + .prepare( + `SELECT is_active FROM conversation_participants WHERE conversation_id = ? AND source_participant_key = 'x:user:30'`, + ) + .get(groupBefore.id), + ).toMatchObject({ is_active: 0 }); } finally { if (previousHelper === undefined) delete process.env.CUED_X_ARCHIVE_HELPER_BINARY; else process.env.CUED_X_ARCHIVE_HELPER_BINARY = previousHelper; diff --git a/src/platforms/x/sync/bundle.test.ts b/src/platforms/x/sync/bundle.test.ts index 4c188c61..ac2a7a72 100644 --- a/src/platforms/x/sync/bundle.test.ts +++ b/src/platforms/x/sync/bundle.test.ts @@ -56,8 +56,9 @@ const snapshot: BirdclawSnapshot = { }, ], hasMore: false, - nextCreatedAt: "2026-08-03T12:00:00.000Z", - nextMessageId: "message-1", + nextMessageOffset: 1, + totalMessageCount: 1, + totalRelationshipCount: 2, }; describe("Birdclaw X sync bundle", () => { @@ -67,10 +68,11 @@ describe("Birdclaw X sync bundle", () => { expect(bundle.sourceAccounts).toEqual([ { platform: "x", accountKey: "10", displayName: "@sam" }, ]); - expect(bundle.rawEvents.map((event) => event.entityKind)).toEqual([ - "contact", - "conversation", - "message", + expect(bundle.rawEvents.map((event) => `${event.entityKind}.${event.eventKind}`)).toEqual([ + "contact.observed", + "conversation.observed", + "message.created", + "contact.snapshot_completed", ]); expect(bundle.rawEvents[0]?.payload).toMatchObject({ sourceEntityKey: "x:user:20", @@ -86,11 +88,13 @@ describe("Birdclaw X sync bundle", () => { followersCount: 100, }, }); - expect(bundle.rawEvents.at(-1)?.payload).toMatchObject({ - content: "Hello from Birdclaw", - senderSourceKey: "x:user:20", - isFromMe: false, - }); + expect(bundle.rawEvents.find((event) => event.entityKind === "message")?.payload).toMatchObject( + { + content: "Hello from Birdclaw", + senderSourceKey: "x:user:20", + isFromMe: false, + }, + ); expect(bundle.proofs?.[0]?.coverage).toMatchObject({ source: "x_archive", generation: "archive-v1", @@ -103,15 +107,14 @@ describe("Birdclaw X sync bundle", () => { {}, { snapshot: { ...snapshot, hasMore: true, nextPhase: "relationships", nextEdgeOffset: 10 }, - sourceCursor: { createdAt: null }, + sourceCursor: { messageOffset: 0 }, }, ); expect(bundle.hasMore).toBe(true); expect(bundle.sourceCursor).toEqual({ generation: "archive-v1", phase: "relationships", - createdAt: "2026-08-03T12:00:00.000Z", - messageId: "message-1", + messageOffset: 1, edgeOffset: 10, }); expect(bundle.proofs?.[0]?.status).toBe("running"); @@ -153,15 +156,18 @@ describe("Birdclaw X sync bundle", () => { const conversation = bundle.rawEvents.find((event) => event.entityKind === "conversation"); expect(conversation?.payload).toMatchObject({ conversationType: "group", + participantsAreSnapshot: true, participants: [ { sourceEntityKey: "x:user:10", isSelf: true }, { sourceEntityKey: "x:user:20" }, { sourceEntityKey: "x:user:30" }, ], }); - expect(bundle.rawEvents.at(-1)?.payload).toMatchObject({ - senderSourceKey: "x:user:10", - isFromMe: true, - }); + expect(bundle.rawEvents.find((event) => event.entityKind === "message")?.payload).toMatchObject( + { + senderSourceKey: "x:user:10", + isFromMe: true, + }, + ); }); }); diff --git a/src/platforms/x/sync/bundle.ts b/src/platforms/x/sync/bundle.ts index beec9979..d56f37ba 100644 --- a/src/platforms/x/sync/bundle.ts +++ b/src/platforms/x/sync/bundle.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { promisify } from "node:util"; import type { ContactObservationPayload, + ContactSnapshotCompletedPayload, ConversationObservationPayload, MessagePayload, ProviderRawEventInput, @@ -53,15 +54,15 @@ export interface BirdclawSnapshot { hasMore: boolean; nextPhase?: string; nextEdgeOffset?: number; - nextCreatedAt?: string; - nextMessageId?: string; + nextMessageOffset?: number; + totalMessageCount?: number; + totalRelationshipCount?: number; } type BirdclawCursor = { generation: string | null; phase: "messages" | "relationships"; - createdAt: string | null; - messageId: string | null; + messageOffset: number; edgeOffset: number; }; @@ -74,8 +75,7 @@ function parseCursor(value: unknown): BirdclawCursor { return { generation: typeof cursor.generation === "string" ? cursor.generation : null, phase: cursor.phase === "relationships" ? "relationships" : "messages", - createdAt: typeof cursor.createdAt === "string" ? cursor.createdAt : null, - messageId: typeof cursor.messageId === "string" ? cursor.messageId : null, + messageOffset: typeof cursor.messageOffset === "number" ? cursor.messageOffset : 0, edgeOffset: typeof cursor.edgeOffset === "number" ? cursor.edgeOffset : 0, }; } @@ -91,13 +91,12 @@ async function readBirdclawSnapshot(cursor: BirdclawCursor): Promise ({ sourceEntityKey: participantKey })), @@ -244,6 +254,19 @@ export function buildBirdclawRawEvents(input: { } satisfies MessagePayload, }); } + if (!snapshot.hasMore) { + const id = stableId(`x:contact-snapshot:${input.accountKey}:${snapshot.generation}`); + rawEvents.push({ + id, + platform: "x", + accountKey: input.accountKey, + entityKind: "contact", + eventKind: "snapshot_completed", + observedAt: input.observedAt, + dedupeKey: id, + payload: { generation: snapshot.generation } satisfies ContactSnapshotCompletedPayload, + } satisfies ProviderRawEventInput); + } return rawEvents; } @@ -260,15 +283,13 @@ export async function buildXSyncBundle( ? { generation: snapshot.generation, phase: snapshot.nextPhase === "relationships" ? "relationships" : "messages", - createdAt: snapshot.nextCreatedAt ?? cursor.createdAt, - messageId: snapshot.nextMessageId ?? cursor.messageId, + messageOffset: snapshot.nextMessageOffset ?? cursor.messageOffset, edgeOffset: snapshot.nextEdgeOffset ?? 0, } : { generation: snapshot.generation, phase: "messages", - createdAt: null, - messageId: null, + messageOffset: 0, edgeOffset: 0, }; return { @@ -298,9 +319,12 @@ export async function buildXSyncBundle( coverage: { source: "x_archive", generation: snapshot.generation, - relationshipEdges: snapshot.followEdges?.length ?? 0, + relationshipEdges: snapshot.totalRelationshipCount ?? snapshot.followEdges?.length ?? 0, + }, + stats: { + messageCount: snapshot.totalMessageCount ?? snapshot.messages?.length ?? 0, + rawEventCount: rawEvents.length, }, - stats: { messageCount: snapshot.messages?.length ?? 0, rawEventCount: rawEvents.length }, }, ], }; diff --git a/src/runtime/projection/events.ts b/src/runtime/projection/events.ts index 92a64095..26f76407 100644 --- a/src/runtime/projection/events.ts +++ b/src/runtime/projection/events.ts @@ -19,7 +19,7 @@ export type NormalizedProjectedRawEvent = { }; const CANONICAL_SCHEMA_REGISTRY = { - contact: new Set(["observed"]), + contact: new Set(["observed", "snapshot_completed"]), conversation: new Set(["observed", "removed"]), call: new Set(["observed", "deleted"]), message: new Set(["created", "updated", "deleted", "read_receipt"]), @@ -53,6 +53,9 @@ export function assertCanonicalRawEventPayloadForWrite( case "contact.observed@1": assertStringField(normalizedSchema, event.payload, "sourceEntityKey"); return; + case "contact.snapshot_completed@1": + assertStringField(normalizedSchema, event.payload, "generation"); + return; case "conversation.observed@1": case "conversation.removed@1": assertStringField(normalizedSchema, event.payload, "sourceConversationKey"); diff --git a/src/runtime/projection/projector.ts b/src/runtime/projection/projector.ts index bc6e9ece..eb0e5301 100644 --- a/src/runtime/projection/projector.ts +++ b/src/runtime/projection/projector.ts @@ -1,8 +1,9 @@ import { createHash } from "node:crypto"; -import { eq, type SQL, sql } from "drizzle-orm"; +import { and, eq, type SQL, sql } from "drizzle-orm"; import type { CallPayload, ContactObservationPayload, + ContactSnapshotCompletedPayload, ConversationObservationPayload, MessagePayload, ParticipantPayload, @@ -1325,6 +1326,39 @@ function projectConversationObservation( .run(); } + if (event.event_kind !== "removed" && payload.participantsAreSnapshot) { + const participantKeys = new Set( + payload.participants.map((participant) => participant.sourceEntityKey), + ); + const existingParticipants = conn + .select({ + contactId: conversationParticipants.contactId, + sourceParticipantKey: conversationParticipants.sourceParticipantKey, + }) + .from(conversationParticipants) + .where(eq(conversationParticipants.conversationId, conversationId)) + .all(); + for (const participant of existingParticipants) { + if ( + !participant.sourceParticipantKey || + participantKeys.has(participant.sourceParticipantKey) + ) { + continue; + } + conn + .update(conversationParticipants) + .set({ isActive: 0, leftAt: event.observed_at, updatedAt: event.observed_at }) + .where( + and( + eq(conversationParticipants.conversationId, conversationId), + eq(conversationParticipants.contactId, participant.contactId), + eq(conversationParticipants.sourceParticipantKey, participant.sourceParticipantKey), + ), + ) + .run(); + } + } + for (const participant of payload.participants) { const contactId = resolveOrEnsureContact( conn, @@ -1372,6 +1406,49 @@ function projectConversationObservation( changes.dirtyConversationIds.add(conversationId); } +function projectContactSnapshotCompleted( + conn: LocalDbExecutor, + changes: ProjectionChangeSet, + event: ProjectableRawEvent, +): void { + const payload = JSON.parse(event.payload_json) as ContactSnapshotCompletedPayload; + const sources = conn + .select({ + id: contactSources.id, + contactId: contactSources.contactId, + metadataJson: contactSources.metadataJson, + }) + .from(contactSources) + .where( + and( + eq(contactSources.platform, event.platform), + eq(contactSources.accountKey, event.account_key), + ), + ) + .all(); + for (const source of sources) { + const metadata = source.metadataJson + ? (JSON.parse(source.metadataJson) as Record) + : {}; + if (metadata.archiveGeneration === payload.generation) continue; + conn + .update(contactSources) + .set({ + metadataJson: JSON.stringify({ + ...metadata, + archiveGeneration: payload.generation, + followsYou: false, + youFollow: false, + mutual: false, + }), + lastSeenAt: event.observed_at, + }) + .where(eq(contactSources.id, source.id)) + .run(); + changes.dirtyContactIds.add(source.contactId); + } +} + function projectMessageEvent( conn: LocalDbExecutor, cache: ProjectionCache, @@ -2391,7 +2468,11 @@ function projectEventBatch( tx.run(sql.raw(`SAVEPOINT projection_event_${shapedEvent.rawEvent.rowid}`)); try { if (shapedEvent.entity_kind === "contact") { - projectContactObservation(tx, eventCache, eventChanges, shapedEvent); + if (shapedEvent.event_kind === "snapshot_completed") { + projectContactSnapshotCompleted(tx, eventChanges, shapedEvent); + } else { + projectContactObservation(tx, eventCache, eventChanges, shapedEvent); + } } else if (shapedEvent.entity_kind === "conversation") { projectConversationObservation(tx, eventCache, eventChanges, shapedEvent); } else if (shapedEvent.entity_kind === "call") { From 1a0b5c2c4a43235375fd1fecb695b11b6ef4a12f Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:43:18 -0400 Subject: [PATCH 05/10] fix(x): report generation proof totals --- src/platforms/x/sync/bundle.test.ts | 5 +++++ src/platforms/x/sync/bundle.ts | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/platforms/x/sync/bundle.test.ts b/src/platforms/x/sync/bundle.test.ts index ac2a7a72..ed9e2c6f 100644 --- a/src/platforms/x/sync/bundle.test.ts +++ b/src/platforms/x/sync/bundle.test.ts @@ -100,6 +100,11 @@ describe("Birdclaw X sync bundle", () => { generation: "archive-v1", relationshipEdges: 2, }); + expect(bundle.proofs?.[0]?.stats).toMatchObject({ + messageCount: 1, + relationshipEdgeCount: 2, + pageRawEventCount: 4, + }); }); it("continues from the helper cursor when another DM page remains", async () => { diff --git a/src/platforms/x/sync/bundle.ts b/src/platforms/x/sync/bundle.ts index d56f37ba..647bbdfd 100644 --- a/src/platforms/x/sync/bundle.ts +++ b/src/platforms/x/sync/bundle.ts @@ -323,7 +323,9 @@ export async function buildXSyncBundle( }, stats: { messageCount: snapshot.totalMessageCount ?? snapshot.messages?.length ?? 0, - rawEventCount: rawEvents.length, + relationshipEdgeCount: + snapshot.totalRelationshipCount ?? snapshot.followEdges?.length ?? 0, + pageRawEventCount: rawEvents.length, }, }, ], From db3694c641aacc36be93be29ac7b1301e02c1bd3 Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:53:50 -0400 Subject: [PATCH 06/10] fix(x): stream archive pages with bounded state --- native/helpers/birdclaw-go/main.go | 450 ++++++++++++++++-------- native/helpers/birdclaw-go/main_test.go | 61 +++- src/platforms/x/e2e.test.ts | 2 +- src/platforms/x/sync/bundle.test.ts | 11 +- src/platforms/x/sync/bundle.ts | 32 +- src/runtime/projection/projector.ts | 1 - 6 files changed, 403 insertions(+), 154 deletions(-) diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index dda3b112..79c7a29f 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -3,10 +3,12 @@ package main import ( "archive/zip" "bufio" + "container/heap" "encoding/json" "errors" "flag" "fmt" + "io" "os" "path/filepath" "regexp" @@ -81,7 +83,7 @@ type syncResult struct { ArchivePath string `json:"archivePath"` Generation string `json:"generation"` NextPhase string `json:"nextPhase,omitempty"` - NextEdgeOffset int `json:"nextEdgeOffset,omitempty"` + NextRelationshipUserID string `json:"nextRelationshipUserId,omitempty"` NextMessageOffset int `json:"nextMessageOffset,omitempty"` TotalMessageCount int `json:"totalMessageCount"` TotalRelationshipCount int `json:"totalRelationshipCount"` @@ -94,12 +96,14 @@ type syncResult struct { } type archiveData struct { - account accountRow - profiles map[string]profileRow - conversations map[string]conversationRow - messages []messageRow - followEdges []followEdgeRow - totalMessages int + account accountRow + profiles map[string]profileRow + conversations map[string]conversationRow + messages []messageRow + followEdges []followEdgeRow + totalMessages int + totalRelationships int + relationshipHasMore bool } func main() { @@ -148,7 +152,9 @@ func runSync(args []string) { messageOffset := flags.Int("message-offset", 0, "message cursor offset") expectedGeneration := flags.String("generation", "", "archive generation for the cursor") phase := flags.String("phase", "messages", "messages or relationships") - edgeOffset := flags.Int("edge-offset", 0, "relationship cursor offset") + afterRelationshipUserID := flags.String("after-relationship-user-id", "", "exclusive relationship user id cursor") + knownMessageCount := flags.Int("known-message-count", -1, "known generation message total") + knownRelationshipCount := flags.Int("known-relationship-count", -1, "known generation relationship total") limit := flags.Int("limit", defaultLimit, "maximum messages per page") if err := flags.Parse(args); err != nil { fail(err) @@ -165,9 +171,10 @@ func runSync(args []string) { fail(err) } if *expectedGeneration != "" && *expectedGeneration != generation { - *messageOffset, *phase, *edgeOffset = 0, "messages", 0 + *messageOffset, *phase, *afterRelationshipUserID = 0, "messages", "" + *knownMessageCount, *knownRelationshipCount = -1, -1 } - data, err := readArchivePage(resolved, *messageOffset, *limit) + data, err := readArchivePage(resolved, *phase, *messageOffset, *afterRelationshipUserID, *knownMessageCount, *knownRelationshipCount, *limit) if err != nil { fail(err) } @@ -175,24 +182,17 @@ func runSync(args []string) { HelperVersion: helperVersion, ProtocolVersion: protocolVersion, ArchivePath: resolved, Generation: generation, Account: data.account, Profiles: []profileRow{}, Conversations: []conversationRow{}, Messages: []messageRow{}, FollowEdges: []followEdgeRow{}, - TotalMessageCount: data.totalMessages, TotalRelationshipCount: len(data.followEdges), + TotalMessageCount: data.totalMessages, TotalRelationshipCount: data.totalRelationships, } if *phase == "relationships" { - start := max(0, *edgeOffset) - if start > len(data.followEdges) { - start = len(data.followEdges) - } - end := min(start+*limit, len(data.followEdges)) - for end < len(data.followEdges) && end > start && - data.followEdges[end].ProfileID == data.followEdges[end-1].ProfileID { - end++ - } - result.FollowEdges = data.followEdges[start:end] + result.FollowEdges = data.followEdges result.Profiles = profilesForEdges(result.FollowEdges, data.profiles) - result.HasMore = end < len(data.followEdges) + result.HasMore = data.relationshipHasMore if result.HasMore { result.NextPhase = "relationships" - result.NextEdgeOffset = end + if len(data.followEdges) > 0 { + result.NextRelationshipUserID = data.followEdges[len(data.followEdges)-1].ExternalUserID + } } } else { filtered := data.messages @@ -200,10 +200,10 @@ func runSync(args []string) { result.Messages = filtered result.Conversations, result.Profiles = entitiesForMessages(filtered, data) result.NextMessageOffset = *messageOffset + len(filtered) - result.HasMore = moreMessages || len(data.followEdges) > 0 + result.HasMore = moreMessages || data.totalRelationships > 0 if moreMessages { result.NextPhase = "messages" - } else if len(data.followEdges) > 0 { + } else if data.totalRelationships > 0 { result.NextPhase = "relationships" } } @@ -261,10 +261,10 @@ func archiveGeneration(path string) (string, error) { } func readArchive(path string) (archiveData, error) { - return readArchivePage(path, 0, int(^uint(0)>>1)) + return readArchivePage(path, "messages", 0, "", -1, -1, int(^uint(0)>>1)) } -func readArchivePage(path string, messageOffset, messageLimit int) (archiveData, error) { +func readArchivePage(path, phase string, messageOffset int, afterRelationshipUserID string, knownMessageCount, knownRelationshipCount, limit int) (archiveData, error) { reader, err := zip.OpenReader(path) if err != nil { return archiveData{}, fmt.Errorf("open X archive: %w", err) @@ -291,141 +291,301 @@ func readArchivePage(path string, messageOffset, messageLimit int) (archiveData, if result.account.ID == "" { return archiveData{}, errors.New("X archive account id is missing") } - mentionDirectory := readMentionDirectory(reader.File) - for _, file := range reader.File { - if dmEntryPattern.MatchString(normalizePath(file.Name)) { - if err := parseDMEntry(file, &result, mentionDirectory, messageOffset, messageLimit); err != nil { - return archiveData{}, err - } + if phase == "relationships" { + if knownMessageCount >= 0 { + result.totalMessages = knownMessageCount } - } - for _, input := range []struct { - pattern *regexp.Regexp - direction string - key string - }{{followerEntryPattern, "followers", "follower"}, {followingEntryPattern, "following", "following"}} { + if err := readRelationshipPage(reader.File, &result, afterRelationshipUserID, limit); err != nil { + return archiveData{}, err + } + } else { for _, file := range reader.File { - if input.pattern.MatchString(normalizePath(file.Name)) { - if err := parseFollowEntry(file, &result, mentionDirectory, input.direction, input.key); err != nil { + if dmEntryPattern.MatchString(normalizePath(file.Name)) { + if err := parseDMEntry(file, &result, messageOffset, limit); err != nil { return archiveData{}, err } } } - } - sort.Slice(result.followEdges, func(i, j int) bool { - if result.followEdges[i].ProfileID == result.followEdges[j].ProfileID { - return result.followEdges[i].Direction < result.followEdges[j].Direction + if knownRelationshipCount >= 0 { + result.totalRelationships = knownRelationshipCount + } else { + count, err := countRelationshipEdges(reader.File) + if err != nil { + return archiveData{}, err + } + result.totalRelationships = count } - return result.followEdges[i].ProfileID < result.followEdges[j].ProfileID - }) + } + enrichProfilesFromMentions(reader.File, result.profiles) return result, nil } -func parseDMEntry(file *zip.File, result *archiveData, directory map[string]profileRow, messageOffset, messageLimit int) error { - return forEachRecord(file, func(record map[string]any) error { - conversation := asMap(record["dmConversation"]) - conversationID := stringValue(conversation["conversationId"]) - if conversationID == "" { - return nil +func parseDMEntry(file *zip.File, result *archiveData, messageOffset, messageLimit int) error { + reader, decoder, err := archiveArrayDecoder(file) + if err != nil { + return err + } + defer reader.Close() + for decoder.More() { + if token, err := decoder.Token(); err != nil || token != json.Delim('{') { + return fmt.Errorf("read %s record: %w", file.Name, err) } - participantSet := make(map[string]bool) - messages := asSlice(conversation["messages"]) - lastMessageAt := "" - selectedConversation := false - for _, rawEvent := range messages { - event := asMap(rawEvent) - messageCreate := asMap(event["messageCreate"]) - if len(messageCreate) == 0 { - for _, userID := range participantIDs(event) { - participantSet[userID] = true + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + if keyToken != "dmConversation" { + var ignored any + if err := decoder.Decode(&ignored); err != nil { + return err } continue } - senderID := stringValue(messageCreate["senderId"]) - recipientID := stringValue(messageCreate["recipientId"]) - if senderID != "" { - participantSet[senderID] = true + if err := parseDMConversation(decoder, result, messageOffset, messageLimit); err != nil { + return err } - if recipientID != "" { - participantSet[recipientID] = true + } + if _, err := decoder.Token(); err != nil { + return err + } + } + return nil +} + +func parseDMConversation(decoder *json.Decoder, result *archiveData, messageOffset, messageLimit int) error { + if token, err := decoder.Token(); err != nil || token != json.Delim('{') { + return fmt.Errorf("read dmConversation: %w", err) + } + conversationID, name := "", "" + membership := map[string]bool{} + knownMembership := map[string]bool{} + selected := []messageRow{} + lastMessageAt := "" + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key := keyToken.(string) + switch key { + case "conversationId": + if err := decoder.Decode(&conversationID); err != nil { + return err } - messageIndex := result.totalMessages - result.totalMessages++ - if messageIndex < messageOffset || len(result.messages) >= messageLimit { - continue + case "name": + if err := decoder.Decode(&name); err != nil { + return err } - selectedConversation = true - messageID := firstNonEmpty(stringValue(messageCreate["id"]), conversationID+"-"+senderID+"-"+strconv.Itoa(messageIndex)) - createdAt := parseTwitterDate(stringValue(messageCreate["createdAt"])) - direction := "inbound" - if senderID == result.account.ExternalUserID { - direction = "outbound" - } else if senderID != "" { - ensureProfile(result, directory, senderID) + case "messages": + if token, err := decoder.Token(); err != nil || token != json.Delim('[') { + return err } - result.messages = append(result.messages, messageRow{ - ID: messageID, ConversationID: conversationID, SenderProfileID: profileID(senderID), - Text: stringValue(messageCreate["text"]), CreatedAt: createdAt, Direction: direction, - }) - if createdAt > lastMessageAt { - lastMessageAt = createdAt + for decoder.More() { + var event map[string]any + if err := decoder.Decode(&event); err != nil { + return err + } + applyMembershipEvent(event, membership, knownMembership) + messageCreate := asMap(event["messageCreate"]) + if len(messageCreate) == 0 { + continue + } + messageIndex := result.totalMessages + result.totalMessages++ + if messageIndex < messageOffset || len(result.messages)+len(selected) >= messageLimit { + continue + } + senderID := stringValue(messageCreate["senderId"]) + createdAt := parseTwitterDate(stringValue(messageCreate["createdAt"])) + direction := "inbound" + if senderID == result.account.ExternalUserID { + direction = "outbound" + } + selected = append(selected, messageRow{ID: firstNonEmpty(stringValue(messageCreate["id"]), "message-"+strconv.Itoa(messageIndex)), SenderProfileID: profileID(senderID), Text: stringValue(messageCreate["text"]), CreatedAt: createdAt, Direction: direction}) + if createdAt > lastMessageAt { + lastMessageAt = createdAt + } } - } - participants := make([]string, 0, len(participantSet)) - for userID := range participantSet { - if userID != "" && userID != result.account.ExternalUserID { - participants = append(participants, userID) - ensureProfile(result, directory, userID) + if _, err := decoder.Token(); err != nil { + return err } - } - sort.Strings(participants) - if selectedConversation { - result.conversations[conversationID] = conversationRow{ - ID: conversationID, ParticipantIDs: participants, - Title: firstNonEmpty(stringValue(conversation["name"]), conversationTitle(participants, result.profiles)), - LastMessageAt: lastMessageAt, IsGroup: len(participants) > 1 || stringValue(conversation["name"]) != "", + default: + var ignored any + if err := decoder.Decode(&ignored); err != nil { + return err } } + } + if _, err := decoder.Token(); err != nil { + return err + } + if conversationID == "" || len(selected) == 0 { return nil - }) + } + participants := []string{} + for userID, active := range membership { + if active && userID != "" && userID != result.account.ExternalUserID { + participants = append(participants, userID) + ensureProfile(result, userID) + } + } + sort.Strings(participants) + for index := range selected { + selected[index].ConversationID = conversationID + if strings.HasPrefix(selected[index].ID, "message-") { + selected[index].ID = conversationID + "-" + selected[index].ID + } + if selected[index].Direction != "outbound" { + ensureProfile(result, strings.TrimPrefix(selected[index].SenderProfileID, "profile_user_")) + } + } + result.messages = append(result.messages, selected...) + result.conversations[conversationID] = conversationRow{ID: conversationID, ParticipantIDs: participants, Title: firstNonEmpty(name, conversationTitle(participants, result.profiles)), LastMessageAt: lastMessageAt, IsGroup: len(participants) > 1 || name != ""} + return nil } -func participantIDs(event map[string]any) []string { - result := []string{} - for _, input := range []struct { - key string - list string - }{{"joinConversation", "participantsSnapshot"}, {"participantsJoin", "userIds"}, {"participantsLeave", "userIds"}} { - payload := asMap(event[input.key]) - for _, value := range asSlice(payload[input.list]) { - if userID := stringValue(value); userID != "" { - result = append(result, userID) - } +func applyMembershipEvent(event map[string]any, membership, known map[string]bool) { + setIfUnknown := func(userID string, active bool) { + if userID == "" || known[userID] { + return } - if userID := stringValue(payload["initiatingUserId"]); userID != "" { - result = append(result, userID) + known[userID], membership[userID] = true, active + } + if payload := asMap(event["participantsLeave"]); len(payload) > 0 { + for _, value := range asSlice(payload["userIds"]) { + setIfUnknown(stringValue(value), false) } + setIfUnknown(stringValue(payload["initiatingUserId"]), false) + } + if payload := asMap(event["participantsJoin"]); len(payload) > 0 { + for _, value := range asSlice(payload["userIds"]) { + setIfUnknown(stringValue(value), true) + } + setIfUnknown(stringValue(payload["initiatingUserId"]), true) + } + if payload := asMap(event["joinConversation"]); len(payload) > 0 { + for _, value := range asSlice(payload["participantsSnapshot"]) { + setIfUnknown(stringValue(value), true) + } + } + if payload := asMap(event["messageCreate"]); len(payload) > 0 { + setIfUnknown(stringValue(payload["senderId"]), true) + setIfUnknown(stringValue(payload["recipientId"]), true) } - return result } -func parseFollowEntry(file *zip.File, result *archiveData, directory map[string]profileRow, direction, key string) error { - return forEachRecord(file, func(record map[string]any) error { - item := asMap(record[key]) - userID := stringValue(item["accountId"]) - if userID == "" { - return nil +type userIDMaxHeap []string + +func (items userIDMaxHeap) Len() int { return len(items) } +func (items userIDMaxHeap) Less(i, j int) bool { return items[i] > items[j] } +func (items userIDMaxHeap) Swap(i, j int) { items[i], items[j] = items[j], items[i] } +func (items *userIDMaxHeap) Push(value any) { *items = append(*items, value.(string)) } +func (items *userIDMaxHeap) Pop() any { + old := *items + last := old[len(old)-1] + *items = old[:len(old)-1] + return last +} + +var followInputs = []struct { + pattern *regexp.Regexp + direction, key string +}{ + {followerEntryPattern, "followers", "follower"}, + {followingEntryPattern, "following", "following"}, +} + +func scanFollowRecords(files []*zip.File, visit func(direction, userID string)) error { + for _, input := range followInputs { + for _, file := range files { + if !input.pattern.MatchString(normalizePath(file.Name)) { + continue + } + if err := forEachRecord(file, func(record map[string]any) error { + userID := stringValue(asMap(record[input.key])["accountId"]) + if userID != "" { + visit(input.direction, userID) + } + return nil + }); err != nil { + return err + } } - ensureProfile(result, directory, userID) - result.followEdges = append(result.followEdges, followEdgeRow{ - Direction: direction, ProfileID: profileID(userID), ExternalUserID: userID, Source: "archive", - }) - return nil - }) + } + return nil +} + +func countRelationshipEdges(files []*zip.File) (int, error) { + count := 0 + err := scanFollowRecords(files, func(_, _ string) { count++ }) + return count, err +} + +func readRelationshipPage(files []*zip.File, result *archiveData, afterUserID string, limit int) error { + candidates := &userIDMaxHeap{} + heap.Init(candidates) + selected := map[string]bool{} + total := 0 + if err := scanFollowRecords(files, func(_ string, userID string) { + total++ + if userID <= afterUserID || selected[userID] { + return + } + if candidates.Len() < limit { + heap.Push(candidates, userID) + selected[userID] = true + return + } + if userID >= (*candidates)[0] { + return + } + delete(selected, heap.Pop(candidates).(string)) + heap.Push(candidates, userID) + selected[userID] = true + }); err != nil { + return err + } + userIDs := make([]string, 0, len(selected)) + for userID := range selected { + userIDs = append(userIDs, userID) + } + sort.Strings(userIDs) + directions := map[string]map[string]bool{} + if err := scanFollowRecords(files, func(direction, userID string) { + if !selected[userID] { + return + } + if directions[userID] == nil { + directions[userID] = map[string]bool{} + } + directions[userID][direction] = true + }); err != nil { + return err + } + for _, userID := range userIDs { + ensureProfile(result, userID) + for _, direction := range []string{"followers", "following"} { + if directions[userID][direction] { + result.followEdges = append(result.followEdges, followEdgeRow{Direction: direction, ProfileID: profileID(userID), ExternalUserID: userID, Source: "archive"}) + } + } + } + result.totalRelationships = total + if len(userIDs) > 0 { + last := userIDs[len(userIDs)-1] + if err := scanFollowRecords(files, func(_ string, userID string) { + if userID > last { + result.relationshipHasMore = true + } + }); err != nil { + return err + } + } + return nil } -func readMentionDirectory(files []*zip.File) map[string]profileRow { - directory := make(map[string]profileRow) +func enrichProfilesFromMentions(files []*zip.File, profiles map[string]profileRow) { for _, file := range files { if !tweetEntryPattern.MatchString(normalizePath(file.Name)) { continue @@ -436,10 +596,11 @@ func readMentionDirectory(files []*zip.File) map[string]profileRow { for _, rawMention := range asSlice(entities["user_mentions"]) { mention := asMap(rawMention) userID := firstNonEmpty(stringValue(mention["id_str"]), stringValue(mention["id"])) - if userID != "" { + id := profileID(userID) + if _, wanted := profiles[id]; wanted { handle := stringValue(mention["screen_name"]) - directory[userID] = profileRow{ - ID: profileID(userID), Handle: handle, + profiles[id] = profileRow{ + ID: id, Handle: handle, DisplayName: firstNonEmpty(stringValue(mention["name"]), handle, "X user "+userID), } } @@ -447,19 +608,32 @@ func readMentionDirectory(files []*zip.File) map[string]profileRow { return nil }) } - return directory } -func ensureProfile(result *archiveData, directory map[string]profileRow, userID string) { +func ensureProfile(result *archiveData, userID string) { id := profileID(userID) if _, exists := result.profiles[id]; exists { return } - profile, exists := directory[userID] - if !exists { - profile = profileRow{ID: id, DisplayName: "X user " + userID} + result.profiles[id] = profileRow{ID: id, DisplayName: "X user " + userID} +} + +func archiveArrayDecoder(file *zip.File) (io.ReadCloser, *json.Decoder, error) { + reader, err := file.Open() + if err != nil { + return nil, nil, err + } + buffered := bufio.NewReader(reader) + if _, err := buffered.ReadString('='); err != nil { + reader.Close() + return nil, nil, fmt.Errorf("read %s assignment: %w", file.Name, err) + } + decoder := json.NewDecoder(buffered) + if token, err := decoder.Token(); err != nil || token != json.Delim('[') { + reader.Close() + return nil, nil, fmt.Errorf("read %s array: %w", file.Name, err) } - result.profiles[id] = profile + return reader, decoder, nil } func forEachRecord(file *zip.File, visit func(map[string]any) error) error { diff --git a/native/helpers/birdclaw-go/main_test.go b/native/helpers/birdclaw-go/main_test.go index 611e442b..99d7104d 100644 --- a/native/helpers/birdclaw-go/main_test.go +++ b/native/helpers/birdclaw-go/main_test.go @@ -17,7 +17,7 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { entries := map[string]string{ "archive/data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, "archive/data/tweets.js": `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"Ava Chen"}]}}}]`, - "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"group-1","name":"Builders","messages":[{"joinConversation":{"participantsSnapshot":["10","20","30"]}},{"participantsJoin":{"initiatingUserId":"20","userIds":["40"]}},{"participantsLeave":{"initiatingUserId":"30","userIds":["30"]}},{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"group-1","name":"Builders","messages":[{"participantsLeave":{"initiatingUserId":"30","userIds":["30"]}},{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}},{"participantsJoin":{"initiatingUserId":"20","userIds":["40"]}},{"joinConversation":{"participantsSnapshot":["10","20","30"]}}]}}]`, "archive/data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]`, "archive/data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, } @@ -48,15 +48,22 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { t.Fatalf("messages = %#v", data.messages) } conversation := data.conversations["group-1"] - if len(conversation.ParticipantIDs) != 3 || conversation.ParticipantIDs[0] != "20" || conversation.ParticipantIDs[1] != "30" || conversation.ParticipantIDs[2] != "40" { + if len(conversation.ParticipantIDs) != 2 || conversation.ParticipantIDs[0] != "20" || conversation.ParticipantIDs[1] != "40" { t.Fatalf("conversation participants = %#v", conversation.ParticipantIDs) } profile := data.profiles["profile_user_20"] if profile.Handle != "ava" || profile.DisplayName != "Ava Chen" { t.Fatalf("profile = %#v", profile) } - if len(data.followEdges) != 2 { - t.Fatalf("followEdges = %#v", data.followEdges) + if data.totalRelationships != 2 { + t.Fatalf("totalRelationships = %d", data.totalRelationships) + } + relationships, err := readArchivePage(archivePath, "relationships", 0, "", data.totalMessages, data.totalRelationships, 1) + if err != nil { + t.Fatal(err) + } + if len(relationships.followEdges) != 2 { + t.Fatalf("followEdges = %#v", relationships.followEdges) } } @@ -86,7 +93,7 @@ func TestReadsOnlyRequestedMessagePage(t *testing.T) { if err := file.Close(); err != nil { t.Fatal(err) } - data, err := readArchivePage(archivePath, 1, 1) + data, err := readArchivePage(archivePath, "messages", 1, "", -1, -1, 1) if err != nil { t.Fatal(err) } @@ -95,6 +102,50 @@ func TestReadsOnlyRequestedMessagePage(t *testing.T) { } } +func TestPagesRelationshipsWithoutParsingDMs(t *testing.T) { + archivePath := filepath.Join(t.TempDir(), "twitter-archive.zip") + file, err := os.Create(archivePath) + if err != nil { + t.Fatal(err) + } + writer := zip.NewWriter(file) + entries := map[string]string{ + "data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam"}}]`, + "data/direct-messages.js": `this is intentionally not a valid archive slice`, + "data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"30"}},{"follower":{"accountId":"10"}},{"follower":{"accountId":"20"}},{"follower":{"accountId":"40"}}]`, + "data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"30"}},{"following":{"accountId":"20"}}]`, + } + for name, content := range entries { + entry, createErr := writer.Create(name) + if createErr != nil { + t.Fatal(createErr) + } + if _, writeErr := entry.Write([]byte(content)); writeErr != nil { + t.Fatal(writeErr) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + first, err := readArchivePage(archivePath, "relationships", 0, "", 7, 6, 2) + if err != nil { + t.Fatal(err) + } + if !first.relationshipHasMore || len(first.profiles) != 2 || len(first.followEdges) != 3 { + t.Fatalf("first relationship page = %#v", first) + } + second, err := readArchivePage(archivePath, "relationships", 0, "20", 7, 6, 2) + if err != nil { + t.Fatal(err) + } + if second.relationshipHasMore || len(second.profiles) != 2 || len(second.followEdges) != 3 { + t.Fatalf("second relationship page = %#v", second) + } +} + func TestParseTwitterDate(t *testing.T) { if got := parseTwitterDate("Mon Feb 19 14:59:19 +0000 2024"); got != "2024-02-19T14:59:19Z" { t.Fatalf("parseTwitterDate = %q", got) diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts index 4f2de1e3..2863e8aa 100644 --- a/src/platforms/x/e2e.test.ts +++ b/src/platforms/x/e2e.test.ts @@ -30,7 +30,7 @@ function writeArchive( ); writeFileSync( join(source, "data/direct-messages.js"), - `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}},{"dmConversation":{"conversationId":"group-1","name":"${input.groupName}","messages":[{"joinConversation":{"participantsSnapshot":["10","20"${input.groupMember ? ',"30"' : ""}]}},{"messageCreate":{"id":"group-message-1","senderId":"20","text":"Hello group","createdAt":"2026-08-03T13:00:00.000Z"}}]}}]`, + `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}},{"dmConversation":{"conversationId":"group-1","name":"${input.groupName}","messages":[${input.groupMember ? "" : '{"participantsLeave":{"initiatingUserId":"30","userIds":["30"],"createdAt":"2026-08-03T14:00:00.000Z"}},'}{"messageCreate":{"id":"group-message-1","senderId":"20","text":"Hello group","createdAt":"2026-08-03T13:00:00.000Z"}},{"joinConversation":{"participantsSnapshot":["10","20","30"],"createdAt":"2026-08-03T11:00:00.000Z"}}]}}]`, ); writeFileSync( join(source, "data/follower.js"), diff --git a/src/platforms/x/sync/bundle.test.ts b/src/platforms/x/sync/bundle.test.ts index ed9e2c6f..4266ed55 100644 --- a/src/platforms/x/sync/bundle.test.ts +++ b/src/platforms/x/sync/bundle.test.ts @@ -111,7 +111,12 @@ describe("Birdclaw X sync bundle", () => { const bundle = await buildXSyncBundle( {}, { - snapshot: { ...snapshot, hasMore: true, nextPhase: "relationships", nextEdgeOffset: 10 }, + snapshot: { + ...snapshot, + hasMore: true, + nextPhase: "relationships", + nextRelationshipUserId: "20", + }, sourceCursor: { messageOffset: 0 }, }, ); @@ -120,7 +125,9 @@ describe("Birdclaw X sync bundle", () => { generation: "archive-v1", phase: "relationships", messageOffset: 1, - edgeOffset: 10, + relationshipUserId: "20", + totalMessageCount: 1, + totalRelationshipCount: 2, }); expect(bundle.proofs?.[0]?.status).toBe("running"); }); diff --git a/src/platforms/x/sync/bundle.ts b/src/platforms/x/sync/bundle.ts index 647bbdfd..b621f992 100644 --- a/src/platforms/x/sync/bundle.ts +++ b/src/platforms/x/sync/bundle.ts @@ -53,7 +53,7 @@ export interface BirdclawSnapshot { }>; hasMore: boolean; nextPhase?: string; - nextEdgeOffset?: number; + nextRelationshipUserId?: string; nextMessageOffset?: number; totalMessageCount?: number; totalRelationshipCount?: number; @@ -63,7 +63,9 @@ type BirdclawCursor = { generation: string | null; phase: "messages" | "relationships"; messageOffset: number; - edgeOffset: number; + relationshipUserId: string | null; + totalMessageCount: number | null; + totalRelationshipCount: number | null; }; function stableId(seed: string): string { @@ -76,7 +78,12 @@ function parseCursor(value: unknown): BirdclawCursor { generation: typeof cursor.generation === "string" ? cursor.generation : null, phase: cursor.phase === "relationships" ? "relationships" : "messages", messageOffset: typeof cursor.messageOffset === "number" ? cursor.messageOffset : 0, - edgeOffset: typeof cursor.edgeOffset === "number" ? cursor.edgeOffset : 0, + relationshipUserId: + typeof cursor.relationshipUserId === "string" ? cursor.relationshipUserId : null, + totalMessageCount: + typeof cursor.totalMessageCount === "number" ? cursor.totalMessageCount : null, + totalRelationshipCount: + typeof cursor.totalRelationshipCount === "number" ? cursor.totalRelationshipCount : null, }; } @@ -93,10 +100,17 @@ async function readBirdclawSnapshot(cursor: BirdclawCursor): Promise Date: Mon, 3 Aug 2026 02:54:19 -0400 Subject: [PATCH 07/10] docs(x): explain archive membership order --- native/helpers/birdclaw-go/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index 79c7a29f..2dc2d839 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -446,6 +446,7 @@ func parseDMConversation(decoder *json.Decoder, result *archiveData, messageOffs } func applyMembershipEvent(event map[string]any, membership, known map[string]bool) { + // X stores DM events newest-first, so the first membership fact for a user is current. setIfUnknown := func(userID string, active bool) { if userID == "" || known[userID] { return From f79dac5469b5332f07638306ea2b2ea091311d67 Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:55:13 -0400 Subject: [PATCH 08/10] fix(x): preserve lifecycle event initiators --- native/helpers/birdclaw-go/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index 2dc2d839..3b924153 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -454,10 +454,13 @@ func applyMembershipEvent(event map[string]any, membership, known map[string]boo known[userID], membership[userID] = true, active } if payload := asMap(event["participantsLeave"]); len(payload) > 0 { - for _, value := range asSlice(payload["userIds"]) { + userIDs := asSlice(payload["userIds"]) + for _, value := range userIDs { setIfUnknown(stringValue(value), false) } - setIfUnknown(stringValue(payload["initiatingUserId"]), false) + if len(userIDs) == 0 { + setIfUnknown(stringValue(payload["initiatingUserId"]), false) + } } if payload := asMap(event["participantsJoin"]); len(payload) > 0 { for _, value := range asSlice(payload["userIds"]) { From ed08470454f19ef88f835e203ff2fdaddc2e35ca Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:56:23 -0400 Subject: [PATCH 09/10] test(x): preserve absent-source observation time --- src/platforms/x/e2e.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts index 2863e8aa..5c22de55 100644 --- a/src/platforms/x/e2e.test.ts +++ b/src/platforms/x/e2e.test.ts @@ -129,10 +129,11 @@ describe("X archive end to end", () => { const readGraphOnly = () => sqlite .prepare( - `SELECT metadata_json FROM contact_sources WHERE platform = 'x' AND source_entity_key = 'x:user:40'`, + `SELECT metadata_json, last_seen_at FROM contact_sources WHERE platform = 'x' AND source_entity_key = 'x:user:40'`, ) - .get() as { metadata_json: string }; + .get() as { metadata_json: string; last_seen_at: number }; expect(JSON.parse(readGraphOnly().metadata_json)).toMatchObject({ mutual: true }); + const graphOnlyLastSeenAt = readGraphOnly().last_seen_at; expect( sqlite .prepare( @@ -167,6 +168,7 @@ describe("X archive end to end", () => { followsYou: false, youFollow: false, }); + expect(readGraphOnly().last_seen_at).toBe(graphOnlyLastSeenAt); expect( sqlite.prepare(`SELECT name FROM conversations WHERE id = ?`).get(groupBefore.id), ).toMatchObject({ name: "Core team" }); From 1f09244f774b6c6a746a003b76714981e0981a08 Mon Sep 17 00:00:00 2001 From: Soham Bafana Date: Mon, 3 Aug 2026 02:59:35 -0400 Subject: [PATCH 10/10] fix(x): finalize enriched conversation identity --- native/helpers/birdclaw-go/main.go | 17 ++++++++++++----- native/helpers/birdclaw-go/main_test.go | 15 +++++++++------ src/platforms/x/e2e.test.ts | 13 ++++++++++++- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/native/helpers/birdclaw-go/main.go b/native/helpers/birdclaw-go/main.go index 3b924153..a5971db3 100644 --- a/native/helpers/birdclaw-go/main.go +++ b/native/helpers/birdclaw-go/main.go @@ -301,7 +301,8 @@ func readArchivePage(path, phase string, messageOffset int, afterRelationshipUse } else { for _, file := range reader.File { if dmEntryPattern.MatchString(normalizePath(file.Name)) { - if err := parseDMEntry(file, &result, messageOffset, limit); err != nil { + isGroupSlice := strings.Contains(strings.ToLower(normalizePath(file.Name)), "direct-messages-group") + if err := parseDMEntry(file, &result, messageOffset, limit, isGroupSlice); err != nil { return archiveData{}, err } } @@ -317,10 +318,16 @@ func readArchivePage(path, phase string, messageOffset int, afterRelationshipUse } } enrichProfilesFromMentions(reader.File, result.profiles) + for id, conversation := range result.conversations { + if conversation.Title == "" { + conversation.Title = conversationTitle(conversation.ParticipantIDs, result.profiles) + result.conversations[id] = conversation + } + } return result, nil } -func parseDMEntry(file *zip.File, result *archiveData, messageOffset, messageLimit int) error { +func parseDMEntry(file *zip.File, result *archiveData, messageOffset, messageLimit int, isGroupSlice bool) error { reader, decoder, err := archiveArrayDecoder(file) if err != nil { return err @@ -342,7 +349,7 @@ func parseDMEntry(file *zip.File, result *archiveData, messageOffset, messageLim } continue } - if err := parseDMConversation(decoder, result, messageOffset, messageLimit); err != nil { + if err := parseDMConversation(decoder, result, messageOffset, messageLimit, isGroupSlice); err != nil { return err } } @@ -353,7 +360,7 @@ func parseDMEntry(file *zip.File, result *archiveData, messageOffset, messageLim return nil } -func parseDMConversation(decoder *json.Decoder, result *archiveData, messageOffset, messageLimit int) error { +func parseDMConversation(decoder *json.Decoder, result *archiveData, messageOffset, messageLimit int, isGroupSlice bool) error { if token, err := decoder.Token(); err != nil || token != json.Delim('{') { return fmt.Errorf("read dmConversation: %w", err) } @@ -441,7 +448,7 @@ func parseDMConversation(decoder *json.Decoder, result *archiveData, messageOffs } } result.messages = append(result.messages, selected...) - result.conversations[conversationID] = conversationRow{ID: conversationID, ParticipantIDs: participants, Title: firstNonEmpty(name, conversationTitle(participants, result.profiles)), LastMessageAt: lastMessageAt, IsGroup: len(participants) > 1 || name != ""} + result.conversations[conversationID] = conversationRow{ID: conversationID, ParticipantIDs: participants, Title: name, LastMessageAt: lastMessageAt, IsGroup: isGroupSlice || len(participants) > 1 || name != ""} return nil } diff --git a/native/helpers/birdclaw-go/main_test.go b/native/helpers/birdclaw-go/main_test.go index 99d7104d..23439816 100644 --- a/native/helpers/birdclaw-go/main_test.go +++ b/native/helpers/birdclaw-go/main_test.go @@ -15,11 +15,11 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { } writer := zip.NewWriter(file) entries := map[string]string{ - "archive/data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, - "archive/data/tweets.js": `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"Ava Chen"}]}}}]`, - "archive/data/direct-messages.js": `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"group-1","name":"Builders","messages":[{"participantsLeave":{"initiatingUserId":"30","userIds":["30"]}},{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}},{"participantsJoin":{"initiatingUserId":"20","userIds":["40"]}},{"joinConversation":{"participantsSnapshot":["10","20","30"]}}]}}]`, - "archive/data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]`, - "archive/data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, + "archive/data/account.js": `window.YTD.account.part0 = [{"account":{"accountId":"10","username":"sam","accountDisplayName":"Sam"}}]`, + "archive/data/tweets.js": `window.YTD.tweets.part0 = [{"tweet":{"entities":{"user_mentions":[{"id_str":"20","screen_name":"ava","name":"Ava Chen"}]}}}]`, + "archive/data/direct-messages-group.js": `window.YTD.direct_messages_group.part0 = [{"dmConversation":{"conversationId":"group-1","messages":[{"participantsLeave":{"initiatingUserId":"30","userIds":["30","40"]}},{"messageCreate":{"id":"m1","senderId":"20","recipientId":"10","text":"hello","createdAt":"2026-08-03T12:00:00.000Z"}},{"participantsJoin":{"initiatingUserId":"20","userIds":["40"]}},{"joinConversation":{"participantsSnapshot":["10","20","30"]}}]}}]`, + "archive/data/follower.js": `window.YTD.follower.part0 = [{"follower":{"accountId":"20"}}]`, + "archive/data/following.js": `window.YTD.following.part0 = [{"following":{"accountId":"20"}}]`, } for name, content := range entries { entry, err := writer.Create(name) @@ -48,9 +48,12 @@ func TestReadsArchiveDMsAndFollowGraph(t *testing.T) { t.Fatalf("messages = %#v", data.messages) } conversation := data.conversations["group-1"] - if len(conversation.ParticipantIDs) != 2 || conversation.ParticipantIDs[0] != "20" || conversation.ParticipantIDs[1] != "40" { + if len(conversation.ParticipantIDs) != 1 || conversation.ParticipantIDs[0] != "20" { t.Fatalf("conversation participants = %#v", conversation.ParticipantIDs) } + if !conversation.IsGroup || conversation.Title != "Ava Chen" { + t.Fatalf("conversation = %#v", conversation) + } profile := data.profiles["profile_user_20"] if profile.Handle != "ava" || profile.DisplayName != "Ava Chen" { t.Fatalf("profile = %#v", profile) diff --git a/src/platforms/x/e2e.test.ts b/src/platforms/x/e2e.test.ts index 5c22de55..edc6515c 100644 --- a/src/platforms/x/e2e.test.ts +++ b/src/platforms/x/e2e.test.ts @@ -30,7 +30,11 @@ function writeArchive( ); writeFileSync( join(source, "data/direct-messages.js"), - `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}},{"dmConversation":{"conversationId":"group-1","name":"${input.groupName}","messages":[${input.groupMember ? "" : '{"participantsLeave":{"initiatingUserId":"30","userIds":["30"],"createdAt":"2026-08-03T14:00:00.000Z"}},'}{"messageCreate":{"id":"group-message-1","senderId":"20","text":"Hello group","createdAt":"2026-08-03T13:00:00.000Z"}},{"joinConversation":{"participantsSnapshot":["10","20","30"],"createdAt":"2026-08-03T11:00:00.000Z"}}]}}]`, + `window.YTD.direct_messages.part0 = [{"dmConversation":{"conversationId":"10-20","messages":[{"messageCreate":{"id":"message-1","senderId":"20","recipientId":"10","text":"Hello from the X archive","createdAt":"2026-08-03T12:00:00.000Z"}}]}}]`, + ); + writeFileSync( + join(source, "data/direct-messages-group.js"), + `window.YTD.direct_messages_group.part0 = [{"dmConversation":{"conversationId":"group-1","name":"${input.groupName}","messages":[${input.groupMember ? "" : '{"participantsLeave":{"initiatingUserId":"30","userIds":["30"],"createdAt":"2026-08-03T14:00:00.000Z"}},'}{"messageCreate":{"id":"group-message-1","senderId":"20","text":"Hello group","createdAt":"2026-08-03T13:00:00.000Z"}},{"joinConversation":{"participantsSnapshot":["10","20","30"],"createdAt":"2026-08-03T11:00:00.000Z"}}]}}]`, ); writeFileSync( join(source, "data/follower.js"), @@ -147,6 +151,13 @@ describe("X archive end to end", () => { ) .get() as { id: string; name: string }; expect(groupBefore.name).toBe("Builders"); + expect( + sqlite + .prepare( + `SELECT name FROM conversations WHERE platform = 'x' AND source_conversation_key = 'x:conversation:10-20'`, + ) + .get(), + ).toMatchObject({ name: "Ava Chen" }); writeArchive(temp, { name: "Ava Patel",