diff --git a/native/helpers/whatsapp-go/main.go b/native/helpers/whatsapp-go/main.go index f912f274..0aa4547a 100644 --- a/native/helpers/whatsapp-go/main.go +++ b/native/helpers/whatsapp-go/main.go @@ -69,37 +69,31 @@ type chatSnapshot struct { } type attachmentSnapshot struct { - ID string `json:"id"` - Kind string `json:"kind"` - Filename *string `json:"filename,omitempty"` - MimeType *string `json:"mime_type,omitempty"` - SizeBytes *int64 `json:"size_bytes,omitempty"` - Text *string `json:"text,omitempty"` - AccessKind string `json:"access_kind"` - AccessRef map[string]interface{} `json:"access_ref,omitempty"` - Availability string `json:"availability_status,omitempty"` - ProviderMeta map[string]interface{} `json:"provider_metadata,omitempty"` + ID string `json:"id"` + Kind string `json:"kind"` + Filename *string `json:"filename,omitempty"` + MimeType *string `json:"mime_type,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + Text *string `json:"text,omitempty"` + AccessKind string `json:"access_kind"` + AccessRef map[string]interface{} `json:"access_ref,omitempty"` } type messageSnapshot struct { - MessageID string `json:"messageID"` - ChatJID string `json:"chatJID"` - SenderJID *string `json:"senderJID,omitempty"` - ParticipantJID *string `json:"participantJID,omitempty"` - FromMe bool `json:"fromMe"` - Timestamp int64 `json:"timestamp"` - Text string `json:"text"` - PushName *string `json:"pushName,omitempty"` - Status *string `json:"status,omitempty"` - DeliveredAt *int64 `json:"deliveredAt,omitempty"` - ReadAt *int64 `json:"readAt,omitempty"` - ReplyToID *string `json:"replyToMessageID,omitempty"` - ReplyToSender *string `json:"replyToSenderJID,omitempty"` - IsForwarded bool `json:"isForwarded,omitempty"` - ForwardingScore *uint32 `json:"forwardingScore,omitempty"` - Revoked bool `json:"revoked,omitempty"` - MessageProto *string `json:"messageProtoBase64,omitempty"` - Attachments []attachmentSnapshot `json:"attachments,omitempty"` + MessageID string `json:"messageID"` + ChatJID string `json:"chatJID"` + SenderJID *string `json:"senderJID,omitempty"` + ParticipantJID *string `json:"participantJID,omitempty"` + FromMe bool `json:"fromMe"` + Timestamp int64 `json:"timestamp"` + Text string `json:"text"` + PushName *string `json:"pushName,omitempty"` + Status *string `json:"status,omitempty"` + DeliveredAt *int64 `json:"deliveredAt,omitempty"` + ReadAt *int64 `json:"readAt,omitempty"` + Revoked bool `json:"revoked,omitempty"` + MessageProto *string `json:"messageProtoBase64,omitempty"` + Attachments []attachmentSnapshot `json:"attachments,omitempty"` } type downloadableMessageSnapshot struct { @@ -128,7 +122,6 @@ type callSnapshot struct { DurationSeconds *int64 `json:"durationSeconds,omitempty"` Status string `json:"status"` Medium string `json:"medium"` - ProviderType *string `json:"providerCallType,omitempty"` } type callDeleteSnapshot struct { @@ -158,8 +151,6 @@ type eventEnvelope struct { type commandEnvelope struct { ID int `json:"id"` Command string `json:"command"` - Target string `json:"target,omitempty"` - Text string `json:"text,omitempty"` Cursor string `json:"cursor,omitempty"` SinceMS *int64 `json:"sinceMs,omitempty"` Limit int `json:"limit,omitempty"` @@ -403,10 +394,6 @@ func openSyncCache(storeDir string) (*syncCache, error) { status TEXT, delivered_at INTEGER, read_at INTEGER, - reply_to_message_id TEXT, - reply_to_sender_jid TEXT, - is_forwarded INTEGER NOT NULL DEFAULT 0, - forwarding_score INTEGER, revoked INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (chat_jid, message_id) @@ -429,7 +416,6 @@ func openSyncCache(storeDir string) (*syncCache, error) { duration_seconds INTEGER, status TEXT NOT NULL, medium TEXT NOT NULL, - provider_call_type TEXT, updated_at INTEGER NOT NULL, PRIMARY KEY (chat_jid, call_id) )`, @@ -453,10 +439,6 @@ func openSyncCache(storeDir string) (*syncCache, error) { name string definition string }{ - {name: "reply_to_message_id", definition: "TEXT"}, - {name: "reply_to_sender_jid", definition: "TEXT"}, - {name: "is_forwarded", definition: "INTEGER NOT NULL DEFAULT 0"}, - {name: "forwarding_score", definition: "INTEGER"}, {name: "revoked", definition: "INTEGER NOT NULL DEFAULT 0"}, } { if err := ensureSQLiteColumn(db, "messages", column.name, column.definition); err != nil { @@ -464,14 +446,53 @@ func openSyncCache(storeDir string) (*syncCache, error) { return nil, err } } + for _, column := range []struct { + table string + name string + }{ + {table: "messages", name: "reply_to_message_id"}, + {table: "messages", name: "reply_to_sender_jid"}, + {table: "messages", name: "is_forwarded"}, + {table: "messages", name: "forwarding_score"}, + {table: "calls", name: "provider_call_type"}, + } { + if err := dropSQLiteColumnIfExists(db, column.table, column.name); err != nil { + _ = db.Close() + return nil, err + } + } return &syncCache{db: db}, nil } func ensureSQLiteColumn(db *sql.DB, table string, column string, definition string) error { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + exists, err := sqliteColumnExistsInDB(db, table, column) + if err != nil { + return err + } + if exists { + return nil + } + _, err = db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, definition)) + return err +} + +func dropSQLiteColumnIfExists(db *sql.DB, table string, column string) error { + exists, err := sqliteColumnExistsInDB(db, table, column) if err != nil { return err } + if !exists { + return nil + } + _, err = db.Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", table, column)) + return err +} + +func sqliteColumnExistsInDB(db *sql.DB, table string, column string) (bool, error) { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return false, err + } defer rows.Close() for rows.Next() { var cid int @@ -481,17 +502,16 @@ func ensureSQLiteColumn(db *sql.DB, table string, column string, definition stri var defaultValue sql.NullString var primaryKey int if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { - return err + return false, err } if name == column { - return nil + return true, nil } } if err := rows.Err(); err != nil { - return err + return false, err } - _, err = db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, definition)) - return err + return false, nil } func (c *syncCache) close() error { @@ -867,7 +887,7 @@ func (c *syncCache) queryMessages( _ = tx.Rollback() }() - selectQuery := `SELECT chat_jid, message_id, sender_jid, participant_jid, from_me, timestamp, text, push_name, status, delivered_at, read_at, reply_to_message_id, reply_to_sender_jid, is_forwarded, forwarding_score, revoked FROM messages` + selectQuery := `SELECT chat_jid, message_id, sender_jid, participant_jid, from_me, timestamp, text, push_name, status, delivered_at, read_at, revoked FROM messages` args := []any{} if cursor == nil || cursor.SnapshotUpdatedAt <= 0 { return nil, nil, false, errors.New("invalid resync cursor") @@ -901,11 +921,11 @@ func (c *syncCache) queryMessages( for rows.Next() { var chatJID string var messageID string - var senderJID, participantJID, pushName, status, replyToID, replyToSender sql.NullString - var fromMe, isForwarded, revoked int64 + var senderJID, participantJID, pushName, status sql.NullString + var fromMe, revoked int64 var timestamp int64 var text string - var deliveredAt, readAt, forwardingScore sql.NullInt64 + var deliveredAt, readAt sql.NullInt64 if err := rows.Scan( &chatJID, &messageID, @@ -918,10 +938,6 @@ func (c *syncCache) queryMessages( &status, &deliveredAt, &readAt, - &replyToID, - &replyToSender, - &isForwarded, - &forwardingScore, &revoked, ); err != nil { return nil, nil, false, err @@ -951,17 +967,6 @@ func (c *syncCache) queryMessages( if readAt.Valid { message.ReadAt = int64Ptr(readAt.Int64) } - if replyToID.Valid { - message.ReplyToID = stringPtr(replyToID.String) - } - if replyToSender.Valid { - message.ReplyToSender = stringPtr(replyToSender.String) - } - message.IsForwarded = isForwarded == 1 - if forwardingScore.Valid { - score := uint32(forwardingScore.Int64) - message.ForwardingScore = &score - } message.Revoked = revoked == 1 messages = append(messages, message) } @@ -993,7 +998,7 @@ func (c *syncCache) queryCalls( cursor *resyncCursor, limit int, ) ([]callSnapshot, *string, bool, error) { - query := `SELECT call_id, chat_jid, initiator_jid, remote_jid, from_me, timestamp, duration_seconds, status, medium, provider_call_type FROM calls` + query := `SELECT call_id, chat_jid, initiator_jid, remote_jid, from_me, timestamp, duration_seconds, status, medium FROM calls` args := []any{} if cursor == nil || cursor.SnapshotUpdatedAt <= 0 { return nil, nil, false, errors.New("invalid resync cursor") @@ -1029,7 +1034,7 @@ func (c *syncCache) queryCalls( var calls []callSnapshot for rows.Next() { var call callSnapshot - var initiatorJID, remoteJID, providerType sql.NullString + var initiatorJID, remoteJID sql.NullString var fromMe int64 var durationSeconds sql.NullInt64 if err := rows.Scan( @@ -1042,7 +1047,6 @@ func (c *syncCache) queryCalls( &durationSeconds, &call.Status, &call.Medium, - &providerType, ); err != nil { return nil, nil, false, err } @@ -1055,9 +1059,6 @@ func (c *syncCache) queryCalls( if durationSeconds.Valid { call.DurationSeconds = int64Ptr(durationSeconds.Int64) } - if providerType.Valid { - call.ProviderType = stringPtr(providerType.String) - } call.FromMe = fromMe == 1 calls = append(calls, call) } @@ -1126,9 +1127,8 @@ func upsertMessageTx(tx *sql.Tx, message messageSnapshot, updatedAt int64) error _, err := tx.Exec( `INSERT INTO messages ( chat_jid, message_id, sender_jid, participant_jid, from_me, timestamp, text, push_name, - status, delivered_at, read_at, reply_to_message_id, reply_to_sender_jid, is_forwarded, - forwarding_score, revoked, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + status, delivered_at, read_at, revoked, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(chat_jid, message_id) DO UPDATE SET sender_jid = excluded.sender_jid, participant_jid = excluded.participant_jid, @@ -1139,10 +1139,6 @@ func upsertMessageTx(tx *sql.Tx, message messageSnapshot, updatedAt int64) error status = excluded.status, delivered_at = COALESCE(excluded.delivered_at, messages.delivered_at), read_at = COALESCE(excluded.read_at, messages.read_at), - reply_to_message_id = excluded.reply_to_message_id, - reply_to_sender_jid = excluded.reply_to_sender_jid, - is_forwarded = excluded.is_forwarded, - forwarding_score = excluded.forwarding_score, revoked = excluded.revoked, updated_at = excluded.updated_at`, normalizeJID(message.ChatJID), @@ -1156,10 +1152,6 @@ func upsertMessageTx(tx *sql.Tx, message messageSnapshot, updatedAt int64) error nullableString(message.Status), nullableInt64(message.DeliveredAt), nullableInt64(message.ReadAt), - nullableString(message.ReplyToID), - nullableString(message.ReplyToSender), - boolToInt(message.IsForwarded), - nullableUint32(message.ForwardingScore), boolToInt(message.Revoked), updatedAt, ) @@ -1170,8 +1162,8 @@ func upsertCallTx(tx *sql.Tx, call callSnapshot, updatedAt int64) error { _, err := tx.Exec( `INSERT INTO calls ( chat_jid, call_id, initiator_jid, remote_jid, from_me, timestamp, duration_seconds, - status, medium, provider_call_type, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + status, medium, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(chat_jid, call_id) DO UPDATE SET initiator_jid = excluded.initiator_jid, remote_jid = excluded.remote_jid, @@ -1180,7 +1172,6 @@ func upsertCallTx(tx *sql.Tx, call callSnapshot, updatedAt int64) error { duration_seconds = COALESCE(excluded.duration_seconds, calls.duration_seconds), status = excluded.status, medium = excluded.medium, - provider_call_type = excluded.provider_call_type, updated_at = excluded.updated_at`, normalizeJID(call.ChatJID), call.CallID, @@ -1191,7 +1182,6 @@ func upsertCallTx(tx *sql.Tx, call callSnapshot, updatedAt int64) error { nullableInt64(call.DurationSeconds), call.Status, call.Medium, - nullableString(call.ProviderType), updatedAt, ) return err @@ -1745,9 +1735,6 @@ func (r *helperRuntime) readCommands(ctx context.Context, errs chan<- error) { case "downloadMedia": result, err := r.downloadMedia(ctx, command.ChatJID, command.MessageID, command.AttachmentIndex) r.writeResponse(command.ID, err == nil, result, err) - case "sendText": - result, err := r.sendText(ctx, command.Target, command.Text) - r.writeResponse(command.ID, err == nil, result, err) default: r.writeResponse(command.ID, false, nil, fmt.Errorf("unsupported command: %s", command.Command)) } @@ -1848,42 +1835,6 @@ doneWaiting: return result, nil } -func (r *helperRuntime) sendText(ctx context.Context, target string, text string) (map[string]interface{}, error) { - targetJID, err := types.ParseJID(target) - if err != nil { - return nil, err - } - - message := &waProto.Message{ - Conversation: proto.String(text), - } - resp, err := r.client.SendMessage(ctx, targetJID, message) - if err != nil { - return nil, err - } - - chatJID := normalizeJID(targetJID.String()) - messageID := resp.ID - timestamp := time.Now().UnixMilli() - snapshot := messageSnapshot{ - MessageID: messageID, - ChatJID: chatJID, - SenderJID: stringPtr(jidString(r.client.Store.ID)), - FromMe: true, - Timestamp: timestamp, - Text: text, - Status: stringPtr("sent"), - Attachments: []attachmentSnapshot{}, - } - r.state.setMessage(snapshot) - - return map[string]interface{}{ - "messageID": messageID, - "chatJID": chatJID, - "timestamp": timestamp, - }, nil -} - func (r *helperRuntime) downloadMedia(ctx context.Context, chatJID string, messageID string, attachmentIndex int) (map[string]interface{}, error) { if attachmentIndex != 0 { return nil, fmt.Errorf("unsupported attachment index: %d", attachmentIndex) @@ -2252,25 +2203,20 @@ func messageFromEvent(event *events.Message) messageSnapshot { status = "sent" } attachments, messageProto := attachmentsFromMessage(messageID, chatJID, effectiveMessage) - replyToID, replyToSender, isForwarded, forwardingScore := messageContextMetadata(effectiveMessage) revoked := messageRevoked(effectiveMessage) return messageSnapshot{ - MessageID: messageID, - ChatJID: chatJID, - SenderJID: stringPtr(senderJID), - ParticipantJID: participantJID, - FromMe: event.Info.IsFromMe, - Timestamp: event.Info.Timestamp.UnixMilli(), - Text: extractText(effectiveMessage), - PushName: pushName, - Status: &status, - ReplyToID: replyToID, - ReplyToSender: replyToSender, - IsForwarded: isForwarded, - ForwardingScore: forwardingScore, - Revoked: revoked, - MessageProto: messageProto, - Attachments: attachments, + MessageID: messageID, + ChatJID: chatJID, + SenderJID: stringPtr(senderJID), + ParticipantJID: participantJID, + FromMe: event.Info.IsFromMe, + Timestamp: event.Info.Timestamp.UnixMilli(), + Text: extractText(effectiveMessage), + PushName: pushName, + Status: &status, + Revoked: revoked, + MessageProto: messageProto, + Attachments: attachments, } } @@ -2302,10 +2248,6 @@ func attachmentsFromMessage(messageID string, chatJID string, message *waProto.M "messageID": messageID, "attachmentIndex": 0, }, - Availability: "available", - ProviderMeta: map[string]interface{}{ - "kind": kind, - }, } } @@ -2573,22 +2515,6 @@ func contextInfoFromMessage(message *waProto.Message) *waProto.ContextInfo { return nil } -func messageContextMetadata(message *waProto.Message) (*string, *string, bool, *uint32) { - contextInfo := contextInfoFromMessage(message) - if contextInfo == nil { - return nil, nil, false, nil - } - var replyToID *string - if stanzaID := strings.TrimSpace(contextInfo.GetStanzaID()); stanzaID != "" { - replyToID = stringPtr(stanzaID) - } - var replyToSender *string - if participant := normalizeJID(contextInfo.GetParticipant()); participant != "" { - replyToSender = stringPtr(participant) - } - return replyToID, replyToSender, contextInfo.GetIsForwarded(), uint32PtrOrNil(contextInfo.GetForwardingScore()) -} - func messageRevoked(message *waProto.Message) bool { _, ok := revokedMessageKey(message) return ok @@ -2663,26 +2589,21 @@ func historyMessageSnapshot( status = "sent" } attachments, messageProto := attachmentsFromMessage(messageID, normalizedChatJID, effectiveMessage) - replyToID, replyToSender, isForwarded, forwardingScore := messageContextMetadata(effectiveMessage) revoked := messageRevoked(effectiveMessage) return messageSnapshot{ - MessageID: messageID, - ChatJID: normalizedChatJID, - SenderJID: stringPtr(sender), - ParticipantJID: participantJID, - FromMe: fromMe, - Timestamp: int64(webMsg.GetMessageTimestamp()) * 1000, - Text: extractText(effectiveMessage), - PushName: emptyToNil(webMsg.GetPushName()), - Status: stringPtr(status), - ReplyToID: replyToID, - ReplyToSender: replyToSender, - IsForwarded: isForwarded, - ForwardingScore: forwardingScore, - Revoked: revoked, - MessageProto: messageProto, - Attachments: attachments, + MessageID: messageID, + ChatJID: normalizedChatJID, + SenderJID: stringPtr(sender), + ParticipantJID: participantJID, + FromMe: fromMe, + Timestamp: int64(webMsg.GetMessageTimestamp()) * 1000, + Text: extractText(effectiveMessage), + PushName: emptyToNil(webMsg.GetPushName()), + Status: stringPtr(status), + Revoked: revoked, + MessageProto: messageProto, + Attachments: attachments, }, true } @@ -2762,7 +2683,6 @@ func callSnapshotFromRecord(client *whatsmeow.Client, record *waSyncAction.CallL DurationSeconds: int64PtrIfPositive(record.GetDuration()), Status: callStatus(record.GetCallResult()), Medium: callMedium(record.GetIsVideo()), - ProviderType: stringPtr(strings.ToLower(record.GetCallType().String())), }, true } @@ -3067,13 +2987,6 @@ func nullableInt64(value *int64) interface{} { return *value } -func nullableUint32(value *uint32) interface{} { - if value == nil { - return nil - } - return int64(*value) -} - func boolToInt(value bool) int { if value { return 1 diff --git a/native/helpers/whatsapp-go/main_test.go b/native/helpers/whatsapp-go/main_test.go index bf0b05c7..cc6e110b 100644 --- a/native/helpers/whatsapp-go/main_test.go +++ b/native/helpers/whatsapp-go/main_test.go @@ -497,44 +497,7 @@ func TestExtractTextUnwrapsFutureProofEditedMessage(t *testing.T) { } } -func TestHistoryMessageSnapshotFallbackRetainsReplyForwardAndRevokeMetadata(t *testing.T) { - forwardingScore := uint32(2) - historyMsg := &waHistorySync.HistorySyncMsg{ - Message: &waWeb.WebMessageInfo{ - Key: &waCommon.MessageKey{ - ID: proto.String("wamid-reply"), - RemoteJID: proto.String("15551234567@s.whatsapp.net"), - FromMe: proto.Bool(false), - }, - MessageTimestamp: proto.Uint64(1_710_000_000), - Message: &waProto.Message{ - ExtendedTextMessage: &waProto.ExtendedTextMessage{ - Text: proto.String("reply"), - ContextInfo: &waProto.ContextInfo{ - StanzaID: proto.String("wamid-parent"), - Participant: proto.String("15557654321@s.whatsapp.net"), - IsForwarded: proto.Bool(true), - ForwardingScore: proto.Uint32(forwardingScore), - }, - }, - }, - }, - } - - snapshot, ok := historyMessageSnapshot(nil, types.NewJID("15551234567", types.DefaultUserServer), historyMsg) - if !ok { - t.Fatal("expected fallback history snapshot") - } - if snapshot.ReplyToID == nil || *snapshot.ReplyToID != "wamid-parent" { - t.Fatalf("expected reply id metadata, got %+v", snapshot) - } - if snapshot.ReplyToSender == nil || *snapshot.ReplyToSender != "15557654321@s.whatsapp.net" { - t.Fatalf("expected reply sender metadata, got %+v", snapshot) - } - if !snapshot.IsForwarded || snapshot.ForwardingScore == nil || *snapshot.ForwardingScore != forwardingScore { - t.Fatalf("expected forwarded metadata, got %+v", snapshot) - } - +func TestHistoryMessageSnapshotFallbackRetainsRevokeMetadata(t *testing.T) { revokeType := waProto.ProtocolMessage_REVOKE revokeSnapshot, ok := historyMessageSnapshot(nil, types.NewJID("15551234567", types.DefaultUserServer), &waHistorySync.HistorySyncMsg{ Message: &waWeb.WebMessageInfo{ @@ -688,6 +651,8 @@ func TestOpenSyncCacheMigratesLegacyMessageColumns(t *testing.T) { message_id TEXT NOT NULL, sender_jid TEXT, participant_jid TEXT, + reply_to_message_id TEXT, + reply_to_sender_jid TEXT, from_me INTEGER NOT NULL, timestamp INTEGER NOT NULL, text TEXT NOT NULL, @@ -695,6 +660,8 @@ func TestOpenSyncCacheMigratesLegacyMessageColumns(t *testing.T) { status TEXT, delivered_at INTEGER, read_at INTEGER, + is_forwarded INTEGER NOT NULL DEFAULT 0, + forwarding_score INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (chat_jid, message_id) )`) @@ -717,6 +684,23 @@ func TestOpenSyncCacheMigratesLegacyMessageColumns(t *testing.T) { if err != nil { t.Fatalf("insert legacy message: %v", err) } + _, err = db.Exec(`CREATE TABLE calls ( + chat_jid TEXT NOT NULL, + call_id TEXT NOT NULL, + initiator_jid TEXT, + remote_jid TEXT, + from_me INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + duration_seconds INTEGER, + status TEXT NOT NULL, + medium TEXT NOT NULL, + provider_call_type TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (chat_jid, call_id) + )`) + if err != nil { + t.Fatalf("create legacy calls table: %v", err) + } if err := db.Close(); err != nil { t.Fatalf("close legacy cache db: %v", err) } @@ -727,17 +711,24 @@ func TestOpenSyncCacheMigratesLegacyMessageColumns(t *testing.T) { } defer cache.close() + for _, column := range []string{"revoked"} { + if !sqliteColumnExists(t, cache.db, "messages", column) { + t.Fatalf("expected migrated messages.%s column", column) + } + } for _, column := range []string{ "reply_to_message_id", "reply_to_sender_jid", "is_forwarded", "forwarding_score", - "revoked", } { - if !sqliteColumnExists(t, cache.db, "messages", column) { - t.Fatalf("expected migrated messages.%s column", column) + if sqliteColumnExists(t, cache.db, "messages", column) { + t.Fatalf("expected migrated messages.%s column to be removed", column) } } + if sqliteColumnExists(t, cache.db, "calls", "provider_call_type") { + t.Fatalf("expected migrated calls.provider_call_type column to be removed") + } messages, _, _, err := cache.queryMessages(nil, &resyncCursor{SnapshotUpdatedAt: 1_710_000_000_001}, 10) if err != nil { @@ -746,11 +737,8 @@ func TestOpenSyncCacheMigratesLegacyMessageColumns(t *testing.T) { if len(messages) != 1 { t.Fatalf("expected legacy message to remain queryable, got %+v", messages) } - if messages[0].IsForwarded || messages[0].Revoked { - t.Fatalf("expected legacy message defaults to be false, got %+v", messages[0]) - } - if messages[0].ForwardingScore != nil || messages[0].ReplyToID != nil || messages[0].ReplyToSender != nil { - t.Fatalf("expected legacy nullable metadata defaults, got %+v", messages[0]) + if messages[0].Revoked { + t.Fatalf("expected legacy message revoked default to be false, got %+v", messages[0]) } } @@ -1179,10 +1167,6 @@ func TestApplySnapshotDoesNotRewriteUnchangedMediaFile(t *testing.T) { "messageID": "message-1", "attachmentIndex": 0, }, - ProviderMeta: map[string]interface{}{ - "kind": "image", - "width": 640, - }, }}, }, { @@ -1198,10 +1182,6 @@ func TestApplySnapshotDoesNotRewriteUnchangedMediaFile(t *testing.T) { "messageID": "message-2", "attachmentIndex": 0, }, - ProviderMeta: map[string]interface{}{ - "kind": "document", - "pageCount": 3, - }, }}, }, }, @@ -1253,10 +1233,6 @@ func TestApplySnapshotDoesNotRewriteUnchangedMediaFileAfterReload(t *testing.T) "messageID": "message-1", "attachmentIndex": 0, }, - ProviderMeta: map[string]interface{}{ - "kind": "image", - "width": 640, - }, }}, }, }, diff --git a/native/macos/CuedNative/Sources/CuedNative/main.swift b/native/macos/CuedNative/Sources/CuedNative/main.swift index f87cb64d..9a8b87c0 100644 --- a/native/macos/CuedNative/Sources/CuedNative/main.swift +++ b/native/macos/CuedNative/Sources/CuedNative/main.swift @@ -130,14 +130,11 @@ struct CallHistoryRecord: Encodable { let remoteAddress: String? let remoteDisplayName: String? let provider: String - let providerCallType: String? let direction: String let medium: String let status: String let startedAt: Int - let endedAt: Int? let durationSeconds: Int? - let disconnectedCause: String? let syntheticConversation: Bool } @@ -1020,13 +1017,6 @@ struct Command { } else { 0 } - let endedAt: Int? = - if let durationSeconds { - startedAt + durationSeconds * 1000 - } else { - nil - } - return CallHistoryRecord( pk: row.pk, sourceCallKey: nilIfEmpty(row.uniqueID) ?? "callhistory:\(row.pk)", @@ -1035,7 +1025,6 @@ struct Command { remoteAddress: remoteAddress, remoteDisplayName: nilIfEmpty(row.name), provider: provider, - providerCallType: row.callType.map(String.init), direction: direction, medium: normalizeCallMedium(provider: provider, callType: row.callType), status: normalizeCallStatus( @@ -1045,9 +1034,7 @@ struct Command { durationSeconds: durationSeconds ), startedAt: startedAt, - endedAt: endedAt, durationSeconds: durationSeconds, - disconnectedCause: row.disconnectedCause.map(String.init), syntheticConversation: chatID == nil ) } diff --git a/skills/cued/SKILL.md b/skills/cued/SKILL.md index 0758d445..e6da18f0 100644 --- a/skills/cued/SKILL.md +++ b/skills/cued/SKILL.md @@ -36,14 +36,14 @@ Do not use `sqlite3 ~/.cued/local.db` unless the user explicitly asks to debug r This is not a full schema reference. Use these tables when CLI commands are too coarse and you need exact counts, joins, ranking, or provenance. Prefer `cued` commands for mutations and attachment fetches. - `contacts`: canonical people/entities. Key fields: `id`, `name`, `company`, `archived`. -- `contact_handles`: email/phone/platform handles. Key fields: `contact_id`, `type`, `value`, `normalized_value`, `platform`, `is_deterministic`. -- `contact_sources`: where a contact came from. Key fields: `contact_id`, `platform`, `account_key`, `source_entity_key`, `profile_url`, `first_seen_at`, `last_seen_at`. +- `contact_handles`: email/phone/platform handles. Key fields: `contact_id`, `type`, `value`, `normalized_value`, `is_deterministic`. +- `contact_sources`: where a contact came from. Key fields: `contact_id`, `platform`, `account_key`, `source_entity_key`. - `contact_memories`: durable agent-written context. Query it for current memories with `stale_at IS NULL`; write through `cued contacts memory ...`, not SQL. -- `conversations`: canonical threads. Key fields: `id`, `platform`, `account_key`, `type`, `name`, `participant_names`, `last_message_at`, `last_message_preview`, `unread_count`. +- `conversations`: canonical threads. Key fields: `id`, `platform`, `account_key`, `source_conversation_key`, `type`, `is_active`, `name`, `participant_names`. - `conversation_participants`: contact membership in threads. Key fields: `conversation_id`, `contact_id`, `participant_name`, `is_self`, `is_active`. -- `messages`: canonical message rows. Key fields: `id`, `platform`, `conversation_id`, `sender_contact_id`, `sender_name`, `sent_at`, `is_from_me`, `content`, `attachment_count`, `reaction_count`, `reply_to_message_id`. -- `message_reactions`: reactions/tapbacks. Key fields: `message_id`, `reactor_contact_id`, `reactor_name`, `emoji`, `is_active`, `created_at`. -- `message_attachments`: attachment metadata. Key fields: `id`, `message_id`, `platform`, `kind`, `mime_type`, `filename`, `title`, `size_bytes`, `text_content`, `access_kind`, `availability_status`. +- `messages`: canonical message rows. Key fields: `id`, `platform`, `conversation_id`, `sender_contact_id`, `sender_source_key`, `sender_name`, `conversation_name`, `sent_at`, `status`, `is_from_me`, `content`, `delivered_at`, `read_at`, `is_deleted`. +- `message_reactions`: reactions/tapbacks. Key fields: `message_id`, `reactor_source_key`, `emoji`, `is_active`, `created_at`. +- `message_attachments`: attachment metadata. Key fields: `id`, `message_id`, `platform`, `kind`, `mime_type`, `filename`, `title`, `size_bytes`, `text_content`, `access_kind`, `access_ref_json`. - `attachment_content`: extracted text for fetched attachments. Key fields: `attachment_id`, `status`, `text_content`, `mime_type`, `extracted_at`, `last_error`. Prefer `cued attachments search`. - `messages_fts`: FTS5 over message search fields: `sender_name`, `conversation_name`, `participant_names`, `attachment_text`, `content`. - `attachment_content_fts`: FTS5 over extracted attachment text. Prefer `cued attachments search `. @@ -55,9 +55,9 @@ Useful SQL examples: ```bash cued sql "select count(*) as messages from messages" cued sql "select platform, count(*) as messages from messages group by platform order by messages desc" -cued sql "select id, platform, name, participant_names, datetime(last_message_at/1000,'unixepoch','localtime') as last_message from conversations order by last_message_at desc limit 20" -cued sql "select id, sender_name, conversation_name, datetime(sent_at/1000,'unixepoch','localtime') as sent, content, attachment_count from messages where conversation_id = 'conversation-id-here' order by sent_at desc limit 50" -cued sql "select ma.id, ma.kind, ma.mime_type, ma.filename, ma.size_bytes, ma.access_kind, ma.availability_status from message_attachments ma where ma.message_id = 'message-id-here'" +cued sql "select c.id, c.platform, c.name, c.participant_names, datetime(max(m.sent_at)/1000,'unixepoch','localtime') as last_message from conversations c left join messages m on m.conversation_id = c.id group by c.id order by max(m.sent_at) desc limit 20" +cued sql "select m.id, m.sender_name, m.conversation_name, datetime(m.sent_at/1000,'unixepoch','localtime') as sent, m.content, (select count(*) from message_attachments ma where ma.message_id = m.id) as attachments from messages m where m.conversation_id = 'conversation-id-here' order by m.sent_at desc limit 50" +cued sql "select ma.id, ma.kind, ma.mime_type, ma.filename, ma.size_bytes, ma.access_kind from message_attachments ma where ma.message_id = 'message-id-here'" ``` ## Attachments @@ -82,12 +82,12 @@ cued attachments search "search terms" --limit 20 ``` Safety rules: -- Inspect `filename`, `mime_type`, `size_bytes`, `access_kind`, and `availability_status` before fetching. +- Inspect `filename`, `mime_type`, `size_bytes`, and `access_kind` before fetching. - The default fetch path has a conservative byte ceiling to prevent accidental large downloads. - Use `--allow-large` only when the user explicitly asks for the file or the task clearly depends on the bytes. - Avoid fetching video, audio, archives, disk images, and opaque binary files unless the user asks for the file itself; agents usually cannot inspect them usefully. - Use `--max-bytes` when you intentionally want a stricter ceiling for a one-off fetch. -- Treat `metadata_only`, `none`, or missing fetch coordinates as not currently fetchable. +- Treat `none` or missing fetch coordinates as not currently fetchable. - Never paste private attachment text into fixtures or broad summaries; summarize only what is needed. ## Relationship Patterns @@ -96,7 +96,7 @@ Safety rules: - **Follow-up needed**: DM where I sent the last message, got no reply or reaction, and it's 3+ days old. If they reacted but didn't reply, it's lower priority. - **Dormant relationship**: Contact with significant message history (10+ messages) but no messages in 30+ days. - **Network search**: Use `messages_fts` to find contacts who've discussed a topic, grouped by `sender_contact_id`. -- **Unread triage**: Conversations with `unread_count > 0` where the last message is not from me. Prioritize DMs over groups. +- **Unread triage**: Messages where `read_at IS NULL` and `is_from_me = 0`, grouped by conversation. Prioritize DMs over groups and use the newest unread `sent_at` as the recency signal. ## Contact Management @@ -144,7 +144,7 @@ Enrichment is a local-first identity task, not a web search task. Prioritize con Queue candidates in this order: - high-recency or high-volume human DMs with missing company/profile context; -- contacts with deterministic handles or profile URLs already in `contact_handles` / `contact_sources`; +- contacts with deterministic handles or profile URLs already in `contact_handles`, or platform identity in `contact_sources`; - duplicate clusters where exact handles can enrich the canonical record; - high-value contacts from recent meetings, calls, or messages that still lack public profile context. @@ -152,7 +152,7 @@ Deprioritize or skip service senders, newsletters, stores, bots, OTP/verificatio Use this source order: 1. Local identity evidence: exact email, phone, LinkedIn URL/id, platform user id, profile URL, message history. -2. Existing public profile URL from `contact_sources.profile_url` or `contact_handles`. +2. Existing public profile URL or handle in `contact_handles`, or platform identity in `contact_sources.source_entity_key`. 3. Web search for the exact profile handle or exact full name plus known affiliation. 4. Cross-profile links from a verified source, such as a personal site linking GitHub/X/LinkedIn. diff --git a/skills/cued/evals/contact-memories.md b/skills/cued/evals/contact-memories.md index d1b4c5ea..adda945f 100644 --- a/skills/cued/evals/contact-memories.md +++ b/skills/cued/evals/contact-memories.md @@ -28,7 +28,7 @@ handle_counts AS ( SELECT contact_id, SUM(CASE WHEN is_deterministic = 1 THEN 1 ELSE 0 END) AS deterministic_handles, - GROUP_CONCAT(DISTINCT type || ':' || COALESCE(platform, '') || ':' || value) AS handles + GROUP_CONCAT(DISTINCT type || ':' || value) AS handles FROM contact_handles GROUP BY contact_id ) diff --git a/skills/cued/evals/evals.json b/skills/cued/evals/evals.json index 4542f925..7afe45a6 100644 --- a/skills/cued/evals/evals.json +++ b/skills/cued/evals/evals.json @@ -101,10 +101,10 @@ "expected_output": "A prioritized list of conversations with unread messages, ranked by importance signals like relationship strength and message volume.", "files": [], "expectations": [ - "Queried conversations where unread_count > 0", - "Filtered to conversations where the last message was NOT sent by the user (is_from_me = 0)", + "Queried unread inbound messages where read_at IS NULL and is_from_me = 0", + "Grouped unread messages by conversation and checked the latest unread sender", "Ordered by some importance signal (message frequency, recency, or DM vs group)", - "Returned conversation names, platforms, unread counts, and last message previews", + "Returned conversation names, platforms, unread message counts, and latest unread message previews", "Distinguished between DM and group conversations, prioritizing DMs" ] }, @@ -266,7 +266,7 @@ "files": [], "expectations": [ "Skill was triggered by a generic message check without mentioning 'cued'", - "Queried recent messages or conversations with unread_count > 0", + "Queried recent messages and unread inbound messages using read_at IS NULL", "Returned conversation names, platforms, and message previews", "Ordered by recency" ] @@ -305,7 +305,7 @@ "Skill was triggered by a contact info lookup without mentioning 'cued'", "Searched contacts for 'Sarah Chen'", "Queried contact_handles for type = 'phone' and type = 'linkedin'", - "Checked contact_sources for profile_url containing linkedin", + "Checked contact_handles and contact_sources source_entity_key values for LinkedIn identifiers", "Returned the handle values" ] }, @@ -396,7 +396,7 @@ "Identified a bounded set of contacts or duplicate clusters with missing or sparse profile fields", "Structured the output as independent per-contact or per-cluster enrichment tasks rather than one monolithic pass", "Searched contact_handles for email, phone, linkedin, and other relevant handle types", - "Checked contact_sources.profile_url and related metadata for enrichment signals", + "Checked contact_handles and contact_sources source_entity_key values for enrichment signals", "Pulled enrichment candidates from duplicate records or cross-platform matches in local data", "Returned only enrichments supported by local evidence", "Did not claim that data had already been updated unless a real command was run" @@ -421,7 +421,7 @@ "expected_output": "A list of contacts with latent enrichment opportunities from existing local metadata, including LinkedIn URLs/handles, company names, and other profile fields that can be surfaced.", "files": [], "expectations": [ - "Queried contact_sources for profile_url and platform metadata that imply enrichable profile details", + "Queried contact_handles and contact_sources source_entity_key values that imply enrichable profile details", "Queried contact_handles for linkedin, email, phone, and other relevant handle types", "Matched local metadata back to the canonical contact", "Returned concrete field/value enrichments rather than vague suggestions", diff --git a/src/core/types/provider.test.ts b/src/core/types/provider.test.ts deleted file mode 100644 index 637ae5df..00000000 --- a/src/core/types/provider.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { normalizeRawEventProvenance } from "./provider.js"; - -describe("raw event provenance", () => { - it("preserves the supported provenance fields", () => { - const provenance = normalizeRawEventProvenance({ - acquisitionMode: "realtime", - providerApiVersion: "2026-03", - } as Parameters[0]); - - expect(provenance).toEqual({ - providerApiVersion: "2026-03", - adapterVersion: null, - acquisitionMode: "realtime", - }); - }); - - it("returns null when provenance is empty", () => { - const provenance = normalizeRawEventProvenance( - {} as Parameters[0], - ); - - expect(provenance).toBeNull(); - }); -}); diff --git a/src/core/types/provider.ts b/src/core/types/provider.ts index 1eb9082b..21610a7b 100644 --- a/src/core/types/provider.ts +++ b/src/core/types/provider.ts @@ -1,9 +1,4 @@ -import type { - ConversationType, - Platform, - RawEventEntityKind, - SyncMode, -} from "../../platforms/core/types.js"; +import type { ConversationType, Platform, RawEventEntityKind } from "../../platforms/core/types.js"; export * from "../../platforms/core/types.js"; @@ -22,7 +17,6 @@ export interface ContactObservationPayload { sourceEntityKey: string; fields: ContactFields; handles: ContactHandleInput[]; - sourceProfileUrl?: string | null; } export interface ConversationParticipantInput { @@ -34,11 +28,6 @@ export interface ConversationObservationPayload { sourceConversationKey: string; conversationType: ConversationType; displayName?: string | null; - nativeConversationKey?: string | null; - service?: string | null; - topic?: string | null; - unreadCount?: number | null; - removalReason?: string | null; participants: ConversationParticipantInput[]; } @@ -48,15 +37,10 @@ export interface MessagePayload { senderSourceKey: string | null; sentAt: number; content: string; - service?: string | null; status?: string | null; isFromMe?: boolean; deliveredAt?: number | null; readAt?: number | null; - editedAt?: number | null; - deletedAt?: number | null; - replyToSourceMessageKey?: string | null; - isEdited?: boolean; isDeleted?: boolean; attachments?: Array>; } @@ -66,7 +50,6 @@ export interface ReactionPayload { sourceConversationKey: string; reactorSourceKey: string | null; emoji: string; - reactionType?: string | null; timestamp: number; isActive: boolean; } @@ -74,10 +57,7 @@ export interface ReactionPayload { export interface ParticipantPayload { sourceConversationKey: string; participantSourceKey: string; - eventAt: number; isSelf?: boolean; - role?: string | null; - metadata?: Record; } export interface TimelineEventPayload { @@ -88,7 +68,7 @@ export interface TimelineEventPayload { subjectSourceKey?: string | null; eventAt: number; text?: string | null; - metadata?: Record; + systemKind?: string | null; } export const CALL_PROVIDER_VALUES = [ @@ -124,20 +104,13 @@ export interface CallPayload { sourceCallKey: string; sourceConversationKey: string; provider: CallProvider; - providerCallType?: string | null; direction: CallDirection; medium: CallMedium; status: CallStatus; startedAt: number; - answeredAt?: number | null; - endedAt?: number | null; durationSeconds?: number | null; initiatorSourceKey?: string | null; primaryRemoteSourceKey?: string | null; - remoteAddress?: string | null; - remoteDisplayName?: string | null; - disconnectedCause?: string | null; - metadata?: Record; } export type RawEventPayload = @@ -190,35 +163,22 @@ export type SyncProofKind = StandardSyncProofKind | (string & {}); export interface SyncScopeInput { kind: SyncScopeKind; key: string; - parent?: { - kind: SyncScopeKind; - key: string; - } | null; - displayName?: string | null; - metadata?: Record | null; } export interface SyncProofInput { scope: SyncScopeInput; proofKind: SyncProofKind; status: SyncProofStatus; - syncMode?: SyncMode | null; observedAt: number; - runStartedAt?: number | null; - completedAt?: number | null; - freshUntil?: number | null; resumeCursor?: unknown; coverage?: Record | null; stats?: Record | null; - error?: unknown; } export const RAW_EVENT_ACQUISITION_MODE_VALUES = ["sync", "realtime"] as const; export type RawEventAcquisitionMode = (typeof RAW_EVENT_ACQUISITION_MODE_VALUES)[number]; export interface RawEventProvenance { - providerApiVersion?: string | null; - adapterVersion?: string | null; acquisitionMode?: RawEventAcquisitionMode | null; } @@ -228,17 +188,14 @@ export interface ProviderRawEventInput { accountKey: string; entityKind: RawEventEntityKind; eventKind: string; - externalEventId?: string | null; externalEntityId?: string | null; conversationExternalId?: string | null; occurredAt?: number | null; observedAt: number; - cursor?: unknown; dedupeKey: string; payload: TPayload; normalizedSchema?: string | null; provenance?: RawEventProvenance | null; - sourceVersion?: string | null; } const contactFieldNameSet = new Set(CONTACT_FIELD_NAME_VALUES); @@ -251,20 +208,6 @@ export function buildNormalizedRawEventSchema( return `${entityKind}.${eventKind}@${version}`; } -export function normalizeRawEventProvenance( - input: Partial | null | undefined, -): RawEventProvenance | null { - const normalized: RawEventProvenance = { - providerApiVersion: input?.providerApiVersion ?? null, - adapterVersion: input?.adapterVersion ?? null, - acquisitionMode: input?.acquisitionMode ?? null, - }; - - return normalized.providerApiVersion || normalized.adapterVersion || normalized.acquisitionMode - ? normalized - : null; -} - export function resolveRawEventNormalizedSchema( input: Pick, ): string { diff --git a/src/db/database.test.ts b/src/db/database.test.ts index 1d19a1aa..7da54859 100644 --- a/src/db/database.test.ts +++ b/src/db/database.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -28,42 +28,44 @@ describe("CuedDatabase", () => { it("counts observed phone calls with a targeted raw event query", () => { const db = createDb(); - db.insertRawEvent({ - id: randomUUID(), - platform: "imessage", - accountKey: "local", - entityKind: "call", - eventKind: "observed", - observedAt: 1, - dedupeKey: "imessage:call:1", - payload: { - sourceCallKey: "call-1", - sourceConversationKey: "conversation-1", - provider: "facetime", - direction: "incoming", - medium: "audio", - status: "missed", - startedAt: 1, + db.insertRawEvents([ + { + id: randomUUID(), + platform: "imessage", + accountKey: "local", + entityKind: "call", + eventKind: "observed", + observedAt: 1, + dedupeKey: "imessage:call:1", + payload: { + sourceCallKey: "call-1", + sourceConversationKey: "conversation-1", + provider: "facetime", + direction: "incoming", + medium: "audio", + status: "missed", + startedAt: 1, + }, }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: randomUUID(), - platform: "slack", - accountKey: "workspace", - entityKind: "message", - eventKind: "created", - observedAt: 2, - dedupeKey: "slack:message:1", - payload: { - sourceMessageKey: "message-1", - sourceConversationKey: "channel-1", - sender: { sourceContactKey: "user-1", displayName: "Ava" }, - sentAt: 2, - text: "Not a call", + ]); + db.insertRawEvents([ + { + id: randomUUID(), + platform: "slack", + accountKey: "workspace", + entityKind: "message", + eventKind: "created", + observedAt: 2, + dedupeKey: "slack:message:1", + payload: { + sourceMessageKey: "message-1", + sourceConversationKey: "channel-1", + sender: { sourceContactKey: "user-1", displayName: "Ava" }, + sentAt: 2, + text: "Not a call", + }, }, - sourceVersion: "slack-v1", - }); + ]); expect(db.countObservedPhoneCalls()).toBe(1); @@ -98,51 +100,6 @@ describe("CuedDatabase", () => { .run(input.id, input.name, timestamp, timestamp); } - function insertHandle( - db: CuedDatabase, - input: { - id: string; - contactId: string; - type: string; - value: string; - normalizedValue: string; - platform: string; - accountKey: string; - isDeterministic?: number; - }, - ): void { - const sqlite = ( - db as unknown as { - sqlite: { - prepare: (sql: string) => { - run: (...params: unknown[]) => void; - }; - }; - } - ).sqlite; - const timestamp = Date.now(); - sqlite - .prepare( - ` - INSERT INTO contact_handles ( - id, contact_id, type, value, normalized_value, platform, account_key, is_deterministic, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `, - ) - .run( - input.id, - input.contactId, - input.type, - input.value, - input.normalizedValue, - input.platform, - input.accountKey, - input.isDeterministic ?? 1, - timestamp, - timestamp, - ); - } - function sqlite(db: CuedDatabase) { return ( db as unknown as { @@ -173,111 +130,26 @@ describe("CuedDatabase", () => { db.close(); }); - it("claims jobs by priority and reclaims expired leases", () => { + it("queues message FTS indexing work independently", () => { const db = createDb(); - const lowPriorityJobId = db.queueJob({ - kind: "ingest", - platform: "slack", - accountKey: "T1", - priority: 30, - trigger: "test_low", - checkpoint: { cursor: "a" }, - }); - const highPriorityJobId = db.queueJob({ - kind: "auth", - platform: "slack", - accountKey: "T1", - priority: 0, - trigger: "test_high", - }); - - const claimedHigh = db.claimNextJob({ - ownerId: "worker-a", - leaseMs: 10_000, - }); - expect(claimedHigh).toMatchObject({ - id: highPriorityJobId, - kind: "auth", - status: "running", - owner_id: "worker-a", - attempt: 1, - }); - - db.updateJobProgress(highPriorityJobId, { - checkpoint: { openedBrowser: true }, - progress: { state: "waiting_for_callback" }, - leaseMs: 20_000, - }); - db.failJob(highPriorityJobId, { - error: new Error("network offline"), - retryAt: Date.now() + 60_000, - }); - - const claimedLow = db.claimNextJob({ - ownerId: "worker-b", - leaseMs: 10_000, - }); - expect(claimedLow).toMatchObject({ - id: lowPriorityJobId, - kind: "ingest", - status: "running", - owner_id: "worker-b", - attempt: 1, - }); - - sqlite(db).prepare("UPDATE jobs SET lease_expires_at = 1 WHERE id = ?").run(lowPriorityJobId); - const reclaimedLow = db.claimNextJob({ - ownerId: "worker-c", - leaseMs: 10_000, - }); - expect(reclaimedLow).toMatchObject({ - id: lowPriorityJobId, - owner_id: "worker-c", - attempt: 2, - }); - - db.completeJob(lowPriorityJobId, { done: true }); - expect( - sqlite(db) - .prepare("SELECT status, owner_id, lease_expires_at FROM jobs WHERE id = ?") - .get(lowPriorityJobId), - ).toEqual({ status: "completed", owner_id: null, lease_expires_at: null }); - - db.close(); - }); - - it("queues and claims message FTS indexing work independently", () => { - const db = createDb(); + expect(db.enqueueMessageFtsIndex(["message-a", "message-b", "message-a"])).toBe(2); + expect(db.enqueueMessageFtsIndex(["message-a"])).toBe(1); - expect(db.enqueueMessageFtsIndex(["message-a", "message-b", "message-a"], "projection")).toBe( - 2, - ); - expect(db.enqueueMessageFtsIndex(["message-a"], "conversation_renamed")).toBe(1); - - const claimed = db.claimMessageFtsIndexBatch(10); - expect(claimed.map((row) => row.message_id).sort()).toEqual(["message-a", "message-b"]); - expect(claimed.every((row) => row.status === "indexing" && row.attempt === 1)).toBe(true); expect(db.getMessageFtsIndexBacklog()).toEqual({ - queued: 0, - indexing: 2, + queued: 2, + indexing: 0, failed: 0, pending: 2, }); - - expect(db.completeMessageFtsIndex(["message-a"])).toBe(1); - expect(db.failMessageFtsIndex(["message-b"], new Error("fts busy"))).toBe(1); - expect(db.getMessageFtsIndexBacklog()).toEqual({ - queued: 0, - indexing: 0, - failed: 1, - pending: 0, - }); expect( sqlite(db) - .prepare("SELECT message_id, status, last_error FROM message_fts_index_queue") + .prepare("SELECT message_id, status FROM message_fts_index_queue ORDER BY message_id") .all(), - ).toEqual([{ message_id: "message-b", status: "failed", last_error: "fts busy" }]); + ).toEqual([ + { message_id: "message-a", status: "queued" }, + { message_id: "message-b", status: "queued" }, + ]); db.close(); }); @@ -289,13 +161,11 @@ describe("CuedDatabase", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, - is_active, removal_reason, service, name, topic, participant_names, last_message_id, - last_message_at, last_message_preview, unread_count, created_at, updated_at + id, platform, account_key, source_conversation_key, type, + is_active, name, participant_names, created_at, updated_at ) VALUES ( - 'conversation-a', 'slack', 'team-a', 'channel-a', NULL, 'group', - 1, NULL, NULL, 'Launch', NULL, 'Theo | Soham', NULL, - NULL, NULL, 0, ?, ? + 'conversation-a', 'slack', 'team-a', 'channel-a', 'group', + 1, 'Launch', 'Theo | Soham', ?, ? ) `, ) @@ -306,20 +176,19 @@ describe("CuedDatabase", () => { INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, sender_source_key, sender_name, conversation_name, sent_at, - service, status, is_from_me, content, delivered_at, read_at, edited_at, - deleted_at, reply_to_message_id, is_deleted, is_edited, attachment_count, - reaction_count, created_at, updated_at + status, is_from_me, content, delivered_at, read_at, + is_deleted, created_at, updated_at ) VALUES ( 'message-a', 'slack', 'team-a', 'ts-a', 'conversation-a', - NULL, NULL, 'Soham', 'Launch', ?, NULL, NULL, 0, - 'queued index content', NULL, NULL, NULL, NULL, NULL, 0, 0, 0, 0, ?, ? + NULL, NULL, 'Soham', 'Launch', ?, NULL, 0, + 'queued index content', NULL, NULL, 0, ?, ? ) `, ) .run(timestamp, timestamp, timestamp); sqlite(db).prepare("DELETE FROM messages_fts").run(); - expect(db.enqueueMessageFtsIndex(["message-a"], "projection")).toBe(1); + expect(db.enqueueMessageFtsIndex(["message-a"])).toBe(1); expect(db.drainMessageFtsIndexQueue(1)).toEqual({ claimed: 1, indexed: 1, failed: 0 }); expect( sqlite(db) @@ -337,11 +206,13 @@ describe("CuedDatabase", () => { const db = createDb(); const timestamp = Date.now(); - expect(db.enqueueMessageFtsIndex(["message-a", "message-b"], "projection")).toBe(2); - expect(db.claimMessageFtsIndexBatch(2)).toHaveLength(2); + expect(db.enqueueMessageFtsIndex(["message-a", "message-b"])).toBe(2); sqlite(db) - .prepare("UPDATE message_fts_index_queue SET updated_at = ? WHERE message_id = 'message-a'") + .prepare("UPDATE message_fts_index_queue SET status = 'indexing', updated_at = ?") .run(timestamp - 10_000); + sqlite(db) + .prepare("UPDATE message_fts_index_queue SET updated_at = ? WHERE message_id = 'message-b'") + .run(timestamp); expect(db.requeueStaleMessageFtsIndexing(timestamp - 5_000)).toBe(1); expect( @@ -427,7 +298,11 @@ describe("CuedDatabase", () => { }); expect(replacement.supersedes_memory_id).toBe(original.id); - expect(db.getContactMemory(original.id)?.stale_at).toEqual(expect.any(Number)); + expect( + db + .listContactMemories({ contactId: "contact-1", includeStale: true }) + .find((row) => row.id === original.id)?.stale_at, + ).toEqual(expect.any(Number)); expect(db.listContactMemories({ contactId: "contact-1" }).map((row) => row.id)).toEqual([ replacement.id, ]); @@ -470,15 +345,17 @@ describe("CuedDatabase", () => { createdBy: "test", }); - db.clearProjectedState(); + sqlite(db).prepare("DELETE FROM contacts").run(); insertContact(db, { id: "contact-1", name: "Ava Chen", updatedAt: 2 }); - expect(db.getContactMemory(memory.id)).toMatchObject({ - id: memory.id, - contact_id: "contact-1", - contact_name: "Ava Chen", - body: "Works on applied AI and prefers concise follow-ups.", - }); + expect(db.listContactMemories({ contactId: "contact-1", includeStale: true })[0]).toMatchObject( + { + id: memory.id, + contact_id: "contact-1", + contact_name: "Ava Chen", + body: "Works on applied AI and prefers concise follow-ups.", + }, + ); db.close(); }); @@ -493,15 +370,17 @@ describe("CuedDatabase", () => { createdBy: "test", }); - db.clearProjectedState(); + sqlite(db).prepare("DELETE FROM contacts").run(); - expect(db.getContactMemory(memory.id)).toMatchObject({ - id: memory.id, - contact_id: "contact-1", - contact_name: null, - body: "Works on applied AI and prefers concise follow-ups.", - stale_at: null, - }); + expect(db.listContactMemories({ contactId: "contact-1", includeStale: true })[0]).toMatchObject( + { + id: memory.id, + contact_id: "contact-1", + contact_name: null, + body: "Works on applied AI and prefers concise follow-ups.", + stale_at: null, + }, + ); expect(db.listContactMemories({ contactId: "contact-1" }).map((row) => row.id)).toEqual([ memory.id, ]); @@ -524,15 +403,9 @@ describe("CuedDatabase", () => { scope: { kind: "conversation", key: "C123", - displayName: "eng", - metadata: { - teamId: "T123", - conversationFamily: "channels", - }, }, proofKind: "messages", status: "running", - syncMode: "full", observedAt: 200, resumeCursor: { historyCursor: "history-2", @@ -551,11 +424,9 @@ describe("CuedDatabase", () => { scope: { kind: "conversation", key: "C123", - displayName: "eng", }, proofKind: "messages", status: "complete", - syncMode: "full", observedAt: 300, coverage: { oldestMessageTs: "1709999999.000000", @@ -567,32 +438,12 @@ describe("CuedDatabase", () => { }, }); - expect(db.listSyncScopes("slack", "workspace-a")).toEqual([ - expect.objectContaining({ - platform: "slack", - account_key: "workspace-a", - scope_kind: "conversation", - scope_key: "C123", - display_name: "eng", - metadata_json: JSON.stringify({ - teamId: "T123", - conversationFamily: "channels", - }), - first_discovered_at: 200, - last_observed_at: 300, - }), - ]); - expect(db.listSyncProofs("slack", "workspace-a")).toEqual([ expect.objectContaining({ - platform: "slack", - account_key: "workspace-a", scope_kind: "conversation", scope_key: "C123", proof_kind: "messages", status: "complete", - sync_mode: "full", - completed_at: 300, resume_cursor_json: null, coverage_json: JSON.stringify({ oldestMessageTs: "1709999999.000000", @@ -637,23 +488,8 @@ describe("CuedDatabase", () => { }, }); - expect(db.listSyncScopes("slack", "a:b")).toEqual([ - expect.objectContaining({ - account_key: "a:b", - scope_kind: "conversation", - scope_key: "d", - }), - ]); - expect(db.listSyncScopes("slack", "a")).toEqual([ - expect.objectContaining({ - account_key: "a", - scope_kind: "conversation", - scope_key: "c:d", - }), - ]); expect(db.listSyncProofs("slack", "a:b")).toEqual([ expect.objectContaining({ - account_key: "a:b", scope_kind: "conversation", scope_key: "d", status: "running", @@ -661,7 +497,6 @@ describe("CuedDatabase", () => { ]); expect(db.listSyncProofs("slack", "a")).toEqual([ expect.objectContaining({ - account_key: "a", scope_kind: "conversation", scope_key: "c:d", status: "complete", @@ -693,7 +528,7 @@ describe("CuedDatabase", () => { db.close(); }); - it("clears completed_at when a generic sync proof returns to running", () => { + it("updates a generic sync proof when it returns to running", () => { const db = createDb(); db.upsertSyncProof({ @@ -726,7 +561,7 @@ describe("CuedDatabase", () => { expect(db.listSyncProofs("slack", "workspace-a")).toEqual([ expect.objectContaining({ status: "running", - completed_at: null, + last_observed_at: 200, }), ]); @@ -736,13 +571,20 @@ describe("CuedDatabase", () => { it("persists app settings and install metadata", () => { const db = createDb(); - db.recordAppMetadata({ - version: "0.1.0-internal.1", - releaseChannel: "internal", - cliSymlinkInstalled: true, - }); + db.setAppSetting("installed_app_version", "0.1.0-internal.1"); + db.setAppSetting("release_channel", "internal"); + db.setAppSetting("cli_symlink_installed", "1"); db.markOnboardingCompleted("0.1.0-internal.1"); - db.markReleaseCheck(123456789); + db.setUpdateReleaseState({ + checkedAt: 123456789, + channel: "internal", + currentVersion: "0.1.0-internal.1", + latestVersion: null, + availableVersion: null, + releaseUrl: null, + tarballUrl: null, + etag: null, + }); expect(db.getAppMetadata()).toEqual({ onboardingCompletedVersion: "0.1.0-internal.1", @@ -750,7 +592,16 @@ describe("CuedDatabase", () => { installedAppVersion: "0.1.0-internal.1", lastReleaseCheckAt: 123456789, cliSymlinkInstalled: true, - updateReleaseState: null, + updateReleaseState: { + checkedAt: 123456789, + channel: "internal", + currentVersion: "0.1.0-internal.1", + latestVersion: null, + availableVersion: null, + releaseUrl: null, + tarballUrl: null, + etag: null, + }, updatePendingRollback: null, updateLastError: null, }); @@ -762,10 +613,8 @@ describe("CuedDatabase", () => { it("opens existing databases without startup metadata writes", () => { const db = createDb(); const dbPath = db.dbPath; - db.recordAppMetadata({ - version: "0.1.0", - releaseChannel: "stable", - }); + db.setAppSetting("installed_app_version", "0.1.0"); + db.setAppSetting("release_channel", "stable"); db.close(); const existing = openExistingCuedDatabase(dbPath); @@ -814,7 +663,7 @@ describe("CuedDatabase", () => { metadata: { source: "test" }, }); - expect(db.listEnabledSyncPlatforms()).toEqual(["linkedin"]); + expect(db.listEnabledSyncTargets()).toEqual([{ platform: "linkedin", account_key: "default" }]); expect(db.getIntegrationState("linkedin", "default")).toEqual( expect.objectContaining({ platform: "linkedin", @@ -950,261 +799,6 @@ describe("CuedDatabase", () => { db.close(); }); - it("queues and retries outbound messages", () => { - const db = createDb(); - - const messageId = db.queueOutboundMessage({ - platform: "signal", - accountKey: "default", - target: "+14155550123", - text: "Hello", - }); - - const claimed = db.claimNextOutboundMessage("signal"); - expect(claimed).toEqual( - expect.objectContaining({ - id: messageId, - platform: "signal", - account_key: "default", - status: "sending", - attempt_count: 1, - }), - ); - - db.failOutboundMessage({ - id: messageId, - retryable: true, - error: "network timeout", - retryDelayMs: 0, - }); - expect(db.hasQueuedOutboundMessages("signal")).toBe(true); - - const retried = db.claimNextOutboundMessage("signal"); - expect(retried?.attempt_count).toBe(2); - db.completeOutboundMessage(messageId); - expect(db.hasQueuedOutboundMessages("signal")).toBe(false); - db.close(); - }); - - it("resolves Signal targets by preferring signal_id over phone without merging contacts", () => { - const db = createDb(); - - insertContact(db, { id: "contact-phone", name: "Soham Bafana", updatedAt: 10 }); - insertContact(db, { id: "contact-signal", name: "Soham Bafana", updatedAt: 20 }); - - insertHandle(db, { - id: "handle-phone", - contactId: "contact-phone", - type: "phone", - value: "+12016824050", - normalizedValue: "2016824050", - platform: "contacts", - accountKey: "local", - }); - insertHandle(db, { - id: "handle-signal", - contactId: "contact-signal", - type: "signal_id", - value: "d6ed1597-758c-4022-96aa-253b334f1f5d", - normalizedValue: "d6ed1597-758c-4022-96aa-253b334f1f5d", - platform: "signal", - accountKey: "default", - }); - - expect(db.resolveSignalSendTarget("Soham Bafana")).toEqual({ - target: "d6ed1597-758c-4022-96aa-253b334f1f5d", - threadId: "dm:d6ed1597-758c-4022-96aa-253b334f1f5d", - resolution: "signal_id", - matchedContactIds: ["contact-signal", "contact-phone"], - matchedName: "Soham Bafana", - }); - - expect(db.resolveSignalSendTarget("+12016824050")).toEqual({ - target: "d6ed1597-758c-4022-96aa-253b334f1f5d", - threadId: "dm:d6ed1597-758c-4022-96aa-253b334f1f5d", - resolution: "signal_id", - matchedContactIds: ["contact-phone", "contact-signal"], - matchedName: "Soham Bafana", - }); - - db.close(); - }); - - it("keeps direct Signal phone sends as passthrough when there is no better contact match", () => { - const db = createDb(); - - expect(db.resolveSignalSendTarget("+14155550123")).toEqual({ - target: "+14155550123", - threadId: "dm:+14155550123", - resolution: "passthrough", - matchedContactIds: [], - matchedName: null, - }); - - db.close(); - }); - - it("formats local Signal phone sends as E.164 passthrough", () => { - const db = createDb(); - - expect(db.resolveSignalSendTarget("4155550123")).toEqual({ - target: "+14155550123", - threadId: "dm:+14155550123", - resolution: "passthrough", - matchedContactIds: [], - matchedName: null, - }); - - db.close(); - }); - - it("resolves WhatsApp targets by preferring whatsapp_jid over phone without merging contacts", () => { - const db = createDb(); - - insertContact(db, { id: "contact-phone", name: "Soham Bafana", updatedAt: 10 }); - insertContact(db, { id: "contact-whatsapp", name: "Soham Bafana", updatedAt: 20 }); - - insertHandle(db, { - id: "wa-handle-phone", - contactId: "contact-phone", - type: "phone", - value: "+12016824050", - normalizedValue: "2016824050", - platform: "contacts", - accountKey: "local", - }); - insertHandle(db, { - id: "wa-handle-jid", - contactId: "contact-whatsapp", - type: "whatsapp_jid", - value: "12016824050@s.whatsapp.net", - normalizedValue: "12016824050@s.whatsapp.net", - platform: "whatsapp", - accountKey: "default", - }); - - expect(db.resolveWhatsAppSendTarget("Soham Bafana")).toEqual({ - target: "12016824050@s.whatsapp.net", - threadId: "dm:12016824050@s.whatsapp.net", - resolution: "whatsapp_jid", - matchedContactIds: ["contact-whatsapp", "contact-phone"], - matchedName: "Soham Bafana", - }); - - expect(db.resolveWhatsAppSendTarget("+12016824050")).toEqual({ - target: "12016824050@s.whatsapp.net", - threadId: "dm:12016824050@s.whatsapp.net", - resolution: "whatsapp_jid", - matchedContactIds: ["contact-phone", "contact-whatsapp"], - matchedName: "Soham Bafana", - }); - - db.close(); - }); - - it("formats local WhatsApp phone sends as JIDs", () => { - const db = createDb(); - - expect(db.resolveWhatsAppSendTarget("4155550123")).toEqual({ - target: "14155550123@s.whatsapp.net", - threadId: "dm:14155550123@s.whatsapp.net", - resolution: "passthrough", - matchedContactIds: [], - matchedName: null, - }); - - db.close(); - }); - - it("resolves Discord send targets only for DMs and group DMs", () => { - const db = createDb(); - const sql = sqlite(db); - const timestamp = Date.now(); - - sql - .prepare( - ` - INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - removal_reason, name, topic, participant_names, last_message_id, last_message_at, - last_message_preview, unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, ?, 1, NULL, ?, NULL, '[]', NULL, NULL, NULL, 0, ?, ?) - `, - ) - .run("discord-dm", "discord:channel:dm-1", "dm-1", "dm", "Jarvis", timestamp, timestamp); - sql - .prepare( - ` - INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - removal_reason, name, topic, participant_names, last_message_id, last_message_at, - last_message_preview, unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, ?, 1, NULL, ?, NULL, '[]', NULL, NULL, NULL, 0, ?, ?) - `, - ) - .run( - "discord-group-dm", - "discord:channel:group-dm-1", - "group-dm-1", - "group", - "Jarvis, Ava", - timestamp, - timestamp, - ); - sql - .prepare( - ` - INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - removal_reason, name, topic, participant_names, last_message_id, last_message_at, - last_message_preview, unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, ?, 1, NULL, ?, NULL, '[]', NULL, NULL, NULL, 0, ?, ?) - `, - ) - .run( - "discord-channel", - "discord:channel:guild-1", - "guild-1", - "channel", - "general", - timestamp, - timestamp, - ); - - expect(db.resolveDiscordSendTarget("Jarvis")).toEqual({ - target: "dm-1", - threadId: "discord:channel:dm-1", - resolution: "conversation_name", - matchedConversationId: "discord-dm", - matchedName: "Jarvis", - }); - expect(db.resolveDiscordSendTarget("Jarvis, Ava")).toEqual({ - target: "group-dm-1", - threadId: "discord:channel:group-dm-1", - resolution: "conversation_name", - matchedConversationId: "discord-group-dm", - matchedName: "Jarvis, Ava", - }); - expect(db.resolveDiscordSendTarget("dm-1")).toEqual({ - target: "dm-1", - threadId: "discord:channel:dm-1", - resolution: "channel_id", - matchedConversationId: "discord-dm", - matchedName: "Jarvis", - }); - expect(db.resolveDiscordSendTarget("discord:channel:dm-1")).toEqual({ - target: "dm-1", - threadId: "discord:channel:dm-1", - resolution: "source_conversation_key", - matchedConversationId: "discord-dm", - matchedName: "Jarvis", - }); - expect(db.resolveDiscordSendTarget("general")).toBeNull(); - expect(db.resolveDiscordSendTarget("guild-1")).toBeNull(); - - db.close(); - }); - it("records manual merge decisions and resolves canonical contact chains", () => { const db = createDb(); @@ -1238,10 +832,15 @@ describe("CuedDatabase", () => { canonicalContactId: "contact-c", }); - expect(db.resolveCanonicalContactId("contact-b")).toBe("contact-c"); - expect(db.resolveCanonicalContactId("contact-a")).toBe("contact-c"); - expect(db.resolveCanonicalContactId("contact-c")).toBe("contact-c"); - expect(db.listContactMergeDecisions()).toHaveLength(2); + expect(() => + db.recordContactMergeDecision({ + primaryContactId: "contact-c", + secondaryContactId: "contact-b", + }), + ).toThrow("Contacts already resolve to the same canonical contact: contact-c"); + expect( + sqlite(db).prepare("SELECT COUNT(*) AS count FROM contact_merge_decisions").get(), + ).toEqual({ count: 2 }); db.close(); }); @@ -1336,22 +935,17 @@ describe("CuedDatabase", () => { true, ); expect(db.getLatestSyncRunError("linkedin", "default")).toEqual({ - sync_run_id: failedId, error_message: "boom", created_at: expect.any(Number), - details_json: JSON.stringify({ code: "x" }), }); expect( sqlite(db) - .prepare( - "SELECT status, owner_id, lease_expires_at, error_code FROM sync_runs WHERE id = ?", - ) + .prepare("SELECT status, owner_id, lease_expires_at FROM sync_runs WHERE id = ?") .get(failedId), ).toEqual({ status: "failed", owner_id: null, lease_expires_at: null, - error_code: null, }); db.close(); @@ -1426,11 +1020,11 @@ describe("CuedDatabase", () => { .run(timestamp + 30_000, runningRunId); expect(db.getNextClaimableRunAt(["sync"], "ingesting")).toBe(timestamp + 30_000); - db.failRun(runningRunId, "forced", undefined, "forced"); + db.failRun(runningRunId, "forced"); expect(db.getNextClaimableRunAt(["sync"], "ingesting")).toBeGreaterThanOrEqual(timestamp); expect(db.getNextClaimableRunAt(["sync"], "ingesting")).toBeLessThanOrEqual(timestamp + 60_000); - db.failRun(queuedRunId, "forced", undefined, "forced"); + db.failRun(queuedRunId, "forced"); expect(db.getNextClaimableRunAt(["sync"], "ingesting")).toBeNull(); db.close(); @@ -1454,7 +1048,7 @@ describe("CuedDatabase", () => { sqlite(db) .prepare( ` - SELECT status, owner_id, lease_expires_at, error_code + SELECT status, owner_id, lease_expires_at FROM sync_runs WHERE id = ? `, @@ -1464,7 +1058,6 @@ describe("CuedDatabase", () => { status: "queued", owner_id: null, lease_expires_at: null, - error_code: "daemon_recovered", }); expect(db.claimNextQueuedRun(["sync"], "ingesting")).toEqual( expect.objectContaining({ id: runId, attempt: 2 }), @@ -1519,11 +1112,13 @@ describe("CuedDatabase", () => { it("stores checkpoints and raw events", () => { const db = createDb(); - db.upsertSourceAccount({ - platform: "contacts", - accountKey: "local", - displayName: "macOS Contacts", - }); + db.upsertSourceAccounts([ + { + platform: "contacts", + accountKey: "local", + displayName: "macOS Contacts", + }, + ]); db.upsertCheckpoint({ platform: "contacts", accountKey: "local", @@ -1543,21 +1138,22 @@ describe("CuedDatabase", () => { }); const rawEventId = randomUUID(); - db.insertRawEvent({ - id: rawEventId, - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1_700_000_000_000, - dedupeKey: "contacts:1", - payload: { - sourceEntityKey: "contacts:1", - fields: { display_name: "Ava" }, - handles: [], + db.insertRawEvents([ + { + id: rawEventId, + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1_700_000_000_000, + dedupeKey: "contacts:1", + payload: { + sourceEntityKey: "contacts:1", + fields: { display_name: "Ava" }, + handles: [], + }, }, - sourceVersion: "contacts-v1", - }); + ]); expect(db.getCheckpoint("contacts", "local")).toEqual({ source_cursor_json: JSON.stringify({ snapshotAt: 456 }), @@ -1584,8 +1180,18 @@ describe("CuedDatabase", () => { last_success_at: 1_700_000_000_500, last_error_summary: "sync failed", }); - expect(db.listCheckpointPlatforms()).toEqual(["contacts"]); - expect(db.listRawEvents()).toEqual([ + expect(db.listCheckpointTargets()).toEqual([{ platform: "contacts", account_key: "local" }]); + expect( + sqlite(db) + .prepare( + ` + SELECT id, platform, account_key, entity_kind, event_kind, normalized_schema, observed_at, payload_json + FROM raw_events + ORDER BY observed_at, id + `, + ) + .all(), + ).toEqual([ { id: rawEventId, platform: "contacts", @@ -1593,7 +1199,6 @@ describe("CuedDatabase", () => { entity_kind: "contact", event_kind: "observed", normalized_schema: "contact.observed@1", - provenance_json: null, observed_at: 1_700_000_000_000, payload_json: JSON.stringify({ sourceEntityKey: "contacts:1", @@ -1606,8 +1211,6 @@ describe("CuedDatabase", () => { expect(db.getProjectionState()).toEqual({ singleton_key: "global", projection_watermark: 0, - last_projected_at: null, - last_rebuild_at: null, updated_at: expect.any(Number), }); expect(db.getProjectionBacklog()).toEqual({ @@ -1624,57 +1227,6 @@ describe("CuedDatabase", () => { db.close(); }); - it("stores source version separately from provenance metadata", () => { - const db = createDb(); - - db.insertRawEvent({ - id: "raw-event-source-version", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 1_710_000_000_000, - dedupeKey: "linkedin:source-version", - payload: { - sourceMessageKey: "msg-source-version", - sourceConversationKey: "thread-source-version", - senderSourceKey: "linkedin:member:ava", - sentAt: 1_710_000_000_000, - content: "hello", - }, - sourceVersion: "linkedin-v7", - provenance: { - acquisitionMode: "realtime", - providerApiVersion: "2026-03", - adapterVersion: "linkedin-adapter@7", - }, - }); - - const row = sqlite(db) - .prepare( - ` - SELECT source_version, provenance_json - FROM raw_events - WHERE id = ? - `, - ) - .get("raw-event-source-version") as { - source_version: string | null; - provenance_json: string | null; - }; - - expect(row).toEqual({ - source_version: "linkedin-v7", - provenance_json: JSON.stringify({ - providerApiVersion: "2026-03", - adapterVersion: "linkedin-adapter@7", - acquisitionMode: "realtime", - }), - }); - - db.close(); - }); - it("upgrades existing databases with refactor columns and lifecycle state", () => { const dir = mkdtempSync(join(tmpdir(), "cued-v2-upgrade-db-")); tempDirs.push(dir); @@ -1751,6 +1303,66 @@ describe("CuedDatabase", () => { UNIQUE(platform, account_key, source_event_key) ) `); + sql.exec(` + CREATE TABLE messages ( + id TEXT PRIMARY KEY, + platform TEXT NOT NULL, + account_key TEXT NOT NULL, + platform_message_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + sender_contact_id TEXT, + sender_source_key TEXT, + sender_name TEXT, + conversation_name TEXT, + sent_at INTEGER NOT NULL, + status TEXT, + is_from_me INTEGER NOT NULL DEFAULT 0, + content TEXT, + delivered_at INTEGER, + read_at INTEGER, + is_deleted INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(platform, account_key, platform_message_id) + ) + `); + sql.exec(` + CREATE TABLE message_attachments ( + id TEXT PRIMARY KEY, + message_id TEXT NOT NULL, + platform TEXT NOT NULL, + account_key TEXT NOT NULL, + source_attachment_key TEXT NOT NULL, + kind TEXT, + mime_type TEXT, + filename TEXT, + title TEXT, + local_path TEXT, + remote_url TEXT, + size_bytes INTEGER, + text_content TEXT, + access_kind TEXT, + access_ref_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(platform, account_key, source_attachment_key) + ) + `); + sql.exec(` + CREATE TABLE message_reactions ( + id TEXT PRIMARY KEY, + message_id TEXT NOT NULL, + platform TEXT NOT NULL, + account_key TEXT NOT NULL, + source_reaction_key TEXT NOT NULL, + reactor_source_key TEXT, + emoji TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(platform, account_key, source_reaction_key) + ) + `); sql .prepare(` INSERT INTO conversations ( @@ -1772,32 +1384,49 @@ describe("CuedDatabase", () => { const timelineColumns = ( sql.prepare("PRAGMA table_info(timeline_events)").all() as Array<{ name: string }> ).map((column) => column.name); + const messageReactionColumns = ( + sql.prepare("PRAGMA table_info(message_reactions)").all() as Array<{ name: string }> + ).map((column) => column.name); + const messageReactionIndex = sql + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?") + .get("idx_message_reactions_message"); - expect(rawEventColumns).toEqual( - expect.arrayContaining(["normalized_schema", "provenance_json"]), + expect(rawEventColumns).toEqual(expect.arrayContaining(["normalized_schema"])); + expect(rawEventColumns).not.toEqual( + expect.arrayContaining([ + "external_event_id", + "cursor_json", + "source_version", + "provenance_json", + ]), ); expect(conversationColumns).toContain("is_active"); - expect(conversationColumns).toContain("removal_reason"); + expect(conversationColumns).not.toEqual( + expect.arrayContaining([ + "native_conversation_key", + "topic", + "removal_reason", + "service", + "last_message_id", + "last_message_at", + "last_message_preview", + "unread_count", + ]), + ); expect(timelineColumns).toContain("subject_source_key"); expect(timelineColumns).toContain("system_kind"); expect(timelineColumns).toContain("call_provider"); expect(timelineColumns).toContain("call_direction"); - expect(timelineColumns).toContain("call_status"); - expect(timelineColumns).toContain("call_medium"); - expect(timelineColumns).toContain("call_started_at"); - expect(timelineColumns).toContain("call_duration_seconds"); - expect(timelineColumns).toContain("call_ended_at"); - expect(timelineColumns).toContain("call_disconnected_cause"); + expect(messageReactionColumns).not.toContain("source_reaction_key"); + expect(messageReactionIndex).toEqual({ name: "idx_message_reactions_message" }); expect( - sql - .prepare("SELECT is_active, removal_reason FROM conversations WHERE id = ?") - .get("legacy-conversation"), - ).toEqual({ is_active: 0, removal_reason: "deleted" }); + sql.prepare("SELECT is_active FROM conversations WHERE id = ?").get("legacy-conversation"), + ).toEqual({ is_active: 0 }); db.close(); }); - it("repairs removal_reason for databases that already marked 0002 applied", () => { + it("records the legacy removal_reason repair without keeping the removed column", () => { const dir = mkdtempSync(join(tmpdir(), "cued-v2-removal-reason-repair-")); tempDirs.push(dir); const db = new CuedDatabase(join(dir, "local.db")); @@ -1842,7 +1471,7 @@ describe("CuedDatabase", () => { sql.prepare("PRAGMA table_info(conversations)").all() as Array<{ name: string }> ).map((column) => column.name); - expect(conversationColumns).toContain("removal_reason"); + expect(conversationColumns).not.toEqual(expect.arrayContaining(["removal_reason", "service"])); expect( sql .prepare("SELECT 1 FROM schema_migrations WHERE id = ?") @@ -2017,7 +1646,6 @@ describe("CuedDatabase", () => { "idx_conversation_participants_source_key_lower", "idx_timeline_events_actor_contact", "idx_timeline_events_conversation_latest_system", - "idx_message_reactions_reactor_contact", "idx_messages_conversation_latest", "idx_messages_conversation_unread", "idx_messages_sender_source_key_lower", @@ -2030,29 +1658,15 @@ describe("CuedDatabase", () => { .get("legacy-run"), ).toEqual({ queued_at: 123, scheduled_at: 123, started_at: null }); expect(syncRunColumns).toEqual( - expect.arrayContaining([ - "attempt", - "owner_id", - "lease_expires_at", - "last_progress_at", - "error_code", - ]), + expect.arrayContaining(["attempt", "owner_id", "lease_expires_at"]), ); + expect(syncRunColumns).not.toEqual(expect.arrayContaining(["last_progress_at", "error_code"])); expect(messageAttachmentColumns).toEqual( - expect.arrayContaining([ - "access_kind", - "access_ref_json", - "preview_ref_json", - "availability_status", - "provider_metadata_json", - ]), + expect.arrayContaining(["access_kind", "access_ref_json"]), ); expect(tables).toEqual( expect.arrayContaining([ - "slack_backfill_proofs", - "sync_scopes", "sync_proofs", - "jobs", "message_fts_index_queue", "attachment_cache", "attachment_content", @@ -2223,10 +1837,13 @@ describe("CuedDatabase", () => { expect(insertResult.insertedEvents.map((event) => event.id)).toEqual(["event-1", "event-2"]); expect(insertResult.firstInsertedRowId).toBe(1); expect(insertResult.lastInsertedRowId).toBe(2); - expect(db.listRawEvents().map((event) => event.id)).toEqual(["event-1", "event-2"]); - expect(db.listRawEvents().map((event) => event.normalized_schema)).toEqual([ - "message.created@1", - "message.created@1", + expect( + sqlite(db) + .prepare("SELECT id, normalized_schema FROM raw_events ORDER BY observed_at, id") + .all(), + ).toEqual([ + { id: "event-1", normalized_schema: "message.created@1" }, + { id: "event-2", normalized_schema: "message.created@1" }, ]); db.close(); @@ -2250,7 +1867,6 @@ describe("CuedDatabase", () => { fields: { display_name: "One" }, handles: [{ type: "phone", value: "+15551234567", deterministic: true }], }, - sourceVersion: "contacts-v1", }, { id: randomUUID(), @@ -2265,7 +1881,6 @@ describe("CuedDatabase", () => { fields: { display_name: "One" }, handles: [{ type: "phone", value: "+15551234567", deterministic: true }], }, - sourceVersion: "contacts-v1", }, ]); @@ -2309,11 +1924,15 @@ describe("CuedDatabase", () => { ]); insertContact(db, { id: "contact-reset-1", name: "Reset Contact", updatedAt: timestamp }); - db.upsertProjectionState({ - projectionWatermark: 2, - lastProjectedAt: timestamp, - lastRebuildAt: timestamp, - }); + sqlite(db) + .prepare( + ` + UPDATE projection_state + SET projection_watermark = ?, updated_at = ? + WHERE singleton_key = 'global' + `, + ) + .run(2, timestamp); expect(db.getOverview().contacts).toBe(1); expect(db.getProjectionBacklog()).toEqual({ @@ -2345,83 +1964,4 @@ describe("CuedDatabase", () => { db.close(); }); - - it("removes cached attachment files when clearing projected state", () => { - const db = createDb(); - const timestamp = Date.now(); - const cacheDir = mkdtempSync(join(tmpdir(), "cued-attachment-cache-")); - tempDirs.push(cacheDir); - const cachePath = join(cacheDir, "cached.txt"); - writeFileSync(cachePath, "cached attachment payload"); - - const sql = sqlite(db); - sql - .prepare( - ` - INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) - `, - ) - .run("conversation-cache-1", "source-conversation-cache-1", "Thread", timestamp, timestamp); - sql - .prepare( - ` - INSERT INTO messages ( - id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) - `, - ) - .run( - "message-cache-1", - "platform-message-cache-1", - "conversation-cache-1", - timestamp, - timestamp, - timestamp, - ); - sql - .prepare( - ` - INSERT INTO message_attachments ( - id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, - title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'cached.txt', 'Cached', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) - `, - ) - .run( - "attachment-cache-1", - "message-cache-1", - "source-attachment-cache-1", - cachePath, - 25, - JSON.stringify({ path: cachePath }), - timestamp, - timestamp, - ); - db.upsertAttachmentCacheEntry({ - attachmentId: "attachment-cache-1", - variant: "original", - status: "ready", - cachePath, - mimeType: "text/plain", - sizeBytes: 25, - sha256: "abc123", - fetchedAt: timestamp, - lastAccessedAt: timestamp, - }); - - db.clearProjectedState(); - - expect(existsSync(cachePath)).toBe(false); - expect(db.getAttachmentCacheEntry("attachment-cache-1", "original")).toBeNull(); - - db.close(); - }); }); diff --git a/src/db/database.ts b/src/db/database.ts index b9e6afac..1c660ad0 100644 --- a/src/db/database.ts +++ b/src/db/database.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { existsSync, rmSync } from "node:fs"; +import { existsSync } from "node:fs"; import type Database from "better-sqlite3-multiple-ciphers"; import { and, asc, desc, eq, inArray, sql } from "drizzle-orm"; import { type BetterSQLite3Database, drizzle } from "drizzle-orm/better-sqlite3"; @@ -11,22 +11,16 @@ import type { ConnectionKind, IntegrationAuthState, IntegrationLaunchStrategy, - JobKind, - JobStatus, Platform, ProviderRawEventInput, RawEventEntityKind, + SourceAccountInput, SyncMode, SyncProofInput, SyncRunStatus, SyncRunType, } from "../core/types/provider.js"; -import { - normalizeRawEventProvenance, - parsePlatform, - resolveRawEventNormalizedSchema, -} from "../core/types/provider.js"; -import { normalizePhone, toE164 } from "../core/utils/phone.js"; +import { parsePlatform, resolveRawEventNormalizedSchema } from "../core/types/provider.js"; import { assertKnownSyncProofKindContract } from "../platforms/core/proofs.js"; import { assertCanonicalNormalizedSchemaForWrite, @@ -53,27 +47,19 @@ const { contactMemories, contactSources, contacts, - conversationParticipants, conversations, daemonState, integrationStates, - jobs, - messageAttachments, messageFtsIndexQueue, - messageReactions, messages, - outboundMessages, projectionState, rawEventProjectionFailures, rawEvents, sourceAccounts, - slackBackfillProofs, syncProofs, - syncScopes, syncCheckpoints, syncRunErrors, syncRuns, - timelineEvents, } = schema; const APP_SETTING_KEYS = { @@ -111,46 +97,22 @@ export interface QueuedSyncRun { attempt: number; owner_id: string | null; lease_expires_at: number | null; - last_progress_at: number | null; details_json: string | null; } -export interface SyncRunClaim { +interface SyncRunClaim { ownerId: string | null; attempt: number; } -export interface QueuedJob { - id: string; - kind: JobKind; - platform: Platform | null; - account_key: string | null; - priority: number; - status: JobStatus; - trigger: string; - queued_at: number; - scheduled_at: number; - started_at: number | null; - attempt: number; - owner_id: string | null; - lease_expires_at: number | null; - last_progress_at: number | null; - checkpoint_json: string | null; - progress_json: string | null; - error_json: string | null; -} - -export interface MessageFtsIndexQueueRow { +interface MessageFtsIndexQueueRow { message_id: string; - reason: string; status: "queued" | "indexing" | "completed" | "failed"; - attempt: number; queued_at: number; updated_at: number; - last_error: string | null; } -export interface ContactMemoryRow { +interface ContactMemoryRow { id: string; contact_id: string; contact_name: string | null; @@ -171,7 +133,7 @@ export interface ContactMergeBatchInput { reason?: string | null; } -export interface PlannedContactMergeDecision { +interface PlannedContactMergeDecision { decisionId: string; primaryContactId: string; secondaryContactId: string; @@ -179,15 +141,13 @@ export interface PlannedContactMergeDecision { reason: string | null; } -export interface ProjectionStateRow { +interface ProjectionStateRow { singleton_key: "global"; projection_watermark: number; - last_projected_at: number | null; - last_rebuild_at: number | null; updated_at: number; } -export interface AppSettingRow { +interface AppSettingRow { key: string; value: string | null; updated_at: number; @@ -204,66 +164,15 @@ export interface AppMetadataSnapshot { updateLastError: UpdateErrorState | null; } -export interface SyncScopeRow { - id: string; - platform: Platform; - account_key: string; - scope_kind: string; - scope_key: string; - parent_scope_id: string | null; - display_name: string | null; - metadata_json: string | null; - first_discovered_at: number; - last_observed_at: number; - created_at: number; - updated_at: number; -} - -export interface SyncProofRow { - id: string; - platform: Platform; - account_key: string; - scope_id: string; +interface SyncProofRow { scope_kind: string; scope_key: string; - parent_scope_id: string | null; - display_name: string | null; - metadata_json: string | null; proof_kind: string; status: string; - sync_mode: SyncMode | null; - run_started_at: number | null; last_observed_at: number; - completed_at: number | null; - fresh_until: number | null; resume_cursor_json: string | null; coverage_json: string | null; stats_json: string | null; - error_json: string | null; - created_at: number; - updated_at: number; -} - -export interface ContactMergeDecisionRow { - id: string; - decision_type: string; - primary_contact_id: string; - secondary_contact_id: string; - canonical_contact_id: string; - reason: string | null; - created_by: string | null; - created_at: number; -} - -function buildSyncScopeId( - platform: Platform, - accountKey: string, - scopeKind: string, - scopeKey: string, -): string { - return `scope:${Buffer.from(JSON.stringify([platform, accountKey, scopeKind, scopeKey])).toString( - "base64url", - )}`; } function buildSyncProofId( @@ -280,7 +189,7 @@ function buildSyncProofId( export type RawEventInput = ProviderRawEventInput; -export interface WhatsAppHistoryBackfillAnchorRow { +interface WhatsAppHistoryBackfillAnchorRow { chatJID: string; messageID: string; timestamp: number; @@ -330,24 +239,6 @@ export interface AuthSessionRow { updated_at: number; } -export interface OutboundMessageRow { - id: string; - platform: Platform; - account_key: string; - target: string; - thread_id: string | null; - text: string; - status: string; - attempt_count: number; - scheduled_for: number; - started_at: number | null; - finished_at: number | null; - last_error: string | null; - metadata_json: string | null; - created_at: number; - updated_at: number; -} - export interface MessageAttachmentRow { id: string; message_id: string; @@ -364,10 +255,6 @@ export interface MessageAttachmentRow { text_content: string | null; access_kind: string | null; access_ref_json: string | null; - preview_ref_json: string | null; - availability_status: string | null; - provider_metadata_json: string | null; - metadata_json: string | null; created_at: number; updated_at: number; conversation_id?: string; @@ -395,7 +282,7 @@ export interface AttachmentCacheRow { updated_at: number; } -export interface AttachmentContentRow { +interface AttachmentContentRow { attachment_id: string; extractor: string | null; status: string; @@ -407,113 +294,6 @@ export interface AttachmentContentRow { updated_at: number; } -export interface SignalSendResolution { - target: string; - threadId: string; - resolution: "group" | "signal_id" | "signal_phone" | "phone" | "imessage_phone" | "passthrough"; - matchedContactIds: string[]; - matchedName: string | null; -} - -export interface WhatsAppSendResolution { - target: string; - threadId: string; - resolution: "whatsapp_jid" | "phone" | "group" | "passthrough"; - matchedContactIds: string[]; - matchedName: string | null; -} - -export interface DiscordSendResolution { - target: string; - threadId: string | null; - resolution: "channel_id" | "source_conversation_key" | "conversation_name"; - matchedConversationId: string | null; - matchedName: string | null; -} - -type SignalSendCandidate = { - rank: number; - target: string; - resolution: Extract< - SignalSendResolution["resolution"], - "signal_id" | "signal_phone" | "phone" | "imessage_phone" - >; -}; - -type WhatsAppSendCandidate = { - rank: number; - target: string; - resolution: Extract; -}; - -const SIGNAL_UUID_PATTERN = - /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; - -function isSignalUuid(value: string): boolean { - return SIGNAL_UUID_PATTERN.test(value.trim()); -} - -function normalizeSignalThreadRecipient( - value: string, - resolution: SignalSendResolution["resolution"], -): string { - const trimmed = value.trim(); - if (resolution === "signal_id") { - return trimmed.toLowerCase(); - } - const e164Phone = toE164(trimmed); - if (e164Phone) { - return e164Phone; - } - const normalizedPhone = normalizePhone(trimmed); - if (normalizedPhone) { - return normalizedPhone; - } - return trimmed.toLowerCase(); -} - -function buildSignalDmThreadId( - recipient: string, - resolution: SignalSendResolution["resolution"], -): string { - return `dm:${normalizeSignalThreadRecipient(recipient, resolution)}`; -} - -function normalizeSignalHandleLookupValue(value: string): string { - const trimmed = value.trim(); - const normalizedPhone = normalizePhone(trimmed); - if (normalizedPhone) { - return normalizedPhone; - } - return trimmed.toLowerCase(); -} - -function normalizeWhatsAppJid(value: string): string { - return value.trim().toLowerCase(); -} - -function buildWhatsAppThreadId(target: string): string { - const normalized = normalizeWhatsAppJid(target); - return normalized.endsWith("@g.us") ? `group:${normalized}` : `dm:${normalized}`; -} - -function normalizeWhatsAppHandleLookupValue(value: string): string { - const trimmed = value.trim(); - const normalizedPhone = normalizePhone(trimmed); - if (normalizedPhone) { - return normalizedPhone; - } - return normalizeWhatsAppJid(trimmed); -} - -function toWhatsAppJidFromPhone(value: string): string | null { - const e164Phone = toE164(value); - if (!e164Phone) { - return null; - } - return `${e164Phone.slice(1)}@s.whatsapp.net`; -} - export type LocalDrizzleDatabase = BetterSQLite3Database; const WRITE_BATCH_SIZE = 250; @@ -554,7 +334,6 @@ function sqlValueList(values: readonly (string | number)[]) { } function buildRawEventValues(event: RawEventInput) { - const provenance = normalizeRawEventProvenance(event.provenance); const normalizedSchema = resolveRawEventNormalizedSchema(event); assertCanonicalNormalizedSchemaForWrite(normalizedSchema); assertCanonicalRawEventPayloadForWrite({ ...event, normalizedSchema }); @@ -564,17 +343,13 @@ function buildRawEventValues(event: RawEventInput) { accountKey: event.accountKey, entityKind: event.entityKind, eventKind: event.eventKind, - externalEventId: event.externalEventId ?? null, externalEntityId: event.externalEntityId ?? null, conversationExternalId: event.conversationExternalId ?? null, occurredAt: event.occurredAt ?? null, observedAt: event.observedAt, - cursorJson: safeStringifyJson(event.cursor), dedupeKey: event.dedupeKey, payloadJson: safeStringifyJson(event.payload) ?? "null", normalizedSchema, - provenanceJson: safeStringifyJson(provenance), - sourceVersion: event.sourceVersion ?? null, }; } @@ -657,23 +432,6 @@ export class CuedDatabase { return statement.all(); } - listContactMergeDecisions(): ContactMergeDecisionRow[] { - return this.db - .select({ - id: contactMergeDecisions.id, - decision_type: contactMergeDecisions.decisionType, - primary_contact_id: contactMergeDecisions.primaryContactId, - secondary_contact_id: contactMergeDecisions.secondaryContactId, - canonical_contact_id: contactMergeDecisions.canonicalContactId, - reason: contactMergeDecisions.reason, - created_by: contactMergeDecisions.createdBy, - created_at: contactMergeDecisions.createdAt, - }) - .from(contactMergeDecisions) - .orderBy(asc(contactMergeDecisions.createdAt), asc(contactMergeDecisions.id)) - .all() as ContactMergeDecisionRow[]; - } - listContactMergeAliases(): Array<{ contact_id: string; canonical_contact_id: string; @@ -684,7 +442,6 @@ export class CuedDatabase { canonical_contact_id: contactMergeDecisions.canonicalContactId, }) .from(contactMergeDecisions) - .where(eq(contactMergeDecisions.decisionType, "merge")) .orderBy(asc(contactMergeDecisions.createdAt), asc(contactMergeDecisions.id)) .all() as Array<{ contact_id: string; @@ -692,13 +449,6 @@ export class CuedDatabase { }>; } - resolveCanonicalContactId(contactId: string): string { - const aliasMap = new Map( - this.listContactMergeAliases().map((row) => [row.contact_id, row.canonical_contact_id]), - ); - return resolveCanonicalContactIdFromAliases(contactId, aliasMap); - } - planContactMergeDecisions(input: ContactMergeBatchInput[]): PlannedContactMergeDecision[] { if (input.length === 0) { throw new Error("At least one contact merge is required."); @@ -783,7 +533,6 @@ export class CuedDatabase { .insert(contactMergeDecisions) .values({ id: decision.decisionId, - decisionType: "merge", primaryContactId: decision.primaryContactId, secondaryContactId: decision.secondaryContactId, canonicalContactId: decision.canonicalContactId, @@ -818,18 +567,6 @@ export class CuedDatabase { } } - listAppSettings(): AppSettingRow[] { - return this.db - .select({ - key: appSettings.key, - value: appSettings.value, - updated_at: appSettings.updatedAt, - }) - .from(appSettings) - .orderBy(asc(appSettings.key)) - .all() as AppSettingRow[]; - } - getAppSetting(key: string): AppSettingRow | null { const row = this.db .select({ @@ -862,26 +599,11 @@ export class CuedDatabase { .run(); } - recordAppMetadata(input: { - version: string; - releaseChannel: string; - cliSymlinkInstalled?: boolean | null; - }): void { - this.setAppSetting(APP_SETTING_KEYS.installedAppVersion, input.version); - this.setAppSetting(APP_SETTING_KEYS.releaseChannel, input.releaseChannel); - if (typeof input.cliSymlinkInstalled === "boolean") { - this.setAppSetting( - APP_SETTING_KEYS.cliSymlinkInstalled, - input.cliSymlinkInstalled ? "1" : "0", - ); - } - } - markOnboardingCompleted(version: string): void { this.setAppSetting(APP_SETTING_KEYS.onboardingCompletedVersion, version); } - markReleaseCheck(at = now()): void { + private markReleaseCheck(at = now()): void { this.setAppSetting(APP_SETTING_KEYS.lastReleaseCheckAt, String(at)); } @@ -916,7 +638,7 @@ export class CuedDatabase { ); } - getUpdateLastError(): UpdateErrorState | null { + private getUpdateLastError(): UpdateErrorState | null { const raw = this.getAppSetting(APP_SETTING_KEYS.updateLastError)?.value ?? null; const parsed = safeParseJson( raw, @@ -954,7 +676,14 @@ export class CuedDatabase { } getAppMetadata(): AppMetadataSnapshot { - const byKey = new Map(this.listAppSettings().map((row) => [row.key, row.value])); + const byKey = new Map( + ( + this.db + .select({ key: appSettings.key, value: appSettings.value }) + .from(appSettings) + .all() as Array<{ key: string; value: string | null }> + ).map((row) => [row.key, row.value]), + ); const lastReleaseCheckAt = Number(byKey.get(APP_SETTING_KEYS.lastReleaseCheckAt) ?? ""); return { onboardingCompletedVersion: byKey.get(APP_SETTING_KEYS.onboardingCompletedVersion) ?? null, @@ -1124,190 +853,44 @@ export class CuedDatabase { .delete(syncProofs) .where(eq(syncProofs.platform, platform)) .run().changes; - const removedSyncScopes = tx - .delete(syncScopes) - .where(eq(syncScopes.platform, platform)) - .run().changes; - const removedSlackBackfillProofs = - platform === "slack" ? tx.delete(slackBackfillProofs).run().changes : 0; return ( Number(removedSourceAccounts) + Number(removedRawEvents) + Number(removedRuns) + Number(removedErrors) + Number(removedCheckpoints) + - Number(removedSyncProofs) + - Number(removedSyncScopes) + - Number(removedSlackBackfillProofs) + Number(removedSyncProofs) ); }); this.upsertProjectionState({ projectionWatermark: 0, - lastProjectedAt: null, - lastRebuildAt: null, }); return removed; } - listSyncScopes(platform: Platform, accountKey: string): SyncScopeRow[] { - return this.db - .select({ - id: syncScopes.id, - platform: syncScopes.platform, - account_key: syncScopes.accountKey, - scope_kind: syncScopes.scopeKind, - scope_key: syncScopes.scopeKey, - parent_scope_id: syncScopes.parentScopeId, - display_name: syncScopes.displayName, - metadata_json: syncScopes.metadataJson, - first_discovered_at: syncScopes.firstDiscoveredAt, - last_observed_at: syncScopes.lastObservedAt, - created_at: syncScopes.createdAt, - updated_at: syncScopes.updatedAt, - }) - .from(syncScopes) - .where(and(eq(syncScopes.platform, platform), eq(syncScopes.accountKey, accountKey))) - .orderBy(asc(syncScopes.scopeKind), asc(syncScopes.scopeKey)) - .all() as SyncScopeRow[]; - } - listSyncProofs(platform: Platform, accountKey: string): SyncProofRow[] { return this.db .select({ - id: syncProofs.id, - platform: syncProofs.platform, - account_key: syncProofs.accountKey, - scope_id: syncProofs.scopeId, - scope_kind: syncScopes.scopeKind, - scope_key: syncScopes.scopeKey, - parent_scope_id: syncScopes.parentScopeId, - display_name: syncScopes.displayName, - metadata_json: syncScopes.metadataJson, + scope_kind: syncProofs.scopeKind, + scope_key: syncProofs.scopeKey, proof_kind: syncProofs.proofKind, status: syncProofs.status, - sync_mode: syncProofs.syncMode, - run_started_at: syncProofs.runStartedAt, last_observed_at: syncProofs.lastObservedAt, - completed_at: syncProofs.completedAt, - fresh_until: syncProofs.freshUntil, resume_cursor_json: syncProofs.resumeCursorJson, coverage_json: syncProofs.coverageJson, stats_json: syncProofs.statsJson, - error_json: syncProofs.errorJson, - created_at: syncProofs.createdAt, - updated_at: syncProofs.updatedAt, }) .from(syncProofs) - .innerJoin(syncScopes, eq(syncProofs.scopeId, syncScopes.id)) .where(and(eq(syncProofs.platform, platform), eq(syncProofs.accountKey, accountKey))) - .orderBy(asc(syncScopes.scopeKind), asc(syncScopes.scopeKey), asc(syncProofs.proofKind)) + .orderBy(asc(syncProofs.scopeKind), asc(syncProofs.scopeKey), asc(syncProofs.proofKind)) .all() as SyncProofRow[]; } upsertSyncProof(input: { platform: Platform; accountKey: string; proof: SyncProofInput }): void { assertKnownSyncProofKindContract(input.platform, input.proof); const timestamp = now(); - const scopeId = buildSyncScopeId( - input.platform, - input.accountKey, - input.proof.scope.kind, - input.proof.scope.key, - ); - const parentScopeId = input.proof.scope.parent - ? buildSyncScopeId( - input.platform, - input.accountKey, - input.proof.scope.parent.kind, - input.proof.scope.parent.key, - ) - : null; - const existingScope = this.db - .select({ - displayName: syncScopes.displayName, - metadataJson: syncScopes.metadataJson, - firstDiscoveredAt: syncScopes.firstDiscoveredAt, - }) - .from(syncScopes) - .where(eq(syncScopes.id, scopeId)) - .get(); - const scopeDisplayName = - input.proof.scope.displayName === undefined - ? (existingScope?.displayName ?? null) - : input.proof.scope.displayName; - const scopeMetadataJson = - input.proof.scope.metadata === undefined - ? (existingScope?.metadataJson ?? null) - : safeStringifyJson(input.proof.scope.metadata); - const scopeFirstDiscoveredAt = existingScope?.firstDiscoveredAt ?? input.proof.observedAt; - - if (input.proof.scope.parent) { - this.db - .insert(syncScopes) - .values({ - id: parentScopeId!, - platform: input.platform, - accountKey: input.accountKey, - scopeKind: input.proof.scope.parent.kind, - scopeKey: input.proof.scope.parent.key, - parentScopeId: null, - displayName: null, - metadataJson: null, - firstDiscoveredAt: input.proof.observedAt, - lastObservedAt: input.proof.observedAt, - createdAt: timestamp, - updatedAt: timestamp, - }) - .onConflictDoUpdate({ - target: [ - syncScopes.platform, - syncScopes.accountKey, - syncScopes.scopeKind, - syncScopes.scopeKey, - ], - set: { - lastObservedAt: input.proof.observedAt, - updatedAt: timestamp, - }, - }) - .run(); - } - - this.db - .insert(syncScopes) - .values({ - id: scopeId, - platform: input.platform, - accountKey: input.accountKey, - scopeKind: input.proof.scope.kind, - scopeKey: input.proof.scope.key, - parentScopeId, - displayName: scopeDisplayName, - metadataJson: scopeMetadataJson, - firstDiscoveredAt: scopeFirstDiscoveredAt, - lastObservedAt: input.proof.observedAt, - createdAt: timestamp, - updatedAt: timestamp, - }) - .onConflictDoUpdate({ - target: [ - syncScopes.platform, - syncScopes.accountKey, - syncScopes.scopeKind, - syncScopes.scopeKey, - ], - set: { - parentScopeId, - displayName: scopeDisplayName, - metadataJson: scopeMetadataJson, - firstDiscoveredAt: scopeFirstDiscoveredAt, - lastObservedAt: input.proof.observedAt, - updatedAt: timestamp, - }, - }) - .run(); - const proofId = buildSyncProofId( input.platform, input.accountKey, @@ -1315,24 +898,9 @@ export class CuedDatabase { input.proof.scope.key, input.proof.proofKind, ); - const existingProof = this.db - .select({ - completedAt: syncProofs.completedAt, - }) - .from(syncProofs) - .where(eq(syncProofs.id, proofId)) - .get(); - const completedAt = - input.proof.status === "complete" - ? (existingProof?.completedAt ?? input.proof.completedAt ?? input.proof.observedAt) - : (input.proof.completedAt ?? null); - const proofSyncMode = input.proof.syncMode ?? null; - const proofRunStartedAt = input.proof.runStartedAt ?? null; - const proofFreshUntil = input.proof.freshUntil ?? null; const resumeCursorJson = safeStringifyJson(input.proof.resumeCursor); const coverageJson = safeStringifyJson(input.proof.coverage); const statsJson = safeStringifyJson(input.proof.stats); - const errorJson = safeStringifyJson(input.proof.error); this.db .insert(syncProofs) @@ -1340,18 +908,14 @@ export class CuedDatabase { id: proofId, platform: input.platform, accountKey: input.accountKey, - scopeId, + scopeKind: input.proof.scope.kind, + scopeKey: input.proof.scope.key, proofKind: input.proof.proofKind, status: input.proof.status, - syncMode: proofSyncMode, - runStartedAt: proofRunStartedAt, lastObservedAt: input.proof.observedAt, - completedAt, - freshUntil: proofFreshUntil, resumeCursorJson, coverageJson, statsJson, - errorJson, createdAt: timestamp, updatedAt: timestamp, }) @@ -1359,20 +923,16 @@ export class CuedDatabase { target: [ syncProofs.platform, syncProofs.accountKey, - syncProofs.scopeId, + syncProofs.scopeKind, + syncProofs.scopeKey, syncProofs.proofKind, ], set: { status: input.proof.status, - syncMode: proofSyncMode, - runStartedAt: proofRunStartedAt, lastObservedAt: input.proof.observedAt, - completedAt, - freshUntil: proofFreshUntil, resumeCursorJson, coverageJson, statsJson, - errorJson, updatedAt: timestamp, }, }) @@ -1430,7 +990,7 @@ export class CuedDatabase { }; } - listMessageCountsByPlatform(): Array<{ + private listMessageCountsByPlatform(): Array<{ platform: Platform; messages: number; }> { @@ -1469,8 +1029,6 @@ export class CuedDatabase { .select({ singleton_key: projectionState.singletonKey, projection_watermark: projectionState.projectionWatermark, - last_projected_at: projectionState.lastProjectedAt, - last_rebuild_at: projectionState.lastRebuildAt, updated_at: projectionState.updatedAt, }) .from(projectionState) @@ -1484,30 +1042,20 @@ export class CuedDatabase { const fallback: ProjectionStateRow = { singleton_key: "global", projection_watermark: 0, - last_projected_at: null, - last_rebuild_at: null, updated_at: now(), }; if (options.initialize !== false) { this.upsertProjectionState({ projectionWatermark: 0, - lastProjectedAt: null, - lastRebuildAt: null, }); } return fallback; } - upsertProjectionState(input: { - projectionWatermark: number; - lastProjectedAt?: number | null; - lastRebuildAt?: number | null; - }): void { + private upsertProjectionState(input: { projectionWatermark: number }): void { const values = { singletonKey: "global" as const, projectionWatermark: input.projectionWatermark, - lastProjectedAt: input.lastProjectedAt ?? null, - lastRebuildAt: input.lastRebuildAt ?? null, updatedAt: now(), }; @@ -1518,8 +1066,6 @@ export class CuedDatabase { target: projectionState.singletonKey, set: { projectionWatermark: values.projectionWatermark, - lastProjectedAt: values.lastProjectedAt, - lastRebuildAt: values.lastRebuildAt, updatedAt: values.updatedAt, }, }) @@ -1670,17 +1216,6 @@ export class CuedDatabase { return rows.filter(hasKnownPlatform); } - listEnabledSyncPlatforms(): Platform[] { - return this.db - .selectDistinct({ platform: integrationStates.platform }) - .from(integrationStates) - .where(and(eq(integrationStates.enabled, 1), eq(integrationStates.syncCapable, 1))) - .orderBy(asc(integrationStates.platform)) - .all() - .filter(hasKnownPlatform) - .map((row) => row.platform); - } - listEnabledSyncTargets(): Array<{ platform: Platform; account_key: string }> { const rows = this.db .select({ @@ -2029,10 +1564,6 @@ export class CuedDatabase { .delete(syncProofs) .where(and(eq(syncProofs.platform, platform), eq(syncProofs.accountKey, accountKey))) .run().changes; - const removedSyncScopes = tx - .delete(syncScopes) - .where(and(eq(syncScopes.platform, platform), eq(syncScopes.accountKey, accountKey))) - .run().changes; const removedRunErrors = tx .delete(syncRunErrors) .where(and(eq(syncRunErrors.platform, platform), eq(syncRunErrors.accountKey, accountKey))) @@ -2041,59 +1572,16 @@ export class CuedDatabase { .delete(syncRuns) .where(and(eq(syncRuns.platform, platform), eq(syncRuns.accountKey, accountKey))) .run().changes; - const removedSlackBackfillProofs = - platform === "slack" - ? tx - .delete(slackBackfillProofs) - .where(eq(slackBackfillProofs.accountKey, accountKey)) - .run().changes - : 0; - return ( Number(removedSourceAccounts) + Number(removedCheckpoints) + Number(removedSyncProofs) + - Number(removedSyncScopes) + Number(removedRunErrors) + - Number(removedRuns) + - Number(removedSlackBackfillProofs) + Number(removedRuns) ); }); } - queueOutboundMessage(input: { - platform: Platform; - accountKey: string; - target: string; - threadId?: string | null; - text: string; - metadata?: unknown; - }): string { - const id = randomUUID(); - const timestamp = now(); - this.db - .insert(outboundMessages) - .values({ - id, - platform: input.platform, - accountKey: input.accountKey, - target: input.target, - threadId: input.threadId ?? null, - text: input.text, - status: "pending", - attemptCount: 0, - scheduledFor: timestamp, - startedAt: null, - finishedAt: null, - lastError: null, - metadataJson: safeStringifyJson(input.metadata), - createdAt: timestamp, - updatedAt: timestamp, - }) - .run(); - return id; - } - addContactMemory(input: { contactId: string; body: string; @@ -2166,7 +1654,7 @@ export class CuedDatabase { return this.getContactMemory(id)!; } - getContactMemory(id: string): ContactMemoryRow | null { + private getContactMemory(id: string): ContactMemoryRow | null { return ( (this.sqlite .prepare( @@ -2254,417 +1742,21 @@ export class CuedDatabase { return memory; } - resolveSignalSendTarget(targetInput: string): SignalSendResolution | null { - const trimmed = targetInput.trim(); - if (trimmed.length === 0) { - return null; - } - - if (trimmed.startsWith("group:")) { - return { - target: trimmed, - threadId: trimmed, - resolution: "group", - matchedContactIds: [], - matchedName: null, - }; - } - - const directLookupValue = normalizeSignalHandleLookupValue(trimmed); - const explicitContact = this.sqlite - .prepare( - ` - SELECT id, name - FROM contacts - WHERE id = ? - LIMIT 1 - `, - ) - .get(trimmed) as { id: string; name: string | null } | undefined; - - const matchingHandleContacts = this.sqlite - .prepare( - ` - SELECT DISTINCT c.id, c.name - FROM contacts c - JOIN contact_handles h ON h.contact_id = c.id - WHERE lower(h.value) = lower(?) - OR lower(h.normalized_value) = lower(?) - ORDER BY c.updated_at DESC, c.id ASC - `, - ) - .all(trimmed, directLookupValue) as Array<{ id: string; name: string | null }>; - - const exactNameContacts = this.sqlite - .prepare( - ` - SELECT id, name - FROM contacts - WHERE lower(name) = lower(?) - ORDER BY updated_at DESC, id ASC - `, - ) - .all(trimmed) as Array<{ id: string; name: string | null }>; - - const seedContacts = explicitContact - ? [explicitContact] - : matchingHandleContacts.length > 0 - ? matchingHandleContacts - : exactNameContacts; - - const matchedName = - seedContacts.find( - (contact) => typeof contact.name === "string" && contact.name.trim().length > 0, - )?.name ?? null; - - const contactIds = new Set(seedContacts.map((contact) => contact.id)); - if (matchedName) { - const sameNameContacts = this.sqlite - .prepare( - ` - SELECT id - FROM contacts - WHERE lower(name) = lower(?) - ORDER BY updated_at DESC, id ASC - `, - ) - .all(matchedName) as Array<{ id: string }>; - for (const contact of sameNameContacts) { - contactIds.add(contact.id); - } - } - - const candidateContactIds = [...contactIds]; - if (candidateContactIds.length > 0) { - const placeholders = candidateContactIds.map(() => "?").join(", "); - const handles = this.sqlite - .prepare( - ` - SELECT contact_id, type, value, normalized_value, platform - FROM contact_handles - WHERE contact_id IN (${placeholders}) - ORDER BY contact_id ASC, platform ASC, type ASC - `, - ) - .all(...candidateContactIds) as Array<{ - contact_id: string; - type: string; - value: string; - normalized_value: string; - platform: string | null; - }>; - - const rankedHandle = handles - .map((handle) => { - if (handle.platform === "signal" && handle.type === "signal_id") { - const recipient = handle.normalized_value.trim().toLowerCase(); - return { - rank: 0, - target: recipient, - resolution: "signal_id" as const, - } satisfies SignalSendCandidate; - } - if (handle.platform === "signal" && handle.type === "phone") { - const recipient = toE164(handle.value) || toE164(handle.normalized_value); - if (!recipient) { - return null; - } - return { - rank: 1, - target: recipient, - resolution: "signal_phone" as const, - } satisfies SignalSendCandidate; - } - if (handle.type === "phone") { - const recipient = toE164(handle.value) || toE164(handle.normalized_value); - if (!recipient) { - return null; - } - return { - rank: 2, - target: recipient, - resolution: "phone" as const, - } satisfies SignalSendCandidate; - } - if (handle.type === "imessage_handle") { - const recipient = toE164(handle.value) || toE164(handle.normalized_value); - if (!recipient) { - return null; - } - return { - rank: 3, - target: recipient, - resolution: "imessage_phone" as const, - } satisfies SignalSendCandidate; - } - return null; - }) - .filter((value): value is SignalSendCandidate => value !== null) - .sort((left, right) => left.rank - right.rank || left.target.localeCompare(right.target)); - - if (rankedHandle[0]) { - return { - target: rankedHandle[0].target, - threadId: buildSignalDmThreadId(rankedHandle[0].target, rankedHandle[0].resolution), - resolution: rankedHandle[0].resolution, - matchedContactIds: candidateContactIds, - matchedName, - }; - } - } - - if (isSignalUuid(trimmed)) { - return { - target: trimmed.toLowerCase(), - threadId: buildSignalDmThreadId(trimmed, "signal_id"), - resolution: "signal_id", - matchedContactIds: [], - matchedName: null, - }; - } - - const directPhone = toE164(trimmed); - if (directPhone) { - return { - target: directPhone, - threadId: buildSignalDmThreadId(directPhone, "passthrough"), - resolution: "passthrough", - matchedContactIds: candidateContactIds, - matchedName, - }; - } - - return null; - } - - resolveWhatsAppSendTarget(targetInput: string): WhatsAppSendResolution | null { - const trimmed = targetInput.trim(); - if (trimmed.length === 0) { - return null; - } - - const normalizedInput = normalizeWhatsAppHandleLookupValue(trimmed); - const directJid = normalizeWhatsAppJid(trimmed); - if (directJid.endsWith("@g.us")) { - return { - target: directJid, - threadId: `group:${directJid}`, - resolution: "group", - matchedContactIds: [], - matchedName: null, - }; - } - - const explicitContact = this.sqlite - .prepare( - ` - SELECT id, name - FROM contacts - WHERE id = ? - LIMIT 1 - `, - ) - .get(trimmed) as { id: string; name: string | null } | undefined; - - const matchingHandleContacts = this.sqlite - .prepare( - ` - SELECT DISTINCT c.id, c.name - FROM contacts c - JOIN contact_handles h ON h.contact_id = c.id - WHERE lower(h.value) = lower(?) - OR lower(h.normalized_value) = lower(?) - ORDER BY c.updated_at DESC, c.id ASC - `, - ) - .all(trimmed, normalizedInput) as Array<{ id: string; name: string | null }>; - - const exactNameContacts = this.sqlite - .prepare( - ` - SELECT id, name - FROM contacts - WHERE lower(name) = lower(?) - ORDER BY updated_at DESC, id ASC - `, - ) - .all(trimmed) as Array<{ id: string; name: string | null }>; - - const seedContacts = explicitContact - ? [explicitContact] - : matchingHandleContacts.length > 0 - ? matchingHandleContacts - : exactNameContacts; - const matchedName = - seedContacts.find( - (contact) => typeof contact.name === "string" && contact.name.trim().length > 0, - )?.name ?? null; - - const contactIds = new Set(seedContacts.map((contact) => contact.id)); - if (matchedName) { - const sameNameContacts = this.sqlite - .prepare( - ` - SELECT id - FROM contacts - WHERE lower(name) = lower(?) - ORDER BY updated_at DESC, id ASC - `, - ) - .all(matchedName) as Array<{ id: string }>; - for (const contact of sameNameContacts) { - contactIds.add(contact.id); - } - } - - const candidateContactIds = [...contactIds]; - if (candidateContactIds.length > 0) { - const placeholders = candidateContactIds.map(() => "?").join(", "); - const handles = this.sqlite - .prepare( - ` - SELECT contact_id, type, value, normalized_value, platform - FROM contact_handles - WHERE contact_id IN (${placeholders}) - ORDER BY contact_id ASC, platform ASC, type ASC - `, - ) - .all(...candidateContactIds) as Array<{ - contact_id: string; - type: string; - value: string; - normalized_value: string; - platform: string | null; - }>; - - const rankedHandle = handles - .map((handle) => { - if (handle.platform === "whatsapp" && handle.type === "whatsapp_jid") { - return { - rank: 0, - target: normalizeWhatsAppJid(handle.normalized_value || handle.value), - resolution: "whatsapp_jid" as const, - } satisfies WhatsAppSendCandidate; - } - if (handle.type === "phone") { - const recipient = - toWhatsAppJidFromPhone(handle.value) || - toWhatsAppJidFromPhone(handle.normalized_value); - if (!recipient) { - return null; - } - return { - rank: 1, - target: recipient, - resolution: "phone" as const, - } satisfies WhatsAppSendCandidate; - } - return null; - }) - .filter((value): value is WhatsAppSendCandidate => value !== null) - .sort((left, right) => left.rank - right.rank || left.target.localeCompare(right.target)); - - if (rankedHandle[0]) { - return { - target: rankedHandle[0].target, - threadId: buildWhatsAppThreadId(rankedHandle[0].target), - resolution: rankedHandle[0].resolution, - matchedContactIds: candidateContactIds, - matchedName, - }; - } - } - - if (directJid.includes("@")) { - return { - target: directJid, - threadId: buildWhatsAppThreadId(directJid), - resolution: directJid.endsWith("@g.us") ? "group" : "passthrough", - matchedContactIds: candidateContactIds, - matchedName, - }; - } - - const directPhone = toWhatsAppJidFromPhone(trimmed); - if (directPhone) { - return { - target: directPhone, - threadId: buildWhatsAppThreadId(directPhone), - resolution: "passthrough", - matchedContactIds: candidateContactIds, - matchedName, - }; - } - - return null; - } - - resolveDiscordSendTarget(targetInput: string): DiscordSendResolution | null { - const trimmed = targetInput.trim(); - if (trimmed.length === 0) { - return null; - } - - const exact = this.sqlite - .prepare( - ` - SELECT id, source_conversation_key, native_conversation_key, name - FROM conversations - WHERE platform = 'discord' - AND type IN ('dm', 'group') - AND ( - source_conversation_key = ? - OR native_conversation_key = ? - OR lower(name) = lower(?) - ) - ORDER BY updated_at DESC, id ASC - LIMIT 1 - `, - ) - .get(trimmed, trimmed, trimmed) as - | { - id: string; - source_conversation_key: string; - native_conversation_key: string | null; - name: string | null; - } - | undefined; - - if (!exact) { - return null; - } - - return { - target: - exact.native_conversation_key ?? - exact.source_conversation_key.replace(/^discord:channel:/, ""), - threadId: exact.source_conversation_key, - resolution: - exact.source_conversation_key === trimmed - ? "source_conversation_key" - : exact.native_conversation_key === trimmed - ? "channel_id" - : "conversation_name", - matchedConversationId: exact.id, - matchedName: exact.name, - }; - } - - findMessageByPlatformKey( - platform: Platform, - accountKey: string, - platformMessageId: string, - ): { - id: string; - sender_source_key: string | null; - sent_at: number | null; - content: string | null; - status: string | null; - delivered_at: number | null; - read_at: number | null; - } | null { - return ( - (this.sqlite + findMessageByPlatformKey( + platform: Platform, + accountKey: string, + platformMessageId: string, + ): { + id: string; + sender_source_key: string | null; + sent_at: number | null; + content: string | null; + status: string | null; + delivered_at: number | null; + read_at: number | null; + } | null { + return ( + (this.sqlite .prepare( ` SELECT @@ -2778,10 +1870,6 @@ export class CuedDatabase { ma.text_content, ma.access_kind, ma.access_ref_json, - ma.preview_ref_json, - ma.availability_status, - ma.provider_metadata_json, - ma.metadata_json, ma.created_at, ma.updated_at, m.conversation_id, @@ -3079,166 +2167,7 @@ export class CuedDatabase { }>; } - claimNextOutboundMessage(platform?: Platform): OutboundMessageRow | null { - return this.db.transaction((tx) => { - const whereCondition = platform - ? and( - eq(outboundMessages.status, "pending"), - eq(outboundMessages.platform, platform), - sql`${outboundMessages.scheduledFor} <= ${now()}`, - ) - : and( - eq(outboundMessages.status, "pending"), - sql`${outboundMessages.scheduledFor} <= ${now()}`, - ); - - const row = tx - .select({ - id: outboundMessages.id, - platform: outboundMessages.platform, - account_key: outboundMessages.accountKey, - target: outboundMessages.target, - thread_id: outboundMessages.threadId, - text: outboundMessages.text, - status: outboundMessages.status, - attempt_count: outboundMessages.attemptCount, - scheduled_for: outboundMessages.scheduledFor, - started_at: outboundMessages.startedAt, - finished_at: outboundMessages.finishedAt, - last_error: outboundMessages.lastError, - metadata_json: outboundMessages.metadataJson, - created_at: outboundMessages.createdAt, - updated_at: outboundMessages.updatedAt, - }) - .from(outboundMessages) - .where(whereCondition) - .orderBy(asc(outboundMessages.scheduledFor), asc(outboundMessages.createdAt)) - .limit(1) - .get() as OutboundMessageRow | undefined; - - if (!row) { - return null; - } - - tx.update(outboundMessages) - .set({ - status: "sending", - attemptCount: row.attempt_count + 1, - startedAt: now(), - updatedAt: now(), - }) - .where(eq(outboundMessages.id, row.id)) - .run(); - - return { - ...row, - status: "sending", - attempt_count: row.attempt_count + 1, - started_at: now(), - updated_at: now(), - }; - }); - } - - completeOutboundMessage(id: string): void { - this.db - .update(outboundMessages) - .set({ - status: "sent", - finishedAt: now(), - updatedAt: now(), - lastError: null, - }) - .where(eq(outboundMessages.id, id)) - .run(); - } - - failOutboundMessage(input: { - id: string; - retryable: boolean; - error: string; - retryDelayMs?: number; - maxAttempts?: number; - }): void { - const current = this.db - .select({ - attempt_count: outboundMessages.attemptCount, - }) - .from(outboundMessages) - .where(eq(outboundMessages.id, input.id)) - .get() as { attempt_count: number } | undefined; - - if (!current) { - throw new Error(`Outbound message not found: ${input.id}`); - } - - const shouldRetry = input.retryable && current.attempt_count < (input.maxAttempts ?? 3); - this.db - .update(outboundMessages) - .set({ - status: shouldRetry ? "pending" : "failed", - scheduledFor: shouldRetry ? now() + (input.retryDelayMs ?? 5_000) : now(), - finishedAt: shouldRetry ? null : now(), - updatedAt: now(), - lastError: input.error, - }) - .where(eq(outboundMessages.id, input.id)) - .run(); - } - - hasQueuedOutboundMessages(platform?: Platform): boolean { - const whereCondition = platform - ? and(eq(outboundMessages.status, "pending"), eq(outboundMessages.platform, platform)) - : eq(outboundMessages.status, "pending"); - const row = this.db - .select({ id: outboundMessages.id }) - .from(outboundMessages) - .where(whereCondition) - .limit(1) - .get(); - return Boolean(row); - } - - failInProgressRuns(errorMessage: string): number { - return this.db.transaction((tx) => { - const stuckRuns = tx - .select({ - id: syncRuns.id, - }) - .from(syncRuns) - .where(sql`${syncRuns.status} IN ('ingesting', 'projecting')`) - .all(); - - if (stuckRuns.length === 0) { - return 0; - } - - const finishedAt = now(); - for (const run of stuckRuns) { - tx.update(syncRuns) - .set({ - status: "failed", - finishedAt, - }) - .where(eq(syncRuns.id, run.id)) - .run(); - - tx.insert(syncRunErrors) - .values({ - id: randomUUID(), - syncRunId: run.id, - errorMessage, - detailsJson: null, - createdAt: finishedAt, - }) - .run(); - } - - return stuckRuns.length; - }); - } - - recoverInProgressRuns(errorCode = "daemon_recovered"): number { + recoverInProgressRuns(): number { const timestamp = now(); const result = this.db .update(syncRuns) @@ -3247,8 +2176,6 @@ export class CuedDatabase { scheduledAt: timestamp, ownerId: null, leaseExpiresAt: null, - lastProgressAt: timestamp, - errorCode, }) .where(sql`${syncRuns.status} IN ('ingesting', 'projecting')`) .run(); @@ -3287,8 +2214,6 @@ export class CuedDatabase { attempt: 0, ownerId: null, leaseExpiresAt: null, - lastProgressAt: null, - errorCode: null, detailsJson: safeStringifyJson(input.details), }) .run(); @@ -3336,8 +2261,6 @@ export class CuedDatabase { attempt: 0, ownerId: null, leaseExpiresAt: null, - lastProgressAt: null, - errorCode: null, detailsJson: safeStringifyJson(input.details), }; }); @@ -3349,189 +2272,7 @@ export class CuedDatabase { return runIds; } - queueJob(input: { - kind: JobKind; - platform?: Platform | null; - accountKey?: string | null; - priority: number; - trigger: string; - checkpoint?: unknown; - progress?: unknown; - scheduledAt?: number; - delayMs?: number; - }): string { - const id = randomUUID(); - const queuedAt = now(); - const scheduledAt = - input.scheduledAt ?? - (input.delayMs != null && Number.isFinite(input.delayMs) - ? queuedAt + Math.max(0, Math.trunc(input.delayMs)) - : queuedAt); - this.db - .insert(jobs) - .values({ - id, - kind: input.kind, - platform: input.platform ?? null, - accountKey: input.accountKey ?? null, - priority: Math.trunc(input.priority), - status: "queued", - trigger: input.trigger, - queuedAt, - scheduledAt, - startedAt: null, - finishedAt: null, - attempt: 0, - ownerId: null, - leaseExpiresAt: null, - lastProgressAt: null, - checkpointJson: safeStringifyJson(input.checkpoint), - progressJson: safeStringifyJson(input.progress), - errorJson: null, - }) - .run(); - return id; - } - - claimNextJob(input: { ownerId: string; leaseMs: number; kinds?: JobKind[] }): QueuedJob | null { - const timestamp = now(); - const kindPredicate = - input.kinds && input.kinds.length > 0 ? inArray(jobs.kind, input.kinds) : sql`1 = 1`; - const claimablePredicate = and( - kindPredicate, - sql`( - (${jobs.status} IN ('queued', 'retry_wait') AND ${jobs.scheduledAt} <= ${timestamp}) - OR (${jobs.status} = 'running' AND ${jobs.leaseExpiresAt} IS NOT NULL AND ${jobs.leaseExpiresAt} <= ${timestamp}) - )`, - ); - - return this.sqlite.transaction(() => { - const row = this.db - .select({ - id: jobs.id, - kind: jobs.kind, - platform: jobs.platform, - account_key: jobs.accountKey, - priority: jobs.priority, - status: jobs.status, - trigger: jobs.trigger, - queued_at: jobs.queuedAt, - scheduled_at: jobs.scheduledAt, - started_at: jobs.startedAt, - attempt: jobs.attempt, - owner_id: jobs.ownerId, - lease_expires_at: jobs.leaseExpiresAt, - last_progress_at: jobs.lastProgressAt, - checkpoint_json: jobs.checkpointJson, - progress_json: jobs.progressJson, - error_json: jobs.errorJson, - }) - .from(jobs) - .where(claimablePredicate) - .orderBy(asc(jobs.priority), asc(jobs.scheduledAt), asc(jobs.queuedAt)) - .limit(1) - .get() as QueuedJob | undefined; - if (!row) { - return null; - } - - const startedAt = row.started_at ?? timestamp; - const leaseExpiresAt = timestamp + Math.max(1, Math.trunc(input.leaseMs)); - this.db - .update(jobs) - .set({ - status: "running", - ownerId: input.ownerId, - startedAt, - attempt: row.attempt + 1, - leaseExpiresAt, - lastProgressAt: row.last_progress_at ?? timestamp, - errorJson: null, - }) - .where(eq(jobs.id, row.id)) - .run(); - - return { - ...row, - status: "running" as const, - owner_id: input.ownerId, - started_at: startedAt, - attempt: row.attempt + 1, - lease_expires_at: leaseExpiresAt, - last_progress_at: row.last_progress_at ?? timestamp, - error_json: null, - }; - })(); - } - - updateJobProgress( - jobId: string, - input: { - checkpoint?: unknown; - progress?: unknown; - leaseMs?: number; - }, - ): void { - const timestamp = now(); - this.db - .update(jobs) - .set({ - checkpointJson: - input.checkpoint === undefined ? undefined : safeStringifyJson(input.checkpoint), - progressJson: input.progress === undefined ? undefined : safeStringifyJson(input.progress), - lastProgressAt: timestamp, - leaseExpiresAt: - input.leaseMs == null ? undefined : timestamp + Math.max(1, Math.trunc(input.leaseMs)), - }) - .where(eq(jobs.id, jobId)) - .run(); - } - - completeJob(jobId: string, progress?: unknown): void { - const timestamp = now(); - this.db - .update(jobs) - .set({ - status: "completed", - finishedAt: timestamp, - ownerId: null, - leaseExpiresAt: null, - lastProgressAt: timestamp, - progressJson: progress === undefined ? undefined : safeStringifyJson(progress), - }) - .where(eq(jobs.id, jobId)) - .run(); - } - - failJob( - jobId: string, - input: { - error: unknown; - retryAt?: number | null; - }, - ): void { - const timestamp = now(); - const retryAt = input.retryAt ?? null; - this.db - .update(jobs) - .set({ - status: retryAt == null ? "failed" : "retry_wait", - scheduledAt: retryAt ?? undefined, - finishedAt: retryAt == null ? timestamp : null, - ownerId: null, - leaseExpiresAt: null, - lastProgressAt: timestamp, - errorJson: safeStringifyJson({ - message: input.error instanceof Error ? input.error.message : String(input.error), - failedAt: timestamp, - retryAt, - }), - }) - .where(eq(jobs.id, jobId)) - .run(); - } - - enqueueMessageFtsIndex(messageIds: Iterable, reason: string): number { + enqueueMessageFtsIndex(messageIds: Iterable): number { const uniqueMessageIds = [...new Set([...messageIds].filter((id) => id.length > 0))]; if (uniqueMessageIds.length === 0) { return 0; @@ -3545,20 +2286,15 @@ export class CuedDatabase { .insert(messageFtsIndexQueue) .values({ messageId, - reason, status: "queued", - attempt: 0, queuedAt: timestamp, updatedAt: timestamp, - lastError: null, }) .onConflictDoUpdate({ target: messageFtsIndexQueue.messageId, set: { - reason, status: "queued", updatedAt: timestamp, - lastError: null, }, }) .run().changes; @@ -3568,19 +2304,16 @@ export class CuedDatabase { return changed; } - claimMessageFtsIndexBatch(limit: number): MessageFtsIndexQueueRow[] { + private claimMessageFtsIndexBatch(limit: number): MessageFtsIndexQueueRow[] { const normalizedLimit = Math.max(1, Math.trunc(limit)); const timestamp = now(); return this.sqlite.transaction(() => { const rows = this.db .select({ message_id: messageFtsIndexQueue.messageId, - reason: messageFtsIndexQueue.reason, status: messageFtsIndexQueue.status, - attempt: messageFtsIndexQueue.attempt, queued_at: messageFtsIndexQueue.queuedAt, updated_at: messageFtsIndexQueue.updatedAt, - last_error: messageFtsIndexQueue.lastError, }) .from(messageFtsIndexQueue) .where(eq(messageFtsIndexQueue.status, "queued")) @@ -3594,9 +2327,7 @@ export class CuedDatabase { .update(messageFtsIndexQueue) .set({ status: "indexing", - attempt: sql`${messageFtsIndexQueue.attempt} + 1`, updatedAt: timestamp, - lastError: null, }) .where( inArray( @@ -3608,9 +2339,7 @@ export class CuedDatabase { return rows.map((row) => ({ ...row, status: "indexing" as const, - attempt: row.attempt + 1, updated_at: timestamp, - last_error: null, })); })(); } @@ -3621,7 +2350,6 @@ export class CuedDatabase { .set({ status: "queued", updatedAt: now(), - lastError: null, }) .where( and( @@ -3632,7 +2360,7 @@ export class CuedDatabase { .run().changes; } - completeMessageFtsIndex(messageIds: Iterable): number { + private completeMessageFtsIndex(messageIds: Iterable): number { const ids = [...new Set([...messageIds])]; if (ids.length === 0) { return 0; @@ -3643,7 +2371,7 @@ export class CuedDatabase { .run().changes; } - failMessageFtsIndex(messageIds: Iterable, error: unknown): number { + private failMessageFtsIndex(messageIds: Iterable): number { const ids = [...new Set([...messageIds])]; if (ids.length === 0) { return 0; @@ -3653,13 +2381,12 @@ export class CuedDatabase { .set({ status: "failed", updatedAt: now(), - lastError: error instanceof Error ? error.message : String(error), }) .where(inArray(messageFtsIndexQueue.messageId, ids)) .run().changes; } - replaceMessageFtsIndexForIds(messageIds: Iterable): number { + private replaceMessageFtsIndexForIds(messageIds: Iterable): number { const ids = [...new Set([...messageIds].filter((id) => id.length > 0))]; if (ids.length === 0) { return 0; @@ -3715,8 +2442,8 @@ export class CuedDatabase { const indexed = this.replaceMessageFtsIndexForIds(messageIds); this.completeMessageFtsIndex(messageIds); return { claimed: rows.length, indexed, failed: 0 }; - } catch (error) { - this.failMessageFtsIndex(messageIds, error); + } catch { + this.failMessageFtsIndex(messageIds); return { claimed: rows.length, indexed: 0, failed: rows.length }; } } @@ -3766,31 +2493,6 @@ export class CuedDatabase { ); } - hasQueuedOrActiveProjectionRun(): boolean { - return Boolean( - this.db - .select({ id: syncRuns.id }) - .from(syncRuns) - .where( - and( - inArray(syncRuns.runType, ["project", "rebuild"]), - sql`${syncRuns.status} IN ('queued', 'projecting')`, - ), - ) - .limit(1) - .get(), - ); - } - - listCheckpointPlatforms(): string[] { - return this.db - .selectDistinct({ platform: syncCheckpoints.platform }) - .from(syncCheckpoints) - .orderBy(asc(syncCheckpoints.platform)) - .all() - .map((row) => row.platform); - } - listCheckpointTargets(): Array<{ platform: Platform; account_key: string }> { return this.db .select({ @@ -3877,7 +2579,6 @@ export class CuedDatabase { attempt: syncRuns.attempt, owner_id: syncRuns.ownerId, lease_expires_at: syncRuns.leaseExpiresAt, - last_progress_at: syncRuns.lastProgressAt, details_json: syncRuns.detailsJson, }) .from(syncRuns) @@ -3898,7 +2599,6 @@ export class CuedDatabase { attempt: number; owner_id: string | null; lease_expires_at: number | null; - last_progress_at: number | null; details_json: string | null; } | undefined; @@ -3918,8 +2618,6 @@ export class CuedDatabase { attempt: row.attempt + 1, ownerId, leaseExpiresAt, - lastProgressAt: timestamp, - errorCode: null, }) .where(eq(syncRuns.id, row.id)) .run(); @@ -3931,7 +2629,6 @@ export class CuedDatabase { attempt: row.attempt + 1, owner_id: ownerId, lease_expires_at: leaseExpiresAt, - last_progress_at: timestamp, }; }); } @@ -3993,21 +2690,6 @@ export class CuedDatabase { return and(...predicates); } - updateRunStatus(runId: string, status: SyncRunStatus, details?: unknown): void { - const values: { - status: SyncRunStatus; - detailsJson?: string | null; - } = - details === undefined - ? { status } - : { - status, - detailsJson: safeStringifyJson(details), - }; - - this.db.update(syncRuns).set(values).where(eq(syncRuns.id, runId)).run(); - } - updateRunProgress( runId: string, input: { details?: unknown; leaseMs?: number; claim?: SyncRunClaim } = {}, @@ -4016,7 +2698,6 @@ export class CuedDatabase { const result = this.db .update(syncRuns) .set({ - lastProgressAt: timestamp, leaseExpiresAt: input.leaseMs == null ? undefined : timestamp + Math.max(1, Math.trunc(input.leaseMs)), detailsJson: input.details === undefined ? undefined : safeStringifyJson(input.details), @@ -4032,8 +2713,6 @@ export class CuedDatabase { finishedAt: number; ownerId: null; leaseExpiresAt: null; - lastProgressAt: number; - errorCode: null; detailsJson?: string | null; } = details === undefined @@ -4042,16 +2721,12 @@ export class CuedDatabase { finishedAt: now(), ownerId: null, leaseExpiresAt: null, - lastProgressAt: now(), - errorCode: null, } : { status: "completed", finishedAt: now(), ownerId: null, leaseExpiresAt: null, - lastProgressAt: now(), - errorCode: null, detailsJson: safeStringifyJson(details), }; @@ -4100,13 +2775,7 @@ export class CuedDatabase { ); } - failRun( - runId: string, - errorMessage: string, - details?: unknown, - errorCode?: string | null, - claim?: SyncRunClaim, - ): boolean { + failRun(runId: string, errorMessage: string, details?: unknown, claim?: SyncRunClaim): boolean { let updated = false; this.db.transaction((tx) => { const run = tx @@ -4122,8 +2791,6 @@ export class CuedDatabase { finishedAt: number; ownerId: null; leaseExpiresAt: null; - lastProgressAt: number; - errorCode: string | null; detailsJson?: string | null; } = details === undefined @@ -4132,16 +2799,12 @@ export class CuedDatabase { finishedAt: now(), ownerId: null, leaseExpiresAt: null, - lastProgressAt: now(), - errorCode: errorCode ?? null, } : { status: "failed", finishedAt: now(), ownerId: null, leaseExpiresAt: null, - lastProgressAt: now(), - errorCode: errorCode ?? null, detailsJson: safeStringifyJson(details), }; @@ -4158,12 +2821,9 @@ export class CuedDatabase { tx.insert(syncRunErrors) .values({ id: randomUUID(), - syncRunId: runId, platform: run?.platform ?? null, accountKey: run?.accountKey ?? null, - errorCode: errorCode ?? null, errorMessage, - detailsJson: safeStringifyJson(details), createdAt: now(), }) .run(); @@ -4175,18 +2835,14 @@ export class CuedDatabase { platform: Platform, accountKey: string, ): { - sync_run_id: string; error_message: string; created_at: number; - details_json: string | null; } | null { return ( (this.db .select({ - sync_run_id: syncRunErrors.syncRunId, error_message: syncRunErrors.errorMessage, created_at: syncRunErrors.createdAt, - details_json: syncRunErrors.detailsJson, }) .from(syncRunErrors) .where(and(eq(syncRunErrors.platform, platform), eq(syncRunErrors.accountKey, accountKey))) @@ -4194,58 +2850,14 @@ export class CuedDatabase { .limit(1) .get() as | { - sync_run_id: string; error_message: string; created_at: number; - details_json: string | null; } | undefined) ?? null ); } - upsertSourceAccount(input: { - platform: Platform; - accountKey: string; - displayName?: string | null; - status?: string; - metadata?: unknown; - }): void { - const timestamp = now(); - const values = { - id: `${input.platform}:${input.accountKey}`, - platform: input.platform, - accountKey: input.accountKey, - displayName: input.displayName ?? null, - status: input.status ?? "active", - metadataJson: safeStringifyJson(input.metadata), - createdAt: timestamp, - updatedAt: timestamp, - }; - - this.db - .insert(sourceAccounts) - .values(values) - .onConflictDoUpdate({ - target: [sourceAccounts.platform, sourceAccounts.accountKey], - set: { - displayName: values.displayName, - status: values.status, - metadataJson: values.metadataJson, - updatedAt: values.updatedAt, - }, - }) - .run(); - } - - upsertSourceAccounts( - inputs: Array<{ - platform: Platform; - accountKey: string; - displayName?: string | null; - status?: string; - metadata?: unknown; - }>, - ): void { + upsertSourceAccounts(inputs: SourceAccountInput[]): void { if (inputs.length === 0) { return; } @@ -4259,9 +2871,7 @@ export class CuedDatabase { id: `${input.platform}:${input.accountKey}`, platform: input.platform, accountKey: input.accountKey, - displayName: input.displayName ?? null, - status: input.status ?? "active", - metadataJson: safeStringifyJson(input.metadata), + displayName: input.displayName, createdAt: timestamp, updatedAt: timestamp, })), @@ -4270,8 +2880,6 @@ export class CuedDatabase { target: [sourceAccounts.platform, sourceAccounts.accountKey], set: { displayName: sql`excluded.display_name`, - status: sql`excluded.status`, - metadataJson: sql`excluded.metadata_json`, updatedAt: timestamp, }, }) @@ -4297,9 +2905,7 @@ export class CuedDatabase { sourceCursorJson: safeStringifyJson(input.sourceCursor), rawIngestWatermark: input.rawIngestWatermark ?? 0, syncMode: input.syncMode, - lastFullSyncAt: input.lastSuccessAt ?? null, lastSuccessAt: input.lastSuccessAt ?? null, - lastErrorAt: null, lastErrorSummary: input.lastErrorSummary ?? null, createdAt: timestamp, updatedAt: timestamp, @@ -4314,10 +2920,6 @@ export class CuedDatabase { sourceCursorJson: values.sourceCursorJson, rawIngestWatermark: values.rawIngestWatermark, syncMode: values.syncMode, - lastFullSyncAt: - input.syncMode === "full" - ? values.lastSuccessAt - : sql`${syncCheckpoints.lastFullSyncAt}`, lastSuccessAt: values.lastSuccessAt, lastErrorSummary: values.lastErrorSummary, updatedAt: values.updatedAt, @@ -4330,7 +2932,6 @@ export class CuedDatabase { this.db .update(syncCheckpoints) .set({ - lastErrorAt: now(), lastErrorSummary: errorSummary, updatedAt: now(), }) @@ -4340,15 +2941,6 @@ export class CuedDatabase { .run(); } - insertRawEvent(event: RawEventInput): boolean { - const result = this.db - .insert(rawEvents) - .values(buildRawEventValues(event)) - .onConflictDoNothing() - .run(); - return Number(result.changes) > 0; - } - insertRawEvents(events: RawEventInput[]): { insertedCount: number; insertedEvents: RawEventInput[]; @@ -4445,35 +3037,20 @@ export class CuedDatabase { quarantineRawEventProjectionFailure( event: { rowid: number; - id: string; - platform: Platform; - account_key: string; - entity_kind: RawEventEntityKind; - event_kind: string; - normalized_schema: string | null; - source_version: string | null; }, error: unknown, ): void { + const errorMessage = error instanceof Error ? error.message : String(error); this.db .insert(rawEventProjectionFailures) .values({ rawEventRowId: event.rowid, - rawEventId: event.id, - platform: event.platform, - accountKey: event.account_key, - entityKind: event.entity_kind, - eventKind: event.event_kind, - normalizedSchema: event.normalized_schema, - sourceVersion: event.source_version, - errorMessage: error instanceof Error ? error.message : String(error), - failedAt: now(), + errorMessage, }) .onConflictDoUpdate({ target: rawEventProjectionFailures.rawEventRowId, set: { - errorMessage: error instanceof Error ? error.message : String(error), - failedAt: now(), + errorMessage, }, }) .run(); @@ -4540,44 +3117,6 @@ export class CuedDatabase { return [...anchors.values()]; } - listRawEvents(): Array<{ - id: string; - platform: Platform; - account_key: string; - entity_kind: RawEventEntityKind; - event_kind: string; - normalized_schema: string | null; - provenance_json: string | null; - observed_at: number; - payload_json: string; - }> { - return this.db - .select({ - id: rawEvents.id, - platform: rawEvents.platform, - account_key: rawEvents.accountKey, - entity_kind: rawEvents.entityKind, - event_kind: rawEvents.eventKind, - normalized_schema: rawEvents.normalizedSchema, - provenance_json: rawEvents.provenanceJson, - observed_at: rawEvents.observedAt, - payload_json: rawEvents.payloadJson, - }) - .from(rawEvents) - .orderBy(asc(rawEvents.observedAt), asc(rawEvents.id)) - .all() as Array<{ - id: string; - platform: Platform; - account_key: string; - entity_kind: RawEventEntityKind; - event_kind: string; - normalized_schema: string | null; - provenance_json: string | null; - observed_at: number; - payload_json: string; - }>; - } - listRawEventsAfter( rowId: number, limit?: number, @@ -4589,8 +3128,6 @@ export class CuedDatabase { entity_kind: RawEventEntityKind; event_kind: string; normalized_schema: string | null; - provenance_json: string | null; - source_version: string | null; observed_at: number; payload_json: string; }> { @@ -4605,8 +3142,6 @@ export class CuedDatabase { entity_kind, event_kind, normalized_schema, - provenance_json, - source_version, observed_at, payload_json FROM raw_events @@ -4623,44 +3158,11 @@ export class CuedDatabase { entity_kind: RawEventEntityKind; event_kind: string; normalized_schema: string | null; - provenance_json: string | null; - source_version: string | null; observed_at: number; payload_json: string; }>; } - getRawEventRowIdRange( - platform: Platform, - accountKey?: string, - ): { minRowId: number; maxRowId: number } | null { - const clauses = ["platform = ?"]; - const params: Array = [platform]; - if (accountKey) { - clauses.push("account_key = ?"); - params.push(accountKey); - } - - const row = this.sqlite - .prepare( - ` - SELECT MIN(rowid) AS min_rowid, MAX(rowid) AS max_rowid - FROM raw_events - WHERE ${clauses.join(" AND ")} - `, - ) - .get(...params) as { min_rowid: number | null; max_rowid: number | null } | undefined; - - if (!row?.min_rowid || !row?.max_rowid) { - return null; - } - - return { - minRowId: row.min_rowid, - maxRowId: row.max_rowid, - }; - } - listRawEventsInRange( startRowId: number, endRowId: number, @@ -4673,8 +3175,6 @@ export class CuedDatabase { entity_kind: RawEventEntityKind; event_kind: string; normalized_schema: string | null; - provenance_json: string | null; - source_version: string | null; observed_at: number; payload_json: string; }> { @@ -4693,8 +3193,6 @@ export class CuedDatabase { entity_kind, event_kind, normalized_schema, - provenance_json, - source_version, observed_at, payload_json FROM raw_events @@ -4712,8 +3210,6 @@ export class CuedDatabase { entity_kind: RawEventEntityKind; event_kind: string; normalized_schema: string | null; - provenance_json: string | null; - source_version: string | null; observed_at: number; payload_json: string; }>; @@ -4735,7 +3231,6 @@ export class CuedDatabase { attempt: syncRuns.attempt, owner_id: syncRuns.ownerId, lease_expires_at: syncRuns.leaseExpiresAt, - last_progress_at: syncRuns.lastProgressAt, details_json: syncRuns.detailsJson, }) .from(syncRuns) @@ -4862,54 +3357,6 @@ export class CuedDatabase { }>; } - listMessageMap(): Array<{ - platform: Platform; - account_key: string; - platform_message_id: string; - message_id: string; - }> { - return this.db - .select({ - platform: messages.platform, - account_key: messages.accountKey, - platform_message_id: messages.platformMessageId, - message_id: messages.id, - }) - .from(messages) - .all() as Array<{ - platform: Platform; - account_key: string; - platform_message_id: string; - message_id: string; - }>; - } - - clearProjectedState(): void { - const cacheEntries = this.listReadyAttachmentCacheEntries(); - for (const entry of cacheEntries) { - if (entry.cache_path) { - rmSync(entry.cache_path, { force: true }); - } - } - - this.db.transaction((tx) => { - tx.run(sql.raw("DELETE FROM messages_fts")); - tx.run(sql.raw("DELETE FROM message_fts_index_queue")); - tx.run(sql.raw("DELETE FROM attachment_content_fts")); - tx.delete(attachmentContent).run(); - tx.delete(attachmentCache).run(); - tx.delete(timelineEvents).run(); - tx.delete(messageAttachments).run(); - tx.delete(messageReactions).run(); - tx.delete(conversationParticipants).run(); - tx.delete(messages).run(); - tx.delete(conversations).run(); - tx.delete(contactHandles).run(); - tx.delete(contactSources).run(); - tx.delete(contacts).run(); - }); - } - private countRows(table: SQLiteTable): number { const row = this.db.select({ count: sql`count(*)` }).from(table).get(); @@ -4920,10 +3367,8 @@ export class CuedDatabase { export function openCuedDatabase(dbPath?: string): CuedDatabase { const db = new CuedDatabase(dbPath); db.migrate(); - db.recordAppMetadata({ - version: getCurrentAppVersion(), - releaseChannel: getCurrentReleaseChannel(), - }); + db.setAppSetting(APP_SETTING_KEYS.installedAppVersion, getCurrentAppVersion()); + db.setAppSetting(APP_SETTING_KEYS.releaseChannel, getCurrentReleaseChannel()); return db; } diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 0a14ba55..b4022f42 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -32,15 +32,60 @@ function addColumnIfMissing(db: MigrationDatabase, tableName: string, definition db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${definition}`); } -function buildSyncScopeId( - platform: string, - accountKey: string, - scopeKind: string, - scopeKey: string, -): string { - return `scope:${Buffer.from(JSON.stringify([platform, accountKey, scopeKind, scopeKey])).toString( - "base64url", - )}`; +function dropColumnIfExists(db: MigrationDatabase, tableName: string, columnName: string): void { + if (!columnExists(db, tableName, columnName)) { + return; + } + db.exec(`ALTER TABLE ${tableName} DROP COLUMN ${columnName}`); +} + +function dropMessageReactionSourceKeyIfNeeded(db: MigrationDatabase): void { + if (!columnExists(db, "message_reactions", "source_reaction_key")) { + return; + } + + db.exec(` + CREATE TABLE IF NOT EXISTS message_reactions_reduced ( + id TEXT PRIMARY KEY, + message_id TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + platform TEXT NOT NULL, + account_key TEXT NOT NULL, + reactor_source_key TEXT, + emoji TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + INSERT OR REPLACE INTO message_reactions_reduced ( + id, + message_id, + platform, + account_key, + reactor_source_key, + emoji, + is_active, + created_at, + updated_at + ) + SELECT + id, + message_id, + platform, + account_key, + reactor_source_key, + emoji, + is_active, + created_at, + updated_at + FROM message_reactions; + + DROP TABLE message_reactions; + ALTER TABLE message_reactions_reduced RENAME TO message_reactions; + + CREATE INDEX IF NOT EXISTS idx_message_reactions_message + ON message_reactions(message_id, is_active); + `); } function buildSyncProofId( @@ -62,14 +107,95 @@ function stringifyJson(value: unknown): string | null { return JSON.stringify(value); } +function collapseSyncProofScopesIfNeeded(db: MigrationDatabase): void { + if (!tableExists(db, "sync_proofs")) { + return; + } + if (columnExists(db, "sync_proofs", "scope_kind")) { + db.exec(` + DROP INDEX IF EXISTS idx_sync_scopes_lookup; + DROP INDEX IF EXISTS idx_sync_scopes_parent; + DROP TABLE IF EXISTS sync_scopes; + `); + return; + } + if (!columnExists(db, "sync_proofs", "scope_id")) { + return; + } + + db.exec(` + DROP INDEX IF EXISTS idx_sync_proofs_lookup; + DROP INDEX IF EXISTS idx_sync_proofs_status; + + CREATE TABLE IF NOT EXISTS sync_proofs_reduced ( + id TEXT PRIMARY KEY, + platform TEXT NOT NULL, + account_key TEXT NOT NULL, + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + proof_kind TEXT NOT NULL, + status TEXT NOT NULL, + last_observed_at INTEGER NOT NULL, + resume_cursor_json TEXT, + coverage_json TEXT, + stats_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(platform, account_key, scope_kind, scope_key, proof_kind) + ); + + INSERT OR REPLACE INTO sync_proofs_reduced ( + id, + platform, + account_key, + scope_kind, + scope_key, + proof_kind, + status, + last_observed_at, + resume_cursor_json, + coverage_json, + stats_json, + created_at, + updated_at + ) + SELECT + sync_proofs.id, + sync_proofs.platform, + sync_proofs.account_key, + COALESCE(sync_scopes.scope_kind, 'account'), + COALESCE(sync_scopes.scope_key, sync_proofs.account_key), + sync_proofs.proof_kind, + sync_proofs.status, + sync_proofs.last_observed_at, + sync_proofs.resume_cursor_json, + sync_proofs.coverage_json, + sync_proofs.stats_json, + sync_proofs.created_at, + sync_proofs.updated_at + FROM sync_proofs + LEFT JOIN sync_scopes ON sync_scopes.id = sync_proofs.scope_id; + + DROP TABLE sync_proofs; + ALTER TABLE sync_proofs_reduced RENAME TO sync_proofs; + + CREATE INDEX IF NOT EXISTS idx_sync_proofs_lookup + ON sync_proofs(platform, account_key, scope_kind, scope_key, proof_kind); + + CREATE INDEX IF NOT EXISTS idx_sync_proofs_status + ON sync_proofs(platform, account_key, status, updated_at); + + DROP INDEX IF EXISTS idx_sync_scopes_lookup; + DROP INDEX IF EXISTS idx_sync_scopes_parent; + DROP TABLE IF EXISTS sync_scopes; + `); +} + function migrateSlackBackfillProofsToGeneric(db: MigrationDatabase): void { - if ( - !tableExists(db, "slack_backfill_proofs") || - !tableExists(db, "sync_scopes") || - !tableExists(db, "sync_proofs") - ) { + if (!tableExists(db, "slack_backfill_proofs") || !tableExists(db, "sync_proofs")) { return; } + collapseSyncProofScopesIfNeeded(db); const rows = db.prepare("SELECT * FROM slack_backfill_proofs").all() as Array<{ account_key: string; @@ -100,54 +226,25 @@ function migrateSlackBackfillProofsToGeneric(db: MigrationDatabase): void { return; } - const upsertScope = db.prepare(` - INSERT INTO sync_scopes ( - id, - platform, - account_key, - scope_kind, - scope_key, - parent_scope_id, - display_name, - metadata_json, - first_discovered_at, - last_observed_at, - created_at, - updated_at - ) VALUES (?, 'slack', ?, 'conversation', ?, NULL, ?, ?, ?, ?, ?, ?) - ON CONFLICT(platform, account_key, scope_kind, scope_key) DO UPDATE SET - display_name = excluded.display_name, - metadata_json = excluded.metadata_json, - first_discovered_at = MIN(sync_scopes.first_discovered_at, excluded.first_discovered_at), - last_observed_at = MAX(sync_scopes.last_observed_at, excluded.last_observed_at), - updated_at = MAX(sync_scopes.updated_at, excluded.updated_at) - `); const upsertProof = db.prepare(` INSERT INTO sync_proofs ( id, platform, account_key, - scope_id, + scope_kind, + scope_key, proof_kind, status, - sync_mode, - run_started_at, last_observed_at, - completed_at, - fresh_until, resume_cursor_json, coverage_json, stats_json, - error_json, created_at, updated_at - ) VALUES (?, 'slack', ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, NULL, ?, ?) - ON CONFLICT(platform, account_key, scope_id, proof_kind) DO UPDATE SET + ) VALUES (?, 'slack', ?, 'conversation', ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(platform, account_key, scope_kind, scope_key, proof_kind) DO UPDATE SET status = excluded.status, - sync_mode = excluded.sync_mode, - run_started_at = excluded.run_started_at, last_observed_at = MAX(sync_proofs.last_observed_at, excluded.last_observed_at), - completed_at = COALESCE(sync_proofs.completed_at, excluded.completed_at), resume_cursor_json = excluded.resume_cursor_json, coverage_json = excluded.coverage_json, stats_json = excluded.stats_json, @@ -155,35 +252,17 @@ function migrateSlackBackfillProofsToGeneric(db: MigrationDatabase): void { `); for (const row of rows) { - const scopeId = buildSyncScopeId("slack", row.account_key, "conversation", row.conversation_id); const firstDiscoveredAt = row.first_discovered_at ?? row.last_observed_at; const updatedAt = row.updated_at ?? row.last_observed_at; - upsertScope.run( - scopeId, - row.account_key, - row.conversation_id, - row.conversation_name, - stringifyJson({ - teamId: row.team_id, - conversationFamily: row.conversation_family, - }), - firstDiscoveredAt, - row.last_observed_at, - firstDiscoveredAt, - updatedAt, - ); const messagesComplete = row.history_complete === 1 || row.conversation_phase !== "history"; upsertProof.run( buildSyncProofId("slack", row.account_key, "conversation", row.conversation_id, "messages"), row.account_key, - scopeId, + row.conversation_id, "messages", messagesComplete ? "complete" : "running", - row.sync_mode, - row.scan_started_at, row.last_observed_at, - messagesComplete ? (row.history_complete_at ?? row.last_observed_at) : null, messagesComplete ? null : stringifyJson({ @@ -211,13 +290,10 @@ function migrateSlackBackfillProofsToGeneric(db: MigrationDatabase): void { upsertProof.run( buildSyncProofId("slack", row.account_key, "conversation", row.conversation_id, "replies"), row.account_key, - scopeId, + row.conversation_id, "replies", repliesComplete ? "complete" : "running", - row.sync_mode, - row.scan_started_at, row.last_observed_at, - repliesComplete ? (row.replies_complete_at ?? row.last_observed_at) : null, repliesComplete ? null : stringifyJson({ @@ -368,9 +444,6 @@ function repairLegacyMessageAttachmentsIfNeeded(db: MigrationDatabase): void { addColumnIfMissing(db, "message_attachments", "access_kind TEXT"); addColumnIfMissing(db, "message_attachments", "access_ref_json TEXT"); - addColumnIfMissing(db, "message_attachments", "preview_ref_json TEXT"); - addColumnIfMissing(db, "message_attachments", "availability_status TEXT"); - addColumnIfMissing(db, "message_attachments", "provider_metadata_json TEXT"); db.exec(` UPDATE message_attachments @@ -384,50 +457,13 @@ function repairLegacyMessageAttachmentsIfNeeded(db: MigrationDatabase): void { WHEN local_path IS NOT NULL AND trim(local_path) <> '' THEN json_object('path', local_path) WHEN remote_url IS NOT NULL AND trim(remote_url) <> '' THEN json_object('url', remote_url) ELSE NULL - END, - availability_status = CASE - WHEN local_path IS NOT NULL AND trim(local_path) <> '' THEN 'available' - WHEN remote_url IS NOT NULL AND trim(remote_url) <> '' THEN 'available' - ELSE 'metadata_only' - END, - provider_metadata_json = COALESCE(provider_metadata_json, metadata_json) + END WHERE access_kind IS NULL; `); } function ensureLegacySupportTables(db: MigrationDatabase): void { db.exec(` - CREATE TABLE IF NOT EXISTS slack_backfill_proofs ( - id TEXT PRIMARY KEY, - account_key TEXT NOT NULL, - team_id TEXT NOT NULL, - conversation_id TEXT NOT NULL, - conversation_name TEXT, - conversation_family TEXT NOT NULL, - sync_mode TEXT NOT NULL, - scan_started_at INTEGER NOT NULL, - known_conversation_count INTEGER NOT NULL, - conversation_phase TEXT NOT NULL, - history_complete INTEGER NOT NULL DEFAULT 0, - history_cursor TEXT, - thread_root_count INTEGER NOT NULL DEFAULT 0, - completed_thread_count INTEGER NOT NULL DEFAULT 0, - pending_thread_count INTEGER NOT NULL DEFAULT 0, - active_thread_ts TEXT, - replies_cursor TEXT, - oldest_message_ts TEXT, - newest_message_ts TEXT, - first_discovered_at INTEGER NOT NULL, - history_complete_at INTEGER, - replies_complete_at INTEGER, - last_observed_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(account_key, conversation_id) - ); - - CREATE INDEX IF NOT EXISTS idx_slack_backfill_proofs_account_phase - ON slack_backfill_proofs(account_key, conversation_phase, updated_at); - CREATE TABLE IF NOT EXISTS attachment_cache ( id TEXT PRIMARY KEY, attachment_id TEXT NOT NULL REFERENCES message_attachments(id) ON DELETE CASCADE, @@ -554,11 +590,7 @@ function rebuildMessagesFtsArtifacts(db: MigrationDatabase): void { AFTER INSERT ON message_attachments BEGIN UPDATE messages - SET - attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = NEW.message_id - ), - updated_at = MAX(updated_at, NEW.updated_at) + SET updated_at = MAX(updated_at, NEW.updated_at) WHERE id = NEW.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = NEW.message_id); @@ -572,11 +604,7 @@ function rebuildMessagesFtsArtifacts(db: MigrationDatabase): void { AFTER UPDATE ON message_attachments BEGIN UPDATE messages - SET - attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = NEW.message_id - ), - updated_at = MAX(updated_at, NEW.updated_at) + SET updated_at = MAX(updated_at, NEW.updated_at) WHERE id = NEW.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = NEW.message_id); @@ -589,11 +617,6 @@ function rebuildMessagesFtsArtifacts(db: MigrationDatabase): void { CREATE TRIGGER trg_message_attachments_deleted AFTER DELETE ON message_attachments BEGIN - UPDATE messages - SET attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = OLD.message_id - ) - WHERE id = OLD.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = OLD.message_id); INSERT INTO messages_fts (rowid, message_id, sender_name, conversation_name, participant_names, attachment_text, content) @@ -604,40 +627,6 @@ function rebuildMessagesFtsArtifacts(db: MigrationDatabase): void { `); } - if (tableExists(db, "message_reactions")) { - db.exec(` - CREATE TRIGGER trg_message_reactions_inserted - AFTER INSERT ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = NEW.message_id AND is_active = 1 - ) - WHERE id = NEW.message_id; - END; - - CREATE TRIGGER trg_message_reactions_updated - AFTER UPDATE ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = NEW.message_id AND is_active = 1 - ) - WHERE id = NEW.message_id; - END; - - CREATE TRIGGER trg_message_reactions_deleted - AFTER DELETE ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = OLD.message_id AND is_active = 1 - ) - WHERE id = OLD.message_id; - END; - `); - } - if ( tableExists(db, "contacts") && tableExists(db, "conversation_participants") && @@ -660,10 +649,6 @@ function rebuildMessagesFtsArtifacts(db: MigrationDatabase): void { SET actor_name = NEW.name WHERE actor_contact_id = NEW.id; - UPDATE message_reactions - SET reactor_name = NEW.name - WHERE reactor_contact_id = NEW.id; - UPDATE conversations SET participant_names = ( SELECT GROUP_CONCAT(cp.participant_name, ' | ') @@ -865,8 +850,6 @@ export const MIGRATIONS: Migration[] = [ platform TEXT NOT NULL, account_key TEXT NOT NULL, display_name TEXT, - status TEXT NOT NULL DEFAULT 'active', - metadata_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key) @@ -879,50 +862,28 @@ export const MIGRATIONS: Migration[] = [ source_cursor_json TEXT, raw_ingest_watermark INTEGER NOT NULL DEFAULT 0, sync_mode TEXT NOT NULL DEFAULT 'full', - last_full_sync_at INTEGER, last_success_at INTEGER, - last_error_at INTEGER, last_error_summary TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key) ); - CREATE TABLE IF NOT EXISTS sync_scopes ( + CREATE TABLE IF NOT EXISTS sync_proofs ( id TEXT PRIMARY KEY, platform TEXT NOT NULL, account_key TEXT NOT NULL, scope_kind TEXT NOT NULL, scope_key TEXT NOT NULL, - parent_scope_id TEXT REFERENCES sync_scopes(id) ON DELETE CASCADE, - display_name TEXT, - metadata_json TEXT, - first_discovered_at INTEGER NOT NULL, - last_observed_at INTEGER NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(platform, account_key, scope_kind, scope_key) - ); - - CREATE TABLE IF NOT EXISTS sync_proofs ( - id TEXT PRIMARY KEY, - platform TEXT NOT NULL, - account_key TEXT NOT NULL, - scope_id TEXT NOT NULL REFERENCES sync_scopes(id) ON DELETE CASCADE, proof_kind TEXT NOT NULL, status TEXT NOT NULL, - sync_mode TEXT, - run_started_at INTEGER, last_observed_at INTEGER NOT NULL, - completed_at INTEGER, - fresh_until INTEGER, resume_cursor_json TEXT, coverage_json TEXT, stats_json TEXT, - error_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, - UNIQUE(platform, account_key, scope_id, proof_kind) + UNIQUE(platform, account_key, scope_kind, scope_key, proof_kind) ); CREATE TABLE IF NOT EXISTS sync_runs ( @@ -939,79 +900,22 @@ export const MIGRATIONS: Migration[] = [ attempt INTEGER NOT NULL DEFAULT 0, owner_id TEXT, lease_expires_at INTEGER, - last_progress_at INTEGER, - error_code TEXT, details_json TEXT ); - CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - platform TEXT, - account_key TEXT, - priority INTEGER NOT NULL, - status TEXT NOT NULL, - trigger TEXT NOT NULL, - queued_at INTEGER NOT NULL, - scheduled_at INTEGER NOT NULL, - started_at INTEGER, - finished_at INTEGER, - attempt INTEGER NOT NULL DEFAULT 0, - owner_id TEXT, - lease_expires_at INTEGER, - last_progress_at INTEGER, - checkpoint_json TEXT, - progress_json TEXT, - error_json TEXT - ); - CREATE TABLE IF NOT EXISTS sync_run_errors ( id TEXT PRIMARY KEY, - sync_run_id TEXT NOT NULL REFERENCES sync_runs(id) ON DELETE CASCADE, platform TEXT, account_key TEXT, - error_code TEXT, error_message TEXT NOT NULL, - details_json TEXT, created_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS message_fts_index_queue ( message_id TEXT PRIMARY KEY, - reason TEXT NOT NULL, status TEXT NOT NULL, - attempt INTEGER NOT NULL DEFAULT 0, queued_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - last_error TEXT - ); - - CREATE TABLE IF NOT EXISTS slack_backfill_proofs ( - id TEXT PRIMARY KEY, - account_key TEXT NOT NULL, - team_id TEXT NOT NULL, - conversation_id TEXT NOT NULL, - conversation_name TEXT, - conversation_family TEXT NOT NULL, - sync_mode TEXT NOT NULL, - scan_started_at INTEGER NOT NULL, - known_conversation_count INTEGER NOT NULL, - conversation_phase TEXT NOT NULL, - history_complete INTEGER NOT NULL DEFAULT 0, - history_cursor TEXT, - thread_root_count INTEGER NOT NULL DEFAULT 0, - completed_thread_count INTEGER NOT NULL DEFAULT 0, - pending_thread_count INTEGER NOT NULL DEFAULT 0, - active_thread_ts TEXT, - replies_cursor TEXT, - oldest_message_ts TEXT, - newest_message_ts TEXT, - first_discovered_at INTEGER NOT NULL, - history_complete_at INTEGER, - replies_complete_at INTEGER, - last_observed_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(account_key, conversation_id) + updated_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS daemon_state ( @@ -1033,14 +937,12 @@ export const MIGRATIONS: Migration[] = [ CREATE TABLE IF NOT EXISTS projection_state ( singleton_key TEXT PRIMARY KEY CHECK (singleton_key = 'global'), projection_watermark INTEGER NOT NULL DEFAULT 0, - last_projected_at INTEGER, - last_rebuild_at INTEGER, updated_at INTEGER NOT NULL ); INSERT OR IGNORE INTO projection_state ( - singleton_key, projection_watermark, last_projected_at, last_rebuild_at, updated_at - ) VALUES ('global', 0, NULL, NULL, strftime('%s','now') * 1000); + singleton_key, projection_watermark, updated_at + ) VALUES ('global', 0, strftime('%s','now') * 1000); CREATE TABLE IF NOT EXISTS raw_events ( id TEXT PRIMARY KEY, @@ -1048,17 +950,13 @@ export const MIGRATIONS: Migration[] = [ account_key TEXT NOT NULL, entity_kind TEXT NOT NULL, event_kind TEXT NOT NULL, - external_event_id TEXT, external_entity_id TEXT, conversation_external_id TEXT, occurred_at INTEGER, observed_at INTEGER NOT NULL, - cursor_json TEXT, dedupe_key TEXT NOT NULL, payload_json TEXT NOT NULL, normalized_schema TEXT, - provenance_json TEXT, - source_version TEXT, UNIQUE(platform, account_key, dedupe_key) ); @@ -1079,8 +977,6 @@ export const MIGRATIONS: Migration[] = [ type TEXT NOT NULL, value TEXT NOT NULL, normalized_value TEXT NOT NULL, - platform TEXT, - account_key TEXT, is_deterministic INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL @@ -1092,10 +988,6 @@ export const MIGRATIONS: Migration[] = [ platform TEXT NOT NULL, account_key TEXT NOT NULL, source_entity_key TEXT NOT NULL, - profile_url TEXT, - metadata_json TEXT, - first_seen_at INTEGER NOT NULL, - last_seen_at INTEGER NOT NULL, UNIQUE(platform, account_key, source_entity_key) ); @@ -1119,18 +1011,10 @@ export const MIGRATIONS: Migration[] = [ platform TEXT NOT NULL, account_key TEXT NOT NULL, source_conversation_key TEXT NOT NULL, - native_conversation_key TEXT, type TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 1, - removal_reason TEXT, - service TEXT, name TEXT, - topic TEXT, participant_names TEXT, - last_message_id TEXT, - last_message_at INTEGER, - last_message_preview TEXT, - unread_count INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key, source_conversation_key) @@ -1141,11 +1025,8 @@ export const MIGRATIONS: Migration[] = [ contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE, source_participant_key TEXT, participant_name TEXT, - role TEXT, is_self INTEGER NOT NULL DEFAULT 0, is_active INTEGER NOT NULL DEFAULT 1, - joined_at INTEGER, - left_at INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (conversation_id, contact_id, source_participant_key) ); @@ -1161,19 +1042,12 @@ export const MIGRATIONS: Migration[] = [ sender_name TEXT, conversation_name TEXT, sent_at INTEGER NOT NULL, - service TEXT, status TEXT, is_from_me INTEGER NOT NULL DEFAULT 0, content TEXT, delivered_at INTEGER, read_at INTEGER, - edited_at INTEGER, - deleted_at INTEGER, - reply_to_message_id TEXT REFERENCES messages(id) ON DELETE SET NULL, is_deleted INTEGER NOT NULL DEFAULT 0, - is_edited INTEGER NOT NULL DEFAULT 0, - attachment_count INTEGER NOT NULL DEFAULT 0, - reaction_count INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key, platform_message_id) @@ -1195,10 +1069,6 @@ export const MIGRATIONS: Migration[] = [ text_content TEXT, access_kind TEXT, access_ref_json TEXT, - preview_ref_json TEXT, - availability_status TEXT, - provider_metadata_json TEXT, - metadata_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key, source_attachment_key) @@ -1209,16 +1079,11 @@ export const MIGRATIONS: Migration[] = [ message_id TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE, platform TEXT NOT NULL, account_key TEXT NOT NULL, - source_reaction_key TEXT NOT NULL, - reactor_contact_id TEXT REFERENCES contacts(id) ON DELETE SET NULL, reactor_source_key TEXT, - reactor_name TEXT, emoji TEXT NOT NULL, - reaction_type TEXT, is_active INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(platform, account_key, source_reaction_key) + updated_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS attachment_cache ( @@ -1268,13 +1133,6 @@ export const MIGRATIONS: Migration[] = [ system_kind TEXT, call_provider TEXT, call_direction TEXT, - call_status TEXT, - call_medium TEXT, - call_started_at INTEGER, - call_duration_seconds INTEGER, - call_ended_at INTEGER, - call_disconnected_cause TEXT, - metadata_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, UNIQUE(platform, account_key, source_event_key) @@ -1318,24 +1176,6 @@ export const MIGRATIONS: Migration[] = [ updated_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS outbound_messages ( - id TEXT PRIMARY KEY, - platform TEXT NOT NULL, - account_key TEXT NOT NULL, - target TEXT NOT NULL, - thread_id TEXT, - text TEXT NOT NULL, - status TEXT NOT NULL, - attempt_count INTEGER NOT NULL DEFAULT 0, - scheduled_for INTEGER NOT NULL, - started_at INTEGER, - finished_at INTEGER, - last_error TEXT, - metadata_json TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( message_id UNINDEXED, sender_name, @@ -1358,14 +1198,8 @@ export const MIGRATIONS: Migration[] = [ CREATE INDEX IF NOT EXISTS idx_sync_checkpoints_lookup ON sync_checkpoints(platform, account_key); - CREATE INDEX IF NOT EXISTS idx_sync_scopes_lookup - ON sync_scopes(platform, account_key, scope_kind, scope_key); - - CREATE INDEX IF NOT EXISTS idx_sync_scopes_parent - ON sync_scopes(parent_scope_id, updated_at); - CREATE INDEX IF NOT EXISTS idx_sync_proofs_lookup - ON sync_proofs(platform, account_key, scope_id, proof_kind); + ON sync_proofs(platform, account_key, scope_kind, scope_key, proof_kind); CREATE INDEX IF NOT EXISTS idx_sync_proofs_status ON sync_proofs(platform, account_key, status, updated_at); @@ -1376,15 +1210,6 @@ export const MIGRATIONS: Migration[] = [ CREATE INDEX IF NOT EXISTS idx_sync_runs_platform_account_status_queue ON sync_runs(platform, account_key, status, scheduled_at, queued_at); - CREATE INDEX IF NOT EXISTS idx_jobs_claim - ON jobs(status, scheduled_at, priority, queued_at); - - CREATE INDEX IF NOT EXISTS idx_jobs_owner_lease - ON jobs(owner_id, lease_expires_at); - - CREATE INDEX IF NOT EXISTS idx_jobs_platform_account_status - ON jobs(platform, account_key, status, scheduled_at); - CREATE INDEX IF NOT EXISTS idx_message_fts_index_queue_status ON message_fts_index_queue(status, queued_at); @@ -1395,7 +1220,7 @@ export const MIGRATIONS: Migration[] = [ ON raw_events(platform, entity_kind, event_kind, normalized_schema); CREATE INDEX IF NOT EXISTS idx_contact_handles_lookup - ON contact_handles(type, normalized_value, account_key); + ON contact_handles(type, normalized_value); CREATE INDEX IF NOT EXISTS idx_contact_sources_contact ON contact_sources(contact_id, platform, account_key); @@ -1422,9 +1247,6 @@ export const MIGRATIONS: Migration[] = [ CREATE INDEX IF NOT EXISTS idx_timeline_events_actor_contact ON timeline_events(actor_contact_id); - CREATE INDEX IF NOT EXISTS idx_message_reactions_reactor_contact - ON message_reactions(reactor_contact_id); - CREATE INDEX IF NOT EXISTS idx_conversations_lookup ON conversations(platform, account_key, source_conversation_key); @@ -1471,15 +1293,6 @@ export const MIGRATIONS: Migration[] = [ CREATE INDEX IF NOT EXISTS idx_auth_sessions_state ON auth_sessions(state, requested_at DESC); - CREATE INDEX IF NOT EXISTS idx_outbound_messages_status_scheduled - ON outbound_messages(status, scheduled_for, created_at); - - CREATE INDEX IF NOT EXISTS idx_outbound_messages_platform_account_status - ON outbound_messages(platform, account_key, status, scheduled_for); - - CREATE INDEX IF NOT EXISTS idx_slack_backfill_proofs_account_phase - ON slack_backfill_proofs(account_key, conversation_phase, updated_at); - CREATE VIEW IF NOT EXISTS message_fts_source AS SELECT m.rowid AS message_rowid, @@ -1529,11 +1342,7 @@ export const MIGRATIONS: Migration[] = [ AFTER INSERT ON message_attachments BEGIN UPDATE messages - SET - attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = NEW.message_id - ), - updated_at = MAX(updated_at, NEW.updated_at) + SET updated_at = MAX(updated_at, NEW.updated_at) WHERE id = NEW.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = NEW.message_id); @@ -1547,11 +1356,7 @@ export const MIGRATIONS: Migration[] = [ AFTER UPDATE ON message_attachments BEGIN UPDATE messages - SET - attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = NEW.message_id - ), - updated_at = MAX(updated_at, NEW.updated_at) + SET updated_at = MAX(updated_at, NEW.updated_at) WHERE id = NEW.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = NEW.message_id); @@ -1564,11 +1369,6 @@ export const MIGRATIONS: Migration[] = [ CREATE TRIGGER IF NOT EXISTS trg_message_attachments_deleted AFTER DELETE ON message_attachments BEGIN - UPDATE messages - SET attachment_count = ( - SELECT COUNT(*) FROM message_attachments WHERE message_id = OLD.message_id - ) - WHERE id = OLD.message_id; DELETE FROM messages_fts WHERE rowid IN (SELECT rowid FROM messages WHERE id = OLD.message_id); INSERT INTO messages_fts (rowid, message_id, sender_name, conversation_name, participant_names, attachment_text, content) @@ -1577,36 +1377,6 @@ export const MIGRATIONS: Migration[] = [ WHERE message_id = OLD.message_id; END; - CREATE TRIGGER IF NOT EXISTS trg_message_reactions_inserted - AFTER INSERT ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = NEW.message_id AND is_active = 1 - ) - WHERE id = NEW.message_id; - END; - - CREATE TRIGGER IF NOT EXISTS trg_message_reactions_updated - AFTER UPDATE ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = NEW.message_id AND is_active = 1 - ) - WHERE id = NEW.message_id; - END; - - CREATE TRIGGER IF NOT EXISTS trg_message_reactions_deleted - AFTER DELETE ON message_reactions - BEGIN - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) FROM message_reactions WHERE message_id = OLD.message_id AND is_active = 1 - ) - WHERE id = OLD.message_id; - END; - CREATE TRIGGER IF NOT EXISTS trg_contacts_name_updated AFTER UPDATE OF name ON contacts BEGIN @@ -1622,10 +1392,6 @@ export const MIGRATIONS: Migration[] = [ SET actor_name = NEW.name WHERE actor_contact_id = NEW.id; - UPDATE message_reactions - SET reactor_name = NEW.name - WHERE reactor_contact_id = NEW.id; - UPDATE conversations SET participant_names = ( SELECT GROUP_CONCAT(cp.participant_name, ' | ') @@ -1779,22 +1545,19 @@ export const MIGRATIONS: Migration[] = [ id: "0002_upgrade_existing_schema_columns", apply: (db) => { addColumnIfMissing(db, "raw_events", "normalized_schema TEXT"); - addColumnIfMissing(db, "raw_events", "provenance_json TEXT"); addColumnIfMissing(db, "conversations", "is_active INTEGER NOT NULL DEFAULT 1"); addColumnIfMissing(db, "conversations", "removal_reason TEXT"); db.exec(` CREATE TABLE IF NOT EXISTS projection_state ( singleton_key TEXT PRIMARY KEY CHECK (singleton_key = 'global'), projection_watermark INTEGER NOT NULL DEFAULT 0, - last_projected_at INTEGER, - last_rebuild_at INTEGER, updated_at INTEGER NOT NULL ) `); db.exec(` INSERT OR IGNORE INTO projection_state ( - singleton_key, projection_watermark, last_projected_at, last_rebuild_at, updated_at - ) VALUES ('global', 0, NULL, NULL, strftime('%s','now') * 1000) + singleton_key, projection_watermark, updated_at + ) VALUES ('global', 0, strftime('%s','now') * 1000) `); if (columnExists(db, "conversations", "subtype")) { db.exec(` @@ -1842,7 +1605,6 @@ export const MIGRATIONS: Migration[] = [ db.exec(` CREATE TABLE IF NOT EXISTS contact_merge_decisions ( id TEXT PRIMARY KEY, - decision_type TEXT NOT NULL, primary_contact_id TEXT NOT NULL, secondary_contact_id TEXT NOT NULL, canonical_contact_id TEXT NOT NULL, @@ -1887,51 +1649,25 @@ export const MIGRATIONS: Migration[] = [ id: "0007_add_generic_sync_proof_tables", apply: (db) => { db.exec(` - CREATE TABLE IF NOT EXISTS sync_scopes ( + CREATE TABLE IF NOT EXISTS sync_proofs ( id TEXT PRIMARY KEY, platform TEXT NOT NULL, account_key TEXT NOT NULL, scope_kind TEXT NOT NULL, scope_key TEXT NOT NULL, - parent_scope_id TEXT REFERENCES sync_scopes(id) ON DELETE CASCADE, - display_name TEXT, - metadata_json TEXT, - first_discovered_at INTEGER NOT NULL, - last_observed_at INTEGER NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(platform, account_key, scope_kind, scope_key) - ); - - CREATE TABLE IF NOT EXISTS sync_proofs ( - id TEXT PRIMARY KEY, - platform TEXT NOT NULL, - account_key TEXT NOT NULL, - scope_id TEXT NOT NULL REFERENCES sync_scopes(id) ON DELETE CASCADE, proof_kind TEXT NOT NULL, status TEXT NOT NULL, - sync_mode TEXT, - run_started_at INTEGER, last_observed_at INTEGER NOT NULL, - completed_at INTEGER, - fresh_until INTEGER, resume_cursor_json TEXT, coverage_json TEXT, stats_json TEXT, - error_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, - UNIQUE(platform, account_key, scope_id, proof_kind) + UNIQUE(platform, account_key, scope_kind, scope_key, proof_kind) ); - CREATE INDEX IF NOT EXISTS idx_sync_scopes_lookup - ON sync_scopes(platform, account_key, scope_kind, scope_key); - - CREATE INDEX IF NOT EXISTS idx_sync_scopes_parent - ON sync_scopes(parent_scope_id, updated_at); - CREATE INDEX IF NOT EXISTS idx_sync_proofs_lookup - ON sync_proofs(platform, account_key, scope_id, proof_kind); + ON sync_proofs(platform, account_key, scope_kind, scope_key, proof_kind); CREATE INDEX IF NOT EXISTS idx_sync_proofs_status ON sync_proofs(platform, account_key, status, updated_at); @@ -1960,12 +1696,6 @@ export const MIGRATIONS: Migration[] = [ ON timeline_events(actor_contact_id); `); } - if (tableExists(db, "message_reactions")) { - db.exec(` - CREATE INDEX IF NOT EXISTS idx_message_reactions_reactor_contact - ON message_reactions(reactor_contact_id); - `); - } }, }, { @@ -1977,12 +1707,6 @@ export const MIGRATIONS: Migration[] = [ addColumnIfMissing(db, "timeline_events", "system_kind TEXT"); addColumnIfMissing(db, "timeline_events", "call_provider TEXT"); addColumnIfMissing(db, "timeline_events", "call_direction TEXT"); - addColumnIfMissing(db, "timeline_events", "call_status TEXT"); - addColumnIfMissing(db, "timeline_events", "call_medium TEXT"); - addColumnIfMissing(db, "timeline_events", "call_started_at INTEGER"); - addColumnIfMissing(db, "timeline_events", "call_duration_seconds INTEGER"); - addColumnIfMissing(db, "timeline_events", "call_ended_at INTEGER"); - addColumnIfMissing(db, "timeline_events", "call_disconnected_cause TEXT"); db.exec(` UPDATE timeline_events SET system_kind = COALESCE(system_kind, 'provider_notice') @@ -1999,7 +1723,6 @@ export const MIGRATIONS: Migration[] = [ "sync_checkpoints", "source_accounts", "sync_proofs", - "sync_scopes", "sync_runs", "sync_run_errors", ]) { @@ -2061,54 +1784,15 @@ export const MIGRATIONS: Migration[] = [ `); }, }, - { - id: "0014_jobs_table", - apply: (db) => { - db.exec(` - CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - platform TEXT, - account_key TEXT, - priority INTEGER NOT NULL, - status TEXT NOT NULL, - trigger TEXT NOT NULL, - queued_at INTEGER NOT NULL, - scheduled_at INTEGER NOT NULL, - started_at INTEGER, - finished_at INTEGER, - attempt INTEGER NOT NULL DEFAULT 0, - owner_id TEXT, - lease_expires_at INTEGER, - last_progress_at INTEGER, - checkpoint_json TEXT, - progress_json TEXT, - error_json TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_jobs_claim - ON jobs(status, scheduled_at, priority, queued_at); - - CREATE INDEX IF NOT EXISTS idx_jobs_owner_lease - ON jobs(owner_id, lease_expires_at); - - CREATE INDEX IF NOT EXISTS idx_jobs_platform_account_status - ON jobs(platform, account_key, status, scheduled_at); - `); - }, - }, { id: "0015_message_fts_index_queue", apply: (db) => { db.exec(` CREATE TABLE IF NOT EXISTS message_fts_index_queue ( message_id TEXT PRIMARY KEY, - reason TEXT NOT NULL, status TEXT NOT NULL, - attempt INTEGER NOT NULL DEFAULT 0, queued_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - last_error TEXT + updated_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_message_fts_index_queue_status @@ -2175,8 +1859,6 @@ export const MIGRATIONS: Migration[] = [ addColumnIfMissing(db, "sync_runs", "attempt INTEGER NOT NULL DEFAULT 0"); addColumnIfMissing(db, "sync_runs", "owner_id TEXT"); addColumnIfMissing(db, "sync_runs", "lease_expires_at INTEGER"); - addColumnIfMissing(db, "sync_runs", "last_progress_at INTEGER"); - addColumnIfMissing(db, "sync_runs", "error_code TEXT"); db.exec(` UPDATE sync_runs SET attempt = COALESCE(attempt, 0) @@ -2187,9 +1869,7 @@ export const MIGRATIONS: Migration[] = [ status = 'queued', scheduled_at = strftime('%s','now') * 1000, owner_id = NULL, - lease_expires_at = NULL, - last_progress_at = COALESCE(last_progress_at, started_at, queued_at), - error_code = 'daemon_recovered' + lease_expires_at = NULL WHERE status IN ('ingesting', 'projecting'); DROP INDEX IF EXISTS idx_sync_runs_status_type_queue; @@ -2213,20 +1893,232 @@ export const MIGRATIONS: Migration[] = [ db.exec(` CREATE TABLE IF NOT EXISTS raw_event_projection_failures ( raw_event_rowid INTEGER PRIMARY KEY, - raw_event_id TEXT NOT NULL, - platform TEXT NOT NULL, - account_key TEXT NOT NULL, - entity_kind TEXT NOT NULL, - event_kind TEXT NOT NULL, - normalized_schema TEXT, - source_version TEXT, - error_message TEXT NOT NULL, - failed_at INTEGER NOT NULL + error_message TEXT NOT NULL ); - - CREATE INDEX IF NOT EXISTS idx_raw_event_projection_failures_platform - ON raw_event_projection_failures(platform, account_key, failed_at); `); }, }, + { + id: "0021_drop_outbound_messages", + apply: (db) => { + db.exec(` + DROP INDEX IF EXISTS idx_outbound_messages_status_scheduled; + DROP INDEX IF EXISTS idx_outbound_messages_platform_account_status; + DROP TABLE IF EXISTS outbound_messages; + `); + }, + }, + { + id: "0022_drop_jobs", + apply: (db) => { + db.exec(` + DROP INDEX IF EXISTS idx_jobs_claim; + DROP INDEX IF EXISTS idx_jobs_owner_lease; + DROP INDEX IF EXISTS idx_jobs_platform_account_status; + DROP TABLE IF EXISTS jobs; + `); + }, + }, + { + id: "0023_drop_dead_db_subfields", + apply: (db) => { + collapseSyncProofScopesIfNeeded(db); + dropColumnIfExists(db, "contact_merge_decisions", "decision_type"); + dropColumnIfExists(db, "sync_checkpoints", "last_full_sync_at"); + dropColumnIfExists(db, "sync_checkpoints", "last_error_at"); + dropColumnIfExists(db, "sync_proofs", "sync_mode"); + dropColumnIfExists(db, "sync_proofs", "run_started_at"); + dropColumnIfExists(db, "sync_proofs", "completed_at"); + dropColumnIfExists(db, "sync_proofs", "fresh_until"); + dropColumnIfExists(db, "sync_proofs", "error_json"); + db.exec(` + DROP INDEX IF EXISTS idx_slack_backfill_proofs_account_phase; + DROP TABLE IF EXISTS slack_backfill_proofs; + `); + }, + }, + { + id: "0024_drop_source_account_dead_fields", + apply: (db) => { + dropColumnIfExists(db, "source_accounts", "status"); + dropColumnIfExists(db, "source_accounts", "metadata_json"); + }, + }, + { + id: "0025_reduce_raw_event_projection_failures", + apply: (db) => { + db.exec("DROP INDEX IF EXISTS idx_raw_event_projection_failures_platform"); + dropColumnIfExists(db, "raw_event_projection_failures", "entity_kind"); + dropColumnIfExists(db, "raw_event_projection_failures", "event_kind"); + dropColumnIfExists(db, "raw_event_projection_failures", "normalized_schema"); + dropColumnIfExists(db, "raw_event_projection_failures", "source_version"); + dropColumnIfExists(db, "raw_event_projection_failures", "failed_at"); + }, + }, + { + id: "0026_drop_message_fts_queue_reason", + apply: (db) => { + dropColumnIfExists(db, "message_fts_index_queue", "reason"); + }, + }, + { + id: "0027_drop_contact_source_metadata", + apply: (db) => { + dropColumnIfExists(db, "contact_sources", "metadata_json"); + }, + }, + { + id: "0028_drop_participant_history_fields", + apply: (db) => { + dropColumnIfExists(db, "conversation_participants", "role"); + dropColumnIfExists(db, "conversation_participants", "joined_at"); + dropColumnIfExists(db, "conversation_participants", "left_at"); + }, + }, + { + id: "0029_drop_timeline_unused_call_fields", + apply: (db) => { + dropColumnIfExists(db, "timeline_events", "call_status"); + dropColumnIfExists(db, "timeline_events", "call_medium"); + dropColumnIfExists(db, "timeline_events", "call_started_at"); + dropColumnIfExists(db, "timeline_events", "call_duration_seconds"); + dropColumnIfExists(db, "timeline_events", "call_ended_at"); + dropColumnIfExists(db, "timeline_events", "call_disconnected_cause"); + dropColumnIfExists(db, "timeline_events", "metadata_json"); + }, + }, + { + id: "0030_drop_reaction_type", + apply: (db) => { + dropColumnIfExists(db, "message_reactions", "reaction_type"); + }, + }, + { + id: "0031_drop_reaction_contact_name_fields", + apply: (db) => { + db.exec("DROP INDEX IF EXISTS idx_message_reactions_reactor_contact"); + dropColumnIfExists(db, "message_reactions", "reactor_contact_id"); + dropColumnIfExists(db, "message_reactions", "reactor_name"); + }, + }, + { + id: "0032_drop_attachment_unused_metadata_fields", + apply: (db) => { + dropColumnIfExists(db, "message_attachments", "preview_ref_json"); + dropColumnIfExists(db, "message_attachments", "availability_status"); + dropColumnIfExists(db, "message_attachments", "provider_metadata_json"); + dropColumnIfExists(db, "message_attachments", "metadata_json"); + }, + }, + { + id: "0033_drop_contact_source_seen_timestamps", + apply: (db) => { + dropColumnIfExists(db, "contact_sources", "first_seen_at"); + dropColumnIfExists(db, "contact_sources", "last_seen_at"); + }, + }, + { + id: "0034_drop_conversation_unused_fields", + apply: (db) => { + dropColumnIfExists(db, "conversations", "native_conversation_key"); + dropColumnIfExists(db, "conversations", "topic"); + }, + }, + { + id: "0035_drop_projection_state_unused_timestamps", + apply: (db) => { + dropColumnIfExists(db, "projection_state", "last_projected_at"); + dropColumnIfExists(db, "projection_state", "last_rebuild_at"); + }, + }, + { + id: "0036_drop_sync_run_unused_fields", + apply: (db) => { + dropColumnIfExists(db, "sync_runs", "last_progress_at"); + dropColumnIfExists(db, "sync_runs", "error_code"); + dropColumnIfExists(db, "sync_run_errors", "sync_run_id"); + dropColumnIfExists(db, "sync_run_errors", "error_code"); + dropColumnIfExists(db, "sync_run_errors", "details_json"); + }, + }, + { + id: "0037_drop_message_fts_unused_fields", + apply: (db) => { + dropColumnIfExists(db, "message_fts_index_queue", "attempt"); + dropColumnIfExists(db, "message_fts_index_queue", "last_error"); + }, + }, + { + id: "0038_drop_contact_handle_origin_fields", + apply: (db) => { + if (!tableExists(db, "contact_handles")) { + return; + } + db.exec("DROP INDEX IF EXISTS idx_contact_handles_lookup"); + dropColumnIfExists(db, "contact_handles", "platform"); + dropColumnIfExists(db, "contact_handles", "account_key"); + db.exec(` + CREATE INDEX IF NOT EXISTS idx_contact_handles_lookup + ON contact_handles(type, normalized_value); + `); + }, + }, + { + id: "0039_drop_contact_source_profile_url", + apply: (db) => { + dropColumnIfExists(db, "contact_sources", "profile_url"); + }, + }, + { + id: "0040_reduce_raw_event_projection_failures_to_rowid", + apply: (db) => { + dropColumnIfExists(db, "raw_event_projection_failures", "raw_event_id"); + dropColumnIfExists(db, "raw_event_projection_failures", "platform"); + dropColumnIfExists(db, "raw_event_projection_failures", "account_key"); + }, + }, + { + id: "0041_drop_unused_conversation_message_fields", + apply: (db) => { + dropColumnIfExists(db, "conversations", "removal_reason"); + dropColumnIfExists(db, "conversations", "service"); + dropColumnIfExists(db, "messages", "service"); + dropColumnIfExists(db, "messages", "edited_at"); + dropColumnIfExists(db, "messages", "deleted_at"); + dropColumnIfExists(db, "messages", "reply_to_message_id"); + dropColumnIfExists(db, "messages", "is_edited"); + }, + }, + { + id: "0042_drop_unused_raw_event_audit_fields", + apply: (db) => { + dropColumnIfExists(db, "raw_events", "external_event_id"); + dropColumnIfExists(db, "raw_events", "cursor_json"); + dropColumnIfExists(db, "raw_events", "source_version"); + }, + }, + { + id: "0043_drop_unused_projection_summary_fields", + apply: (db) => { + dropSynchronousMessageFtsTriggers(db); + dropColumnIfExists(db, "conversations", "last_message_id"); + dropColumnIfExists(db, "conversations", "last_message_at"); + dropColumnIfExists(db, "conversations", "last_message_preview"); + dropColumnIfExists(db, "conversations", "unread_count"); + dropColumnIfExists(db, "messages", "attachment_count"); + dropColumnIfExists(db, "messages", "reaction_count"); + }, + }, + { + id: "0044_drop_unused_raw_event_provenance_json", + apply: (db) => { + dropColumnIfExists(db, "raw_events", "provenance_json"); + }, + }, + { + id: "0045_drop_message_reaction_source_key", + apply: (db) => { + dropMessageReactionSourceKeyIfNeeded(db); + }, + }, ]; diff --git a/src/db/schema.ts b/src/db/schema.ts index 6a417d38..cf5d6745 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -6,8 +6,6 @@ import { CONVERSATION_TYPE_VALUES, INTEGRATION_AUTH_STATE_VALUES, INTEGRATION_LAUNCH_STRATEGY_VALUES, - JOB_KIND_VALUES, - JOB_STATUS_VALUES, PLATFORM_VALUES, RAW_EVENT_ENTITY_KIND_VALUES, SYNC_MODE_VALUES, @@ -32,8 +30,6 @@ export const sourceAccounts = sqliteTable("source_accounts", { platform: textEnum("platform", PLATFORM_VALUES).notNull(), accountKey: text("account_key").notNull(), displayName: text("display_name"), - status: text("status").notNull(), - metadataJson: text("metadata_json"), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -45,45 +41,24 @@ export const syncCheckpoints = sqliteTable("sync_checkpoints", { sourceCursorJson: text("source_cursor_json"), rawIngestWatermark: integer("raw_ingest_watermark").notNull(), syncMode: textEnum("sync_mode", SYNC_MODE_VALUES).notNull(), - lastFullSyncAt: integer("last_full_sync_at"), lastSuccessAt: integer("last_success_at"), - lastErrorAt: integer("last_error_at"), lastErrorSummary: text("last_error_summary"), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); -export const syncScopes = sqliteTable("sync_scopes", { +export const syncProofs = sqliteTable("sync_proofs", { id: text("id").primaryKey(), platform: textEnum("platform", PLATFORM_VALUES).notNull(), accountKey: text("account_key").notNull(), scopeKind: text("scope_kind").notNull(), scopeKey: text("scope_key").notNull(), - parentScopeId: text("parent_scope_id"), - displayName: text("display_name"), - metadataJson: text("metadata_json"), - firstDiscoveredAt: integer("first_discovered_at").notNull(), - lastObservedAt: integer("last_observed_at").notNull(), - createdAt: integer("created_at").notNull(), - updatedAt: integer("updated_at").notNull(), -}); - -export const syncProofs = sqliteTable("sync_proofs", { - id: text("id").primaryKey(), - platform: textEnum("platform", PLATFORM_VALUES).notNull(), - accountKey: text("account_key").notNull(), - scopeId: text("scope_id").notNull(), proofKind: text("proof_kind").notNull(), status: text("status").notNull(), - syncMode: textEnum("sync_mode", SYNC_MODE_VALUES), - runStartedAt: integer("run_started_at"), lastObservedAt: integer("last_observed_at").notNull(), - completedAt: integer("completed_at"), - freshUntil: integer("fresh_until"), resumeCursorJson: text("resume_cursor_json"), coverageJson: text("coverage_json"), statsJson: text("stats_json"), - errorJson: text("error_json"), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -102,78 +77,22 @@ export const syncRuns = sqliteTable("sync_runs", { attempt: integer("attempt").notNull(), ownerId: text("owner_id"), leaseExpiresAt: integer("lease_expires_at"), - lastProgressAt: integer("last_progress_at"), - errorCode: text("error_code"), detailsJson: text("details_json"), }); -export const jobs = sqliteTable("jobs", { - id: text("id").primaryKey(), - kind: textEnum("kind", JOB_KIND_VALUES).notNull(), - platform: textEnum("platform", PLATFORM_VALUES), - accountKey: text("account_key"), - priority: integer("priority").notNull(), - status: textEnum("status", JOB_STATUS_VALUES).notNull(), - trigger: text("trigger").notNull(), - queuedAt: integer("queued_at").notNull(), - scheduledAt: integer("scheduled_at").notNull(), - startedAt: integer("started_at"), - finishedAt: integer("finished_at"), - attempt: integer("attempt").notNull(), - ownerId: text("owner_id"), - leaseExpiresAt: integer("lease_expires_at"), - lastProgressAt: integer("last_progress_at"), - checkpointJson: text("checkpoint_json"), - progressJson: text("progress_json"), - errorJson: text("error_json"), -}); - export const syncRunErrors = sqliteTable("sync_run_errors", { id: text("id").primaryKey(), - syncRunId: text("sync_run_id").notNull(), platform: textEnum("platform", PLATFORM_VALUES), accountKey: text("account_key"), - errorCode: text("error_code"), errorMessage: text("error_message").notNull(), - detailsJson: text("details_json"), createdAt: integer("created_at").notNull(), }); export const messageFtsIndexQueue = sqliteTable("message_fts_index_queue", { messageId: text("message_id").primaryKey(), - reason: text("reason").notNull(), status: text("status", { enum: ["queued", "indexing", "completed", "failed"] }).notNull(), - attempt: integer("attempt").notNull(), queuedAt: integer("queued_at").notNull(), updatedAt: integer("updated_at").notNull(), - lastError: text("last_error"), -}); - -export const slackBackfillProofs = sqliteTable("slack_backfill_proofs", { - id: text("id").primaryKey(), - accountKey: text("account_key").notNull(), - teamId: text("team_id").notNull(), - conversationId: text("conversation_id").notNull(), - conversationName: text("conversation_name"), - conversationFamily: text("conversation_family").notNull(), - syncMode: text("sync_mode").notNull(), - scanStartedAt: integer("scan_started_at").notNull(), - knownConversationCount: integer("known_conversation_count").notNull(), - conversationPhase: text("conversation_phase").notNull(), - historyComplete: integer("history_complete").notNull(), - historyCursor: text("history_cursor"), - threadRootCount: integer("thread_root_count").notNull(), - completedThreadCount: integer("completed_thread_count").notNull(), - pendingThreadCount: integer("pending_thread_count").notNull(), - activeThreadTs: text("active_thread_ts"), - repliesCursor: text("replies_cursor"), - oldestMessageTs: text("oldest_message_ts"), - newestMessageTs: text("newest_message_ts"), - firstDiscoveredAt: integer("first_discovered_at").notNull(), - historyCompleteAt: integer("history_complete_at"), - repliesCompleteAt: integer("replies_complete_at"), - lastObservedAt: integer("last_observed_at").notNull(), - updatedAt: integer("updated_at").notNull(), }); export const daemonState = sqliteTable("daemon_state", { @@ -195,8 +114,6 @@ export const appSettings = sqliteTable("app_settings", { export const projectionState = sqliteTable("projection_state", { singletonKey: text("singleton_key").primaryKey(), projectionWatermark: integer("projection_watermark").notNull(), - lastProjectedAt: integer("last_projected_at"), - lastRebuildAt: integer("last_rebuild_at"), updatedAt: integer("updated_at").notNull(), }); @@ -206,30 +123,18 @@ export const rawEvents = sqliteTable("raw_events", { accountKey: text("account_key").notNull(), entityKind: textEnum("entity_kind", RAW_EVENT_ENTITY_KIND_VALUES).notNull(), eventKind: text("event_kind").notNull(), - externalEventId: text("external_event_id"), externalEntityId: text("external_entity_id"), conversationExternalId: text("conversation_external_id"), occurredAt: integer("occurred_at"), observedAt: integer("observed_at").notNull(), - cursorJson: text("cursor_json"), dedupeKey: text("dedupe_key").notNull(), payloadJson: text("payload_json").notNull(), normalizedSchema: text("normalized_schema"), - provenanceJson: text("provenance_json"), - sourceVersion: text("source_version"), }); export const rawEventProjectionFailures = sqliteTable("raw_event_projection_failures", { rawEventRowId: integer("raw_event_rowid").primaryKey(), - rawEventId: text("raw_event_id").notNull(), - platform: textEnum("platform", PLATFORM_VALUES).notNull(), - accountKey: text("account_key").notNull(), - entityKind: textEnum("entity_kind", RAW_EVENT_ENTITY_KIND_VALUES).notNull(), - eventKind: text("event_kind").notNull(), - normalizedSchema: text("normalized_schema"), - sourceVersion: text("source_version"), errorMessage: text("error_message").notNull(), - failedAt: integer("failed_at").notNull(), }); export const messageReactions = sqliteTable("message_reactions", { @@ -237,12 +142,8 @@ export const messageReactions = sqliteTable("message_reactions", { messageId: text("message_id").notNull(), accountKey: text("account_key").notNull(), platform: textEnum("platform", PLATFORM_VALUES).notNull(), - sourceReactionKey: text("source_reaction_key").notNull(), - reactionType: text("reaction_type"), emoji: text("emoji").notNull(), - reactorContactId: text("reactor_contact_id"), reactorSourceKey: text("reactor_source_key"), - reactorName: text("reactor_name"), isActive: integer("is_active").notNull(), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), @@ -265,8 +166,6 @@ export const contactHandles = sqliteTable("contact_handles", { type: text("type").notNull(), value: text("value").notNull(), normalizedValue: text("normalized_value").notNull(), - platform: textEnum("platform", PLATFORM_VALUES), - accountKey: text("account_key"), isDeterministic: integer("is_deterministic").notNull(), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), @@ -278,15 +177,10 @@ export const contactSources = sqliteTable("contact_sources", { platform: textEnum("platform", PLATFORM_VALUES).notNull(), accountKey: text("account_key").notNull(), sourceEntityKey: text("source_entity_key").notNull(), - profileUrl: text("profile_url"), - firstSeenAt: integer("first_seen_at").notNull(), - lastSeenAt: integer("last_seen_at").notNull(), - metadataJson: text("metadata_json"), }); export const contactMergeDecisions = sqliteTable("contact_merge_decisions", { id: text("id").primaryKey(), - decisionType: text("decision_type").notNull(), primaryContactId: text("primary_contact_id").notNull(), secondaryContactId: text("secondary_contact_id").notNull(), canonicalContactId: text("canonical_contact_id").notNull(), @@ -314,18 +208,10 @@ export const conversations = sqliteTable("conversations", { platform: textEnum("platform", PLATFORM_VALUES).notNull(), accountKey: text("account_key").notNull(), sourceConversationKey: text("source_conversation_key").notNull(), - nativeConversationKey: text("native_conversation_key"), type: textEnum("type", CONVERSATION_TYPE_VALUES).notNull(), isActive: integer("is_active").notNull(), - removalReason: text("removal_reason"), - service: text("service"), name: text("name"), - topic: text("topic"), participantNames: text("participant_names"), - lastMessageId: text("last_message_id"), - lastMessageAt: integer("last_message_at"), - lastMessagePreview: text("last_message_preview"), - unreadCount: integer("unread_count").notNull(), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -334,10 +220,7 @@ export const conversationParticipants = sqliteTable("conversation_participants", conversationId: text("conversation_id").notNull(), contactId: text("contact_id").notNull(), participantName: text("participant_name"), - role: text("role"), isSelf: integer("is_self").notNull(), - joinedAt: integer("joined_at"), - leftAt: integer("left_at"), isActive: integer("is_active").notNull(), sourceParticipantKey: text("source_participant_key"), updatedAt: integer("updated_at").notNull(), @@ -354,19 +237,12 @@ export const messages = sqliteTable("messages", { senderName: text("sender_name"), conversationName: text("conversation_name"), sentAt: integer("sent_at").notNull(), - service: text("service"), status: text("status"), isFromMe: integer("is_from_me").notNull(), content: text("content"), deliveredAt: integer("delivered_at"), readAt: integer("read_at"), - editedAt: integer("edited_at"), - deletedAt: integer("deleted_at"), - replyToMessageId: text("reply_to_message_id"), isDeleted: integer("is_deleted").notNull(), - isEdited: integer("is_edited").notNull(), - attachmentCount: integer("attachment_count").notNull(), - reactionCount: integer("reaction_count").notNull(), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -387,10 +263,6 @@ export const messageAttachments = sqliteTable("message_attachments", { textContent: text("text_content"), accessKind: text("access_kind"), accessRefJson: text("access_ref_json"), - previewRefJson: text("preview_ref_json"), - availabilityStatus: text("availability_status"), - providerMetadataJson: text("provider_metadata_json"), - metadataJson: text("metadata_json"), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -441,13 +313,6 @@ export const timelineEvents = sqliteTable("timeline_events", { systemKind: text("system_kind"), callProvider: text("call_provider"), callDirection: text("call_direction"), - callStatus: text("call_status"), - callMedium: text("call_medium"), - callStartedAt: integer("call_started_at"), - callEndedAt: integer("call_ended_at"), - callDurationSeconds: integer("call_duration_seconds"), - callDisconnectedCause: text("call_disconnected_cause"), - metadataJson: text("metadata_json"), createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); @@ -488,21 +353,3 @@ export const authSessions = sqliteTable("auth_sessions", { createdAt: integer("created_at").notNull(), updatedAt: integer("updated_at").notNull(), }); - -export const outboundMessages = sqliteTable("outbound_messages", { - id: text("id").primaryKey(), - platform: textEnum("platform", PLATFORM_VALUES).notNull(), - accountKey: text("account_key").notNull(), - target: text("target").notNull(), - threadId: text("thread_id"), - text: text("text").notNull(), - status: text("status").notNull(), - attemptCount: integer("attempt_count").notNull(), - scheduledFor: integer("scheduled_for").notNull(), - startedAt: integer("started_at"), - finishedAt: integer("finished_at"), - lastError: text("last_error"), - metadataJson: text("metadata_json"), - createdAt: integer("created_at").notNull(), - updatedAt: integer("updated_at").notNull(), -}); diff --git a/src/platforms/contacts/sync.ts b/src/platforms/contacts/sync.ts index aeae46b1..ecb66020 100644 --- a/src/platforms/contacts/sync.ts +++ b/src/platforms/contacts/sync.ts @@ -188,7 +188,6 @@ export function buildContactsSyncBundle(): SyncBundle { })), ], } satisfies ContactObservationPayload, - sourceVersion: "contacts-v1", })), sourceCursor: { snapshotAt: observedBase }, syncMode: "full", diff --git a/src/platforms/core/state/integration-state.test.ts b/src/platforms/core/state/integration-state.test.ts index c7d2a074..db29b8ca 100644 --- a/src/platforms/core/state/integration-state.test.ts +++ b/src/platforms/core/state/integration-state.test.ts @@ -67,17 +67,6 @@ describe("integration state management", () => { return db; } - type RawSql = { - prepare(sql: string): { - run(...params: unknown[]): unknown; - get(...params: unknown[]): unknown; - }; - }; - - function rawSql(db: CuedDatabase): RawSql { - return (db as unknown as { sqlite: RawSql }).sqlite; - } - function createPackagedSignalHelper(version = "0.12.9"): string { process.env.CUED_SIGNAL_DIR = createTempDir("cued-signal-config-"); const appPath = join(createTempDir("cued-app-"), "Cued.app"); @@ -754,11 +743,13 @@ process.exit(44); importedFrom: "local-cli", metadata: {}, }); - db.upsertSourceAccount({ - platform: "linkedin", - accountKey: "default", - displayName: "Avery Example", - }); + db.upsertSourceAccounts([ + { + platform: "linkedin", + accountKey: "default", + displayName: "Avery Example", + }, + ]); expect(listIntegrationStates(db)).toEqual( expect.arrayContaining([ @@ -1331,11 +1322,13 @@ process.exit(44); importedFrom: "local-cli", metadata: {}, }); - db.upsertSourceAccount({ - platform: "signal", - accountKey: "default", - displayName: "Signal", - }); + db.upsertSourceAccounts([ + { + platform: "signal", + accountKey: "default", + displayName: "Signal", + }, + ]); db.upsertCheckpoint({ platform: "signal", accountKey: "default", @@ -1350,7 +1343,6 @@ process.exit(44); scope: { kind: "account", key: "default" }, proofKind: "messages", status: "complete", - syncMode: "incremental", observedAt: Date.now(), }, }); @@ -1376,95 +1368,6 @@ process.exit(44); db.close(); }); - it("clears legacy Slack backfill proofs when removing a Slack integration", () => { - const db = createDb(); - const timestamp = Date.now(); - db.upsertIntegrationState({ - platform: "slack", - accountKey: "T123", - displayName: "Acme", - authState: "authenticated", - enabled: true, - connectionKind: "browser-session", - syncCapable: true, - launchStrategy: "chromium-auth", - launchTarget: "https://slack.com/signin", - importedFrom: "local-cli", - metadata: {}, - }); - rawSql(db) - .prepare( - `INSERT INTO slack_backfill_proofs ( - id, - account_key, - team_id, - conversation_id, - conversation_name, - conversation_family, - sync_mode, - scan_started_at, - known_conversation_count, - conversation_phase, - history_complete, - history_cursor, - thread_root_count, - completed_thread_count, - pending_thread_count, - active_thread_ts, - replies_cursor, - oldest_message_ts, - newest_message_ts, - first_discovered_at, - history_complete_at, - replies_complete_at, - last_observed_at, - updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ) - .run( - "proof-1", - "T123", - "T123", - "C123", - "general", - "channel", - "full", - timestamp, - 1, - "complete", - 1, - null, - 0, - 0, - 0, - null, - null, - null, - null, - timestamp, - timestamp, - timestamp, - timestamp, - timestamp, - ); - - expect( - rawSql(db) - .prepare("SELECT COUNT(*) AS count FROM slack_backfill_proofs WHERE account_key = ?") - .get("T123"), - ).toEqual({ count: 1 }); - - removeIntegration(db, "slack", "T123"); - - expect( - rawSql(db) - .prepare("SELECT COUNT(*) AS count FROM slack_backfill_proofs WHERE account_key = ?") - .get("T123"), - ).toEqual({ count: 0 }); - - db.close(); - }); - it("reuses the same stable slack workspace key after remove and reconnect", () => { const db = createDb(); diff --git a/src/platforms/core/types.ts b/src/platforms/core/types.ts index fd30efc2..79a53c06 100644 --- a/src/platforms/core/types.ts +++ b/src/platforms/core/types.ts @@ -278,19 +278,6 @@ export const SYNC_RUN_STATUS_VALUES = [ ] as const; export type SyncRunStatus = (typeof SYNC_RUN_STATUS_VALUES)[number]; -export const JOB_KIND_VALUES = ["auth", "ingest", "project", "index", "cleanup", "status"] as const; -export type JobKind = (typeof JOB_KIND_VALUES)[number]; - -export const JOB_STATUS_VALUES = [ - "queued", - "running", - "retry_wait", - "completed", - "failed", - "canceled", -] as const; -export type JobStatus = (typeof JOB_STATUS_VALUES)[number]; - export const RAW_EVENT_ENTITY_KIND_VALUES = [ "contact", "conversation", diff --git a/src/platforms/discord/api/client.ts b/src/platforms/discord/api/client.ts index 72897b79..29402704 100644 --- a/src/platforms/discord/api/client.ts +++ b/src/platforms/discord/api/client.ts @@ -74,27 +74,6 @@ export class DiscordApiClient { return await this.request(`/channels/${channelId}/messages${suffix}`); } - async sendMessage( - channelId: string, - content: string, - options: { replyToMessageId?: string | null } = {}, - ): Promise { - return await this.request(`/channels/${channelId}/messages`, { - method: "POST", - body: { - content, - allowed_mentions: { parse: [] as string[] }, - ...(options.replyToMessageId - ? { - message_reference: { - message_id: options.replyToMessageId, - }, - } - : {}), - }, - }); - } - private async request( path: string, options: { diff --git a/src/platforms/discord/realtime/session.test.ts b/src/platforms/discord/realtime/session.test.ts index 85034cb1..324c8d47 100644 --- a/src/platforms/discord/realtime/session.test.ts +++ b/src/platforms/discord/realtime/session.test.ts @@ -71,9 +71,6 @@ describe("discord realtime", () => { }, ]; }, - async sendMessage() { - throw new Error("not used"); - }, } as unknown as DiscordApiClient, }); @@ -129,9 +126,6 @@ describe("discord realtime", () => { async listChannelMessages() { return []; }, - async sendMessage() { - throw new Error("not used"); - }, } as unknown as DiscordApiClient, }); @@ -168,9 +162,6 @@ describe("discord realtime", () => { async listChannelMessages() { return []; }, - async sendMessage() { - throw new Error("not used"); - }, } as unknown as DiscordApiClient, }); @@ -216,9 +207,6 @@ describe("discord realtime", () => { async listChannelMessages() { return []; }, - async sendMessage() { - throw new Error("not used"); - }, } as unknown as DiscordApiClient, }); @@ -278,9 +266,6 @@ describe("discord realtime", () => { async listChannelMessages() { return []; }, - async sendMessage() { - throw new Error("not used"); - }, } as unknown as DiscordApiClient, }); diff --git a/src/platforms/discord/realtime/session.ts b/src/platforms/discord/realtime/session.ts index b47008ec..403afbca 100644 --- a/src/platforms/discord/realtime/session.ts +++ b/src/platforms/discord/realtime/session.ts @@ -61,11 +61,6 @@ export interface DiscordRealtimeSessionLike { stop(): void; getStatus(): DiscordRealtimeStatus; isConnected(): boolean; - sendMessage( - channelId: string, - text: string, - options?: { replyToMessageId?: string | null }, - ): Promise; } export interface DiscordRealtimeSupervisorSessionInput { @@ -197,14 +192,6 @@ export class DiscordRealtimeSession implements DiscordRealtimeSessionLike { return this.status.state === "connected"; } - async sendMessage( - channelId: string, - text: string, - options: { replyToMessageId?: string | null } = {}, - ): Promise { - return await this.client.sendMessage(channelId, text, options); - } - private async bootstrap(): Promise { try { const reconnected = this.hasEverConnected; diff --git a/src/platforms/discord/sync/bundle.test.ts b/src/platforms/discord/sync/bundle.test.ts index bebc321e..8fc18095 100644 --- a/src/platforms/discord/sync/bundle.test.ts +++ b/src/platforms/discord/sync/bundle.test.ts @@ -167,7 +167,6 @@ describe("buildDiscordSyncBundle", () => { expect(findDiscordProof(bundle, "account", "u-self", "discovery")).toEqual( expect.objectContaining({ status: "complete", - completedAt: expect.any(Number), stats: { discoveredDmCount: 3, }, @@ -318,7 +317,6 @@ describe("buildDiscordSyncBundle", () => { expect(findDiscordProof(bundle, "conversation", "dm-2", "latest_messages")).toEqual( expect.objectContaining({ status: "failed", - completedAt: null, resumeCursor: { latestMessageId: "200", }, @@ -326,10 +324,6 @@ describe("buildDiscordSyncBundle", () => { latestMessageId: "200", previousLatestMessageId: null, }, - error: { - message: "Discord API rate limited", - retryAfterMs: null, - }, }), ); expect(findDiscordProof(bundle, "conversation", "dm-3", "latest_messages")).toBeUndefined(); @@ -819,11 +813,6 @@ describe("buildDiscordSyncBundle", () => { resumeCursor: { before: "250", }, - error: { - message: "Discord backfill cursor did not advance before '250'", - retryAfterMs: null, - rateLimited: false, - }, }), ); expect(bundle.hasMore).toBe(false); diff --git a/src/platforms/discord/sync/bundle.ts b/src/platforms/discord/sync/bundle.ts index 0560c33c..dcc681e8 100644 --- a/src/platforms/discord/sync/bundle.ts +++ b/src/platforms/discord/sync/bundle.ts @@ -11,7 +11,6 @@ import type { DiscordMessage, DiscordStoredCredentials, DiscordUser } from "../t import { discordDisplayName, isDiscordDmChannel } from "../types.js"; import { buildDiscordContactEvent, - buildDiscordConversationDisplayName, buildDiscordConversationEvent, buildDiscordMessageEvent, } from "./events.js"; @@ -416,13 +415,10 @@ function buildDiscordSyncProofs(input: { scope: { kind: "account", key: input.currentUser.id, - displayName: discordDisplayName(input.currentUser), }, proofKind: "discovery", status: "complete", - syncMode: "incremental", observedAt: input.observedAt, - completedAt: input.observedAt, stats: { discoveredDmCount: input.channels.length, }, @@ -438,28 +434,16 @@ function buildDiscordSyncProofs(input: { for (const channel of input.channels) { const latestMessageId = input.nextChannelCursor[channel.id]?.latestMessageId ?? null; - const displayName = buildDiscordConversationDisplayName(channel, input.currentUser); const scope = { kind: "conversation" as const, key: channel.id, - parent: { - kind: "account" as const, - key: input.currentUser.id, - }, - displayName, - metadata: { - type: channel.type, - dmOnly: true, - }, }; proofs.push({ scope, proofKind: "discovery", status: "complete", - syncMode: "incremental", observedAt: input.observedAt, - completedAt: input.observedAt, coverage: { latestMessageId, }, @@ -473,9 +457,7 @@ function buildDiscordSyncProofs(input: { scope, proofKind: "latest_messages", status: "failed", - syncMode: "incremental", observedAt: input.observedAt, - completedAt: null, resumeCursor: latestMessageId ? { latestMessageId, @@ -488,10 +470,6 @@ function buildDiscordSyncProofs(input: { stats: { hydratedThisRun: false, }, - error: { - message: input.hydrationError, - retryAfterMs: input.hydrationRetryAfterMs, - }, }); continue; } @@ -504,9 +482,7 @@ function buildDiscordSyncProofs(input: { scope, proofKind: "latest_messages", status: "complete", - syncMode: "incremental", observedAt: input.observedAt, - completedAt: input.observedAt, coverage: { latestMessageId, previousLatestMessageId: proofPreviousLatestMessageId, @@ -541,9 +517,7 @@ function buildDiscordSyncProofs(input: { scope, proofKind: "messages", status: historyComplete ? "complete" : "running", - syncMode: "incremental", observedAt: input.observedAt, - completedAt: historyComplete ? input.observedAt : null, resumeCursor: historyComplete ? null : { @@ -590,9 +564,7 @@ function buildDiscordBackfillProof(input: { scope: input.scope, proofKind: "messages", status, - syncMode: "incremental", observedAt: input.observedAt, - completedAt: status === "complete" ? input.observedAt : null, resumeCursor: status === "complete" ? null @@ -608,13 +580,6 @@ function buildDiscordBackfillProof(input: { messageLimit: DISCORD_INCREMENTAL_PAGE_LIMIT, backfill: true, }, - error: input.backfilled.error - ? { - message: input.backfilled.error, - retryAfterMs: input.backfilled.retryAfterMs, - rateLimited: input.backfilled.rateLimited, - } - : null, }; } diff --git a/src/platforms/discord/sync/events.test.ts b/src/platforms/discord/sync/events.test.ts index 203a5fc8..5ad2757e 100644 --- a/src/platforms/discord/sync/events.test.ts +++ b/src/platforms/discord/sync/events.test.ts @@ -55,7 +55,6 @@ describe("discord sync events", () => { sourceConversationKey: "discord:channel:c-1", displayName: "planning", conversationType: "group", - topic: "Team chat", }); }); @@ -101,8 +100,6 @@ describe("discord sync events", () => { sourceConversationKey: "discord:channel:c-1", senderSourceKey: "discord:u-peer", isFromMe: false, - isEdited: true, - replyToSourceMessageKey: "discord:message:c-1:m-0", }); const attachments = (event.payload as { attachments?: Array> }) .attachments; diff --git a/src/platforms/discord/sync/events.ts b/src/platforms/discord/sync/events.ts index 3a72e0e3..74342026 100644 --- a/src/platforms/discord/sync/events.ts +++ b/src/platforms/discord/sync/events.ts @@ -57,7 +57,6 @@ export function buildDiscordContactEvent(input: { }, ], } satisfies ContactObservationPayload, - sourceVersion: "discord-v1", }; } @@ -93,12 +92,8 @@ export function buildDiscordConversationEvent(input: { sourceConversationKey: discordConversationSourceKey(input.channel.id), conversationType: input.channel.type === 3 ? "group" : "dm", displayName, - nativeConversationKey: input.channel.id, - service: "discord", - topic: input.channel.topic ?? null, participants, } satisfies ConversationObservationPayload, - sourceVersion: "discord-v1", }; } @@ -126,11 +121,6 @@ export function buildDiscordMessageEvent(input: { height: attachment.height ?? null, access_kind: attachment.url ? "remote_url" : "none", access_ref: attachment.url ? { url: attachment.url } : null, - preview_ref: attachment.proxy_url ? { url: attachment.proxy_url } : null, - availability_status: attachment.url ? "available" : "metadata_only", - provider_metadata: { - proxyUrl: attachment.proxy_url ?? null, - }, })) ?? []; const fallbackContent = attachments @@ -158,20 +148,10 @@ export function buildDiscordMessageEvent(input: { : discordSourceKey(input.message.author.id), sentAt, content: input.message.content || fallbackContent, - service: "discord", status: null, isFromMe: input.message.author.id === input.currentUserId, - editedAt: discordTimestampMs(input.message.edited_timestamp), - isEdited: Boolean(input.message.edited_timestamp), - replyToSourceMessageKey: input.message.message_reference?.message_id - ? discordMessageSourceKey( - input.message.message_reference.channel_id ?? input.message.channel_id, - input.message.message_reference.message_id, - ) - : null, attachments, } satisfies MessagePayload, - sourceVersion: "discord-v1", }; } diff --git a/src/platforms/gmail/sync/bundle.test.ts b/src/platforms/gmail/sync/bundle.test.ts index 3c79cd92..c9f506f1 100644 --- a/src/platforms/gmail/sync/bundle.test.ts +++ b/src/platforms/gmail/sync/bundle.test.ts @@ -157,7 +157,6 @@ describe("Gmail sync bundle", () => { const messageEvents = bundle.rawEvents.filter((event) => event.entityKind === "message"); expect(messageEvents).toHaveLength(1); - expect(messageEvents[0]?.externalEventId).toBe("m-1"); expect(bundle.proofs?.[0]?.stats).toEqual( expect.objectContaining({ listedMessageCount: 2, @@ -246,7 +245,6 @@ describe("Gmail sync bundle", () => { mime_type: "application/pdf", access_kind: "provider_fetch", access_ref: { messageId: "m-attachment", attachmentId: "att-1" }, - availability_status: "available", }), ], }), diff --git a/src/platforms/gmail/sync/bundle.ts b/src/platforms/gmail/sync/bundle.ts index fffca936..ec4cf13e 100644 --- a/src/platforms/gmail/sync/bundle.ts +++ b/src/platforms/gmail/sync/bundle.ts @@ -191,15 +191,10 @@ export async function buildGmailSyncBundle( scope: { kind: "account", key: "all_mail_except_spam_trash", - displayName: "All Gmail mail except spam and trash", - metadata: { emailAddress }, }, proofKind: "messages", status: hasMore ? "running" : "complete", - syncMode: "incremental", observedAt, - runStartedAt: cursor.startedAt ?? observedAt, - completedAt: hasMore ? null : observedAt, resumeCursor: hasMore ? sourceCursor : null, coverage: { emailAddress, @@ -306,15 +301,10 @@ export async function buildGmailSyncBundle( scope: { kind: "account", key: "all_mail_except_spam_trash", - displayName: "All Gmail mail except spam and trash", - metadata: { emailAddress }, }, proofKind: "messages", status: historicalSyncComplete ? "complete" : "running", - syncMode: "full", observedAt, - runStartedAt: startedAt, - completedAt: historicalSyncComplete ? observedAt : null, resumeCursor: hasMore ? sourceCursor : null, coverage: { emailAddress, diff --git a/src/platforms/gmail/sync/events.ts b/src/platforms/gmail/sync/events.ts index 3de9d4c4..899807ab 100644 --- a/src/platforms/gmail/sync/events.ts +++ b/src/platforms/gmail/sync/events.ts @@ -51,10 +51,6 @@ function collectAttachmentParts( size_bytes: payload.body?.size ?? null, access_kind: attachmentId ? "provider_fetch" : "none", access_ref: attachmentId ? { messageId, attachmentId } : null, - availability_status: attachmentId ? "available" : "metadata_only", - provider_metadata: { - partId: payload.partId ?? null, - }, }); } @@ -175,7 +171,6 @@ export function buildGmailRawEvents(input: { }, handles: [{ type: "email", value: participant.email, deterministic: true }], } satisfies ContactObservationPayload, - sourceVersion: "gmail-v1", }); } @@ -210,7 +205,6 @@ export function buildGmailRawEvents(input: { accountKey: input.accountKey, entityKind: "message", eventKind: "created", - externalEventId: message.id, externalEntityId: message.id, conversationExternalId: message.threadId, occurredAt: sentAt, @@ -222,11 +216,9 @@ export function buildGmailRawEvents(input: { senderSourceKey: sourceKey(sender), sentAt, content: subject ? `Subject: ${subject}\n\n${content}` : content, - service: "gmail", isFromMe: sender === selfEmail, attachments, } satisfies MessagePayload, - sourceVersion: "gmail-v1", }); } @@ -244,14 +236,11 @@ export function buildGmailRawEvents(input: { sourceConversationKey: conversation.conversationKey, conversationType: conversation.participants.size > 2 ? "group" : "dm", displayName: conversation.subject, - nativeConversationKey: threadId, - service: "gmail", participants: [...conversation.participants.values()].map((participant) => ({ sourceEntityKey: sourceKey(participant.email), isSelf: participant.email === selfEmail, })), } satisfies ConversationObservationPayload, - sourceVersion: "gmail-v1", }); } diff --git a/src/platforms/imessage/call-history.ts b/src/platforms/imessage/call-history.ts index 36da830b..68fc1645 100644 --- a/src/platforms/imessage/call-history.ts +++ b/src/platforms/imessage/call-history.ts @@ -45,14 +45,11 @@ export interface ImsCallRecord { remoteAddress: string | null; remoteDisplayName: string | null; provider: CallProvider; - providerCallType: string | null; direction: CallDirection; medium: CallMedium; status: CallStatus; startedAt: number; - endedAt: number | null; durationSeconds: number | null; - disconnectedCause: string | null; syntheticConversation: boolean; } @@ -234,8 +231,6 @@ export function loadCallHistoryBatch(options?: { typeof row.date_value === "number" && Number.isFinite(row.date_value) ? Math.round((row.date_value + APPLE_EPOCH_OFFSET) * 1000) : 0; - const endedAt = - typeof durationSeconds === "number" ? startedAt + durationSeconds * 1000 : null; return { pk: row.pk, sourceCallKey: row.unique_id?.trim() || `callhistory:${row.pk}`, @@ -244,10 +239,6 @@ export function loadCallHistoryBatch(options?: { remoteAddress, remoteDisplayName: row.name?.trim() || null, provider, - providerCallType: - typeof row.call_type === "number" && Number.isFinite(row.call_type) - ? String(row.call_type) - : null, direction, medium: normalizeCallMedium(provider, row.call_type), status: normalizeCallStatus( @@ -257,12 +248,7 @@ export function loadCallHistoryBatch(options?: { durationSeconds, ), startedAt, - endedAt, durationSeconds, - disconnectedCause: - typeof row.disconnected_cause === "number" && Number.isFinite(row.disconnected_cause) - ? String(row.disconnected_cause) - : null, syntheticConversation: conversationChatId === null, } satisfies ImsCallRecord; }); diff --git a/src/platforms/imessage/sync.ts b/src/platforms/imessage/sync.ts index 36f181d6..88d8c0c2 100644 --- a/src/platforms/imessage/sync.ts +++ b/src/platforms/imessage/sync.ts @@ -232,7 +232,6 @@ export function buildIMessageSyncBundle(options?: { }, ], } satisfies ContactObservationPayload, - sourceVersion: "imessage-v1", }); } @@ -251,12 +250,10 @@ export function buildIMessageSyncBundle(options?: { sourceConversationKey: String(chat.id), conversationType: chat.isGroup ? "group" : "dm", displayName: chat.displayName ?? null, - nativeConversationKey: chat.identifier, participants: chat.participants.map((participant) => ({ sourceEntityKey: `imessage:${participant.identifier}`, })), } satisfies ConversationObservationPayload, - sourceVersion: "imessage-v1", }); } @@ -278,11 +275,9 @@ export function buildIMessageSyncBundle(options?: { senderSourceKey: message.sender ? `imessage:${message.sender.identifier}` : null, sentAt: message.timestamp * 1000, content: message.text ?? "", - service: message.sender?.service ?? "iMessage", status: message.status, isFromMe: message.isFromMe, readAt: message.readAt ? message.readAt * 1000 : null, - isEdited: false, isDeleted: false, attachments: message.attachments.map((attachment) => ({ id: attachment.guid, @@ -292,18 +287,9 @@ export function buildIMessageSyncBundle(options?: { mime_type: attachment.mimeType, size_bytes: attachment.totalBytes, access_kind: attachment.filename ? "local_path" : "none", - availability_status: attachment.filename ? "available" : "metadata_only", access_ref: attachment.filename ? { path: attachment.filename } : null, - provider_metadata: { - uti: attachment.uti, - isSticker: attachment.isSticker, - hideAttachment: attachment.hideAttachment, - ckRecordId: attachment.ckRecordId, - sourceFilename: attachment.filename, - }, })), } satisfies MessagePayload, - sourceVersion: "imessage-v1", }); for (const reaction of message.reactions) { @@ -328,7 +314,6 @@ export function buildIMessageSyncBundle(options?: { timestamp: reaction.timestamp * 1000, isActive: true, } satisfies ReactionPayload, - sourceVersion: "imessage-v1", }); } } @@ -361,7 +346,6 @@ export function buildIMessageSyncBundle(options?: { ] : [], } satisfies ContactObservationPayload, - sourceVersion: "imessage-v1", }); } @@ -383,10 +367,8 @@ export function buildIMessageSyncBundle(options?: { sourceConversationKey: call.sourceConversationKey, conversationType: "dm", displayName, - service: call.provider, participants: call.remoteSourceKey ? [{ sourceEntityKey: call.remoteSourceKey }] : [], } satisfies ConversationObservationPayload, - sourceVersion: "imessage-v1", }); } @@ -405,23 +387,14 @@ export function buildIMessageSyncBundle(options?: { sourceCallKey: call.sourceCallKey, sourceConversationKey: call.sourceConversationKey, provider: call.provider, - providerCallType: call.providerCallType, direction: call.direction, medium: call.medium, status: call.status, startedAt: call.startedAt, - endedAt: call.endedAt, durationSeconds: call.durationSeconds, initiatorSourceKey: call.direction === "incoming" ? call.remoteSourceKey : null, primaryRemoteSourceKey: call.remoteSourceKey, - remoteAddress: call.remoteAddress, - remoteDisplayName: call.remoteDisplayName, - disconnectedCause: call.disconnectedCause, - metadata: { - syntheticConversation: call.syntheticConversation, - }, } satisfies CallPayload, - sourceVersion: "imessage-v1", }); } diff --git a/src/platforms/linkedin/api/conversations.ts b/src/platforms/linkedin/api/conversations.ts index fcceb7d5..3d581e04 100644 --- a/src/platforms/linkedin/api/conversations.ts +++ b/src/platforms/linkedin/api/conversations.ts @@ -182,7 +182,6 @@ function parseConversation(raw: RawConversation): Conversation { read: raw.read ?? true, messages, categories: raw.categories ?? [], - unreadCount: raw.read === false ? 1 : 0, }; } diff --git a/src/platforms/linkedin/api/types.ts b/src/platforms/linkedin/api/types.ts index b042719f..338cfc27 100644 --- a/src/platforms/linkedin/api/types.ts +++ b/src/platforms/linkedin/api/types.ts @@ -103,7 +103,6 @@ export interface Conversation { read: boolean; messages?: CollectionResponse; categories: string[]; - unreadCount?: number; } export interface SeenReceipt { diff --git a/src/platforms/linkedin/realtime/events.test.ts b/src/platforms/linkedin/realtime/events.test.ts index b4be3673..f3e1bbe0 100644 --- a/src/platforms/linkedin/realtime/events.test.ts +++ b/src/platforms/linkedin/realtime/events.test.ts @@ -56,7 +56,6 @@ describe("buildLinkedInRawEventsFromRealtimeEnvelope", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [selfParticipant, peerParticipant], }, }, @@ -110,7 +109,6 @@ describe("buildLinkedInRawEventsFromRealtimeEnvelope", () => { groupChat: true, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [selfParticipant, peerParticipant], }, }, @@ -156,7 +154,6 @@ describe("buildLinkedInRawEventsFromRealtimeEnvelope", () => { groupChat: false, read: true, categories: ["ARCHIVED"], - unreadCount: 0, conversationParticipants: [selfParticipant, peerParticipant], }, }, @@ -178,7 +175,6 @@ describe("buildLinkedInRawEventsFromRealtimeEnvelope", () => { )?.payload, ).toEqual( expect.objectContaining({ - removalReason: "archived", participants: expect.arrayContaining([ expect.objectContaining({ sourceEntityKey: "linkedin:urn:li:member:SELF123", @@ -220,7 +216,6 @@ describe("buildLinkedInRawEventsFromRealtimeEnvelope", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [selfParticipant, peerParticipant], }, }, diff --git a/src/platforms/linkedin/realtime/events.ts b/src/platforms/linkedin/realtime/events.ts index e8e6991b..903600fa 100644 --- a/src/platforms/linkedin/realtime/events.ts +++ b/src/platforms/linkedin/realtime/events.ts @@ -79,7 +79,7 @@ function userIsConversationMember(conversation: Conversation, userEntityUrn: str ); } -function removalReason(conversation: Conversation, userEntityUrn: string): string | null { +function conversationRemovalKind(conversation: Conversation, userEntityUrn: string): string | null { if (isSpamConversation(conversation)) { return "spam"; } @@ -117,7 +117,7 @@ export function buildLinkedInRawEventsFromRealtimeEnvelope(input: { if (!conversation) { return []; } - const reason = removalReason(conversation, input.userEntityUrn); + const reason = conversationRemovalKind(conversation, input.userEntityUrn); if (reason) { return buildLinkedInConversationRemovalEvents({ accountKey: input.accountKey, @@ -158,7 +158,7 @@ export function buildLinkedInRawEventsFromRealtimeEnvelope(input: { return []; } const reason = message.conversation - ? removalReason(message.conversation, input.userEntityUrn) + ? conversationRemovalKind(message.conversation, input.userEntityUrn) : null; if (reason) { return buildLinkedInConversationRemovalEvents({ @@ -206,7 +206,7 @@ export function buildLinkedInRawEventsFromRealtimeEnvelope(input: { return []; } const reason = reaction.message.conversation - ? removalReason(reaction.message.conversation, input.userEntityUrn) + ? conversationRemovalKind(reaction.message.conversation, input.userEntityUrn) : null; if (reason) { return buildLinkedInConversationRemovalEvents({ @@ -255,7 +255,7 @@ export function buildLinkedInRawEventsFromRealtimeEnvelope(input: { return []; } const reason = receipt.message.conversation - ? removalReason(receipt.message.conversation, input.userEntityUrn) + ? conversationRemovalKind(receipt.message.conversation, input.userEntityUrn) : null; if (reason) { return buildLinkedInConversationRemovalEvents({ diff --git a/src/platforms/linkedin/sync/bundle.test.ts b/src/platforms/linkedin/sync/bundle.test.ts index b6410196..9073e922 100644 --- a/src/platforms/linkedin/sync/bundle.test.ts +++ b/src/platforms/linkedin/sync/bundle.test.ts @@ -37,7 +37,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -139,7 +138,6 @@ describe("buildLinkedInSyncBundle", () => { sourceMessageKey: "linkedin:urn:li:fsd_message:MSG123", senderSourceKey: "linkedin:urn:li:member:ACoAAA1", content: "Let’s catch up next week.", - service: "linkedin", isFromMe: false, }), ); @@ -156,7 +154,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -201,7 +198,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -272,7 +268,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -321,7 +316,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -356,7 +350,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -429,7 +422,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -457,7 +449,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["SPAM"], - unreadCount: 0, conversationParticipants: [], }, ], @@ -568,15 +559,6 @@ describe("buildLinkedInSyncBundle", () => { (event) => event.entityKind === "conversation" && event.eventKind === "removed", ), ).toBe(true); - expect( - bundle.rawEvents.find( - (event) => event.entityKind === "conversation" && event.eventKind === "removed", - )?.payload, - ).toEqual( - expect.objectContaining({ - removalReason: "deleted", - }), - ); expect( bundle.rawEvents.some( (event) => event.entityKind === "timeline_event" && event.eventKind === "system_message", @@ -589,8 +571,6 @@ describe("buildLinkedInSyncBundle", () => { ); expect(messageEvent?.payload).toEqual( expect.objectContaining({ - replyToSourceMessageKey: "linkedin:urn:li:fsd_message:MSG_PARENT", - isEdited: true, attachments: [ expect.objectContaining({ kind: "file", @@ -651,7 +631,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", @@ -723,7 +702,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["INBOX", "PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:msg_messagingParticipant:urn:li:fsd_profile:SELF123", @@ -835,7 +813,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["INBOX", "PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:msg_messagingParticipant:urn:li:fsd_profile:SELF123", @@ -916,7 +893,6 @@ describe("buildLinkedInSyncBundle", () => { groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF123", diff --git a/src/platforms/linkedin/sync/bundle.ts b/src/platforms/linkedin/sync/bundle.ts index 6ade2321..c29e5255 100644 --- a/src/platforms/linkedin/sync/bundle.ts +++ b/src/platforms/linkedin/sync/bundle.ts @@ -319,7 +319,6 @@ function buildLinkedInConnectionContactEvent( company: connection.headline ?? null, photo_url: connection.picture?.url ?? null, }, - sourceProfileUrl: connection.profileUrl ?? null, handles: [ { type: "linkedin_profile_id", @@ -337,7 +336,6 @@ function buildLinkedInConnectionContactEvent( : []), ], }, - sourceVersion: "linkedin-v1", }; } @@ -667,12 +665,6 @@ function conversationScope(conversation: Conversation): SyncProofInput["scope"] return { kind: "conversation", key: normalizedConversation, - displayName: conversation.title || null, - metadata: { - sourceConversationKey: conversationSourceKey(normalizedConversation), - groupChat: conversation.groupChat, - categories: conversation.categories, - }, }; } @@ -708,8 +700,6 @@ function buildLinkedInDiscoveryProof(input: { accountKey: string; accountDisplayName: string; observedAt: number; - runStartedAt: number; - syncMode: "full" | "incremental"; complete: boolean; resumeScan: LinkedInFullScanCursor | null; conversationCount: number; @@ -719,14 +709,10 @@ function buildLinkedInDiscoveryProof(input: { scope: { kind: "account", key: input.accountKey, - displayName: input.accountDisplayName, }, proofKind: "discovery", status: input.complete ? "complete" : "running", - syncMode: input.syncMode, observedAt: input.observedAt, - runStartedAt: input.runStartedAt, - completedAt: input.complete ? input.observedAt : null, resumeCursor: input.complete ? null : input.resumeScan, stats: { conversationCount: input.conversationCount, @@ -744,8 +730,6 @@ function buildLinkedInMessagesProof(input: { }; messageCount: number; observedAt: number; - runStartedAt: number; - syncMode: "full" | "incremental"; complete: boolean; resumeCursor: LinkedInMessageResumeCursor | null; error?: unknown; @@ -754,16 +738,12 @@ function buildLinkedInMessagesProof(input: { scope: conversationScope(input.conversation), proofKind: "messages", status: input.error ? "failed" : input.complete ? "complete" : "running", - syncMode: input.syncMode, observedAt: input.observedAt, - runStartedAt: input.runStartedAt, - completedAt: input.complete && !input.error ? input.observedAt : null, resumeCursor: input.complete || input.error ? null : input.resumeCursor, coverage: input.coverage, stats: { messageCount: input.messageCount, }, - error: input.error, }; } @@ -1028,8 +1008,6 @@ export async function buildLinkedInSyncBundle(options?: { coverage: messageResult.coverage, messageCount: messageResult.messageCount, observedAt: observedBase, - runStartedAt: scan?.startedAt ?? observedBase, - syncMode, complete: messageResult.complete, resumeCursor: messageResult.resumeCursor, error: messageResult.error, @@ -1102,8 +1080,6 @@ export async function buildLinkedInSyncBundle(options?: { accountKey, accountDisplayName, observedAt: observedBase, - runStartedAt: scan?.startedAt ?? observedBase, - syncMode, complete: discoveryComplete, resumeScan: discoveryComplete ? null : (conversationResult.resumeScan ?? null), conversationCount: conversationResult.conversations.length, diff --git a/src/platforms/linkedin/sync/events.ts b/src/platforms/linkedin/sync/events.ts index aa616091..b3ab8d9b 100644 --- a/src/platforms/linkedin/sync/events.ts +++ b/src/platforms/linkedin/sync/events.ts @@ -129,12 +129,8 @@ export function buildParticipantContactEvent( company: participant.participantType.member?.headline ?? null, photo_url: bestParticipantPhoto(participant), }, - sourceProfileUrl: - participant.participantType.member?.profileUrl ?? - linkedinProfileUrlFromUrn(participant.entityURN), handles: participantHandles(participant), }, - sourceVersion: "linkedin-v1", }; } @@ -170,16 +166,6 @@ function largestVectorImageUrl(value: unknown): string | null { return `${image.rootUrl}${artifact.fileIdentifyingUrlPathSegment}`; } -export function replyToSourceMessageKey(message: Message): string | null { - for (const item of message.renderContent ?? []) { - const originalUrn = item.repliedMessageContent?.originalMessage?.entityUrn; - if (typeof originalUrn === "string" && originalUrn.length > 0) { - return `linkedin:${originalUrn}`; - } - } - return null; -} - export function normalizeLinkedInAttachments(message: Message): Array> { const attachments: Array> = []; for (const [index, item] of (message.renderContent ?? []).entries()) { @@ -197,7 +183,6 @@ export function normalizeLinkedInAttachments(message: Message): Array typeof location.url === "string")?.url ?? null, mime_type: firstStream?.mediaType ?? null, size_bytes: firstStream?.size ?? null, - preview_url: largestVectorImageUrl(video.thumbnail ?? null), - metadata: video, }); continue; } @@ -243,7 +223,6 @@ export function normalizeLinkedInAttachments(message: Message): Array | undefined)?.url === "string" - ? (media.previewMedia as Record).url - : null, - metadata: media, }); continue; } @@ -268,7 +242,6 @@ export function normalizeLinkedInAttachments(message: Message): Array ({ sourceEntityKey: participantSourceKey(participant), isSelf: normalizeMemberUrn(participant.entityURN) === userEntityUrn, })), }, - sourceVersion: "linkedin-v1", }; } @@ -362,10 +327,6 @@ export function buildLinkedInConversationRemovalEvents(input: { sourceConversationKey, conversationType: input.conversation?.groupChat ? "group" : "dm", displayName: input.conversation?.title || null, - nativeConversationKey: normalizedConversation, - service: "linkedin", - unreadCount: 0, - removalReason: input.reason, participants: input.conversation?.conversationParticipants.map((participant) => ({ sourceEntityKey: participantSourceKey(participant), @@ -374,7 +335,6 @@ export function buildLinkedInConversationRemovalEvents(input: { : undefined, })) ?? [], } satisfies ConversationObservationPayload, - sourceVersion: "linkedin-v1", }, ]; } @@ -406,13 +366,8 @@ export function buildLinkedInSystemTimelineEvent( actorSourceKey: sender, eventAt: message.deliveredAt, text: message.body.text || null, - metadata: { - systemKind: "provider_notice", - renderFormat: message.messageBodyRenderFormat, - renderContent: message.renderContent ?? [], - }, + systemKind: "provider_notice", }, - sourceVersion: "linkedin-v1", }; } @@ -430,7 +385,6 @@ export function buildLinkedInMessageEvent(input: { const normalizedConversation = normalizeConversationUrn(conversationUrn); const senderUrn = normalizeMemberUrn(input.message.sender.entityURN); const isDeleted = isLinkedInMessageDeleted(input.message); - const isEdited = isLinkedInMessageEdited(input.message); const id = stableId( `linkedin:message:${input.accountKey}:${normalizedConversation}:${input.message.entityURN}:${input.message.body.text}:${input.message.messageBodyRenderFormat}:${input.readAt ?? ""}`, ); @@ -451,19 +405,13 @@ export function buildLinkedInMessageEvent(input: { senderSourceKey: senderUrn === input.userEntityUrn ? null : `linkedin:${senderUrn}`, sentAt: input.message.deliveredAt, content: isDeleted ? "" : input.message.body.text, - service: "linkedin", status: input.status ?? (input.readAt ? "read" : "delivered"), isFromMe: senderUrn === input.userEntityUrn, deliveredAt: input.message.deliveredAt, readAt: input.readAt ?? null, - editedAt: isEdited ? input.message.deliveredAt : null, - deletedAt: isDeleted ? input.message.deliveredAt : null, - replyToSourceMessageKey: replyToSourceMessageKey(input.message), - isEdited, isDeleted, attachments: normalizeLinkedInAttachments(input.message), }, - sourceVersion: "linkedin-v1", }; } @@ -503,7 +451,6 @@ export function buildLinkedInReactionEvent(input: { timestamp: transitionTime, isActive: input.isActive, }, - sourceVersion: "linkedin-v1", }; } @@ -538,13 +485,8 @@ export function buildLinkedInGroupReceiptTimelineEvent(input: { actorSourceKey, eventAt: input.receipt.seenAt, text: `${bestParticipantName(input.receipt.seenByParticipant)} read a message.`, - metadata: { - systemKind: "read_receipt_notice", - sourceMessageKey: messageSourceKey(input.receipt.message.entityURN), - seenAt: input.receipt.seenAt, - }, + systemKind: "read_receipt_notice", }, - sourceVersion: "linkedin-v1", }; } diff --git a/src/platforms/signal/cli/client.ts b/src/platforms/signal/cli/client.ts index 753e5521..707b5c00 100644 --- a/src/platforms/signal/cli/client.ts +++ b/src/platforms/signal/cli/client.ts @@ -57,10 +57,6 @@ export interface SignalReceivedMessage { attachments: Array>; } -export interface SignalSendResult { - timestamp: number; -} - export interface SignalLinkHandle { child: ChildProcess; provisioningUri: Promise; @@ -510,35 +506,6 @@ export class SignalCliClient { .filter((value): value is SignalReceivedMessage => value !== null); } - async sendMessage( - text: string, - target: { recipient?: string; groupId?: string }, - ): Promise { - const message = text.trim(); - if (message.length === 0) { - throw new Error("Signal message text is required"); - } - - const args = this.accountArgs(["send", "-m", message]); - if (target.groupId) { - args.push("-g", target.groupId); - } else if (target.recipient) { - args.push(target.recipient); - } else { - throw new Error("Signal message requires a recipient or groupId"); - } - - const { stdout } = await execFileAsync(this.cliPath, args, { - timeout: 20_000, - maxBuffer: 2 * 1024 * 1024, - }); - - const timestampMatch = stdout.match(/\d{10,16}/); - return { - timestamp: timestampMatch ? Number(timestampMatch[0]) : Date.now(), - }; - } - accountArgs(args: string[]): string[] { return ["--config", this.configDir, "-u", this.account, ...args]; } diff --git a/src/platforms/signal/cli/link.ts b/src/platforms/signal/cli/link.ts deleted file mode 100644 index fc6c4069..00000000 --- a/src/platforms/signal/cli/link.ts +++ /dev/null @@ -1,2 +0,0 @@ -export type { SignalLinkHandle } from "./client.js"; -export { startSignalLinkSession } from "./client.js"; diff --git a/src/platforms/signal/cli/types.ts b/src/platforms/signal/cli/types.ts deleted file mode 100644 index 89ad71d4..00000000 --- a/src/platforms/signal/cli/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type { - SignalCliVersion, - SignalContact, - SignalGroup, - SignalLinkHandle, - SignalReceivedMessage, - SignalSendResult, -} from "./client.js"; diff --git a/src/platforms/signal/realtime/session.test.ts b/src/platforms/signal/realtime/session.test.ts index 2b5f4876..0b44b795 100644 --- a/src/platforms/signal/realtime/session.test.ts +++ b/src/platforms/signal/realtime/session.test.ts @@ -87,27 +87,6 @@ describe("signal realtime", () => { }), ); - const sendPromise = session.sendMessage("Ping", { recipient: "+14155550123" }); - expect(child.stdinWrites).toHaveLength(1); - expect(JSON.parse(child.stdinWrites[0]!.trim())).toEqual({ - jsonrpc: "2.0", - id: 1, - method: "send", - params: { - message: "Ping", - recipient: ["+14155550123"], - }, - }); - - child.stdout.write( - `${JSON.stringify({ - jsonrpc: "2.0", - id: 1, - result: { timestamp: 123 }, - })}\n`, - ); - - await expect(sendPromise).resolves.toEqual({ timestamp: 123 }); expect(parseSignalJsonRpcLine('{"jsonrpc":"2.0","method":"receive"}')).toEqual({ jsonrpc: "2.0", method: "receive", @@ -247,9 +226,6 @@ describe("signal realtime", () => { isConnected() { return true; }, - async sendMessage() { - return { timestamp: 1 }; - }, }), }); diff --git a/src/platforms/signal/realtime/session.ts b/src/platforms/signal/realtime/session.ts index 60d77579..0a53ef9b 100644 --- a/src/platforms/signal/realtime/session.ts +++ b/src/platforms/signal/realtime/session.ts @@ -65,10 +65,6 @@ export interface SignalRealtimeSessionLike { stopAndWait(): Promise; getStatus(): SignalRealtimeStatus; isConnected(): boolean; - sendMessage( - text: string, - target: { recipient?: string; groupId?: string }, - ): Promise<{ timestamp: number }>; } export interface SignalRealtimeSupervisorSessionInput { @@ -220,30 +216,6 @@ export class SignalRealtimeSession implements SignalRealtimeSessionLike { return this.status.state === "connected" && Boolean(this.child?.stdin?.writable); } - async sendMessage( - text: string, - target: { recipient?: string; groupId?: string }, - ): Promise<{ timestamp: number }> { - const message = text.trim(); - if (message.length === 0) { - throw new Error("Signal message text is required"); - } - - const params: Record = { message }; - if (target.groupId) { - params.groupId = target.groupId; - } else if (target.recipient) { - params.recipient = [target.recipient]; - } else { - throw new Error("Signal message requires a recipient or groupId"); - } - - const result = await this.request<{ timestamp?: number }>("send", params); - return { - timestamp: typeof result?.timestamp === "number" ? result.timestamp : now(), - }; - } - async request(method: string, params: Record): Promise { if (!this.child?.stdin?.writable || !this.isConnected()) { throw new Error("Signal realtime session is not connected"); diff --git a/src/platforms/signal/sync/bundle.test.ts b/src/platforms/signal/sync/bundle.test.ts index d9f33bef..df11b709 100644 --- a/src/platforms/signal/sync/bundle.test.ts +++ b/src/platforms/signal/sync/bundle.test.ts @@ -58,7 +58,6 @@ describe("signal worker lib", () => { sourceConversationKey: "signal:dm:+14155550123", senderSourceKey: "signal:+14155550123", content: "Hello from Signal", - service: "signal", isFromMe: false, }), ); diff --git a/src/platforms/signal/sync/events.ts b/src/platforms/signal/sync/events.ts index b4eb6eeb..f7e51f7f 100644 --- a/src/platforms/signal/sync/events.ts +++ b/src/platforms/signal/sync/events.ts @@ -71,9 +71,7 @@ function normalizeSignalAttachments( size_bytes: size, local_path: localPath, access_kind: localPath ? "local_path" : "none", - availability_status: localPath ? "available" : "unsupported_pending_signal_fetch", access_ref: localPath ? { path: localPath } : null, - provider_metadata: { ...attachment }, }; }); } @@ -123,7 +121,6 @@ export function buildSignalContactEvent( }, handles: contactHandles(contact), } satisfies ContactObservationPayload, - sourceVersion: "signal-v1", }; } @@ -150,14 +147,11 @@ export function buildSignalGroupConversationEvent( sourceConversationKey: signalSourceConversationKey(`group:${groupId}`), conversationType: "group", displayName: group.name?.trim() || group.title?.trim() || "Signal Group", - nativeConversationKey: groupId, - service: "signal", participants: (group.members ?? []) .map((member) => member.number?.trim() || member.uuid?.trim()?.toLowerCase() || "") .filter((member): member is string => member.length > 0) .map((member) => ({ sourceEntityKey: signalSourceEntityKey(member) })), } satisfies ConversationObservationPayload, - sourceVersion: "signal-v1", }; } @@ -186,11 +180,8 @@ export function buildSignalMessageConversationEvent( sourceConversationKey: signalSourceConversationKey(message.threadId), conversationType: message.threadType, displayName: message.threadName ?? message.peerHandle ?? null, - nativeConversationKey: message.threadId, - service: "signal", participants, } satisfies ConversationObservationPayload, - sourceVersion: "signal-v1", }; } @@ -222,7 +213,6 @@ export function buildSyntheticSignalContactEvent( }, ], } satisfies ContactObservationPayload, - sourceVersion: "signal-v1", }; } @@ -251,13 +241,11 @@ export function buildSignalMessageEvent( senderSourceKey: senderHandle ? signalSourceEntityKey(senderHandle) : null, sentAt: message.sentAt, content: message.text, - service: "signal", status: message.isFromMe ? "sent" : "delivered", isFromMe: message.isFromMe, deliveredAt: message.sentAt, attachments: normalizeSignalAttachments(message.attachments), } satisfies MessagePayload, - sourceVersion: "signal-v1", }; } diff --git a/src/platforms/slack/e2e.test.ts b/src/platforms/slack/e2e.test.ts index 85f03bbc..6fae461d 100644 --- a/src/platforms/slack/e2e.test.ts +++ b/src/platforms/slack/e2e.test.ts @@ -479,19 +479,25 @@ echo '{"token":"xoxc-test","cookie":"cookie-test","teamId":"T123","teamName":"Ac { source_conversation_key: "slack:T123:G_TEAM" }, ]); - const firstMessages = db.orm().all<{ content: string | null; reaction_count: number }>(sql` - SELECT content, reaction_count + const firstMessages = db.orm().all<{ content: string | null }>(sql` + SELECT content FROM messages ORDER BY content ASC `); expect(firstMessages).toEqual([ - { content: "Hi Ava", reaction_count: 0 }, - { content: "Kickoff notes", reaction_count: 0 }, - { content: "Older context", reaction_count: 0 }, - { content: "Reply one", reaction_count: 0 }, - { content: "Reply two", reaction_count: 0 }, - { content: "Weekly sync", reaction_count: 1 }, + { content: "Hi Ava" }, + { content: "Kickoff notes" }, + { content: "Older context" }, + { content: "Reply one" }, + { content: "Reply two" }, + { content: "Weekly sync" }, ]); + expect( + db.orm().get<{ count: number }>(sql` + SELECT COUNT(*) AS count + FROM message_reactions + `), + ).toEqual({ count: 1 }); const attachmentRows = db.orm().all<{ source_attachment_key: string }>(sql` SELECT source_attachment_key diff --git a/src/platforms/slack/sync/bundle.test.ts b/src/platforms/slack/sync/bundle.test.ts index 26e47f36..0833a05e 100644 --- a/src/platforms/slack/sync/bundle.test.ts +++ b/src/platforms/slack/sync/bundle.test.ts @@ -116,18 +116,9 @@ describe("slack worker lib", () => { senderSourceKey: "slack:T123:U_BEN", content: "Hi from Slack", isFromMe: false, - service: "slack", attachments: expect.any(Array), }), ); - expect( - bundle.rawEvents.some( - (event) => - event.entityKind === "message" && - (event.payload as Record).replyToSourceMessageKey === - "slack:T123:D123:1710000000.000100", - ), - ).toBe(true); expect(listedTypes).toEqual(["im,mpim"]); expect(historyOldestValues).toEqual([""]); expect(repliesOldestValues).toEqual([""]); @@ -508,12 +499,6 @@ describe("slack worker lib", () => { expect.objectContaining({ proofKind: "messages", status: "blocked", - error: { - code: "not_in_channel", - message: "not_in_channel", - retryable: false, - kind: "slack_api", - }, }), ]); expect(bundle.rawEvents.some((event) => event.entityKind === "conversation")).toBe(true); @@ -595,12 +580,6 @@ describe("slack worker lib", () => { completedThreadCount: 0, pendingThreadCount: 1, }), - error: { - code: "missing_scope", - message: "missing_scope", - retryable: false, - kind: "slack_api", - }, }), ]); }); @@ -704,12 +683,6 @@ describe("slack worker lib", () => { completedThreadCount: 1, pendingThreadCount: 0, }), - error: { - code: "not_in_channel", - message: "not_in_channel", - retryable: false, - kind: "slack_api", - }, }), ]); }); diff --git a/src/platforms/slack/sync/bundle.ts b/src/platforms/slack/sync/bundle.ts index d45d36ce..22ccada7 100644 --- a/src/platforms/slack/sync/bundle.ts +++ b/src/platforms/slack/sync/bundle.ts @@ -731,25 +731,15 @@ function summarizeMessageRange(messages: SlackMessage[]): { } function buildSlackBackfillConversationProof(input: { - teamId: string; - accountKey: string; conversation: SlackConversation; - family: SlackConversationFamily; - scan: SlackScanCursor; knownConversationCount: number; observedAt: number; messageBatch: SlackConversationBatchResult; }): SlackBackfillConversationProof { const range = summarizeMessageRange(input.messageBatch.messages); return { - teamId: input.teamId, - accountKey: input.accountKey, - syncMode: input.scan.mode, - scanStartedAt: input.scan.startedAt, knownConversationCount: input.knownConversationCount, conversationId: input.conversation.id, - conversationName: input.conversation.name, - conversationFamily: input.family, conversationPhase: input.messageBatch.resumeState.conversationPhase, historyComplete: input.messageBatch.resumeState.historyComplete, historyCursor: input.messageBatch.resumeState.historyCursor, @@ -770,12 +760,7 @@ function buildSlackBackfillConversationProof(input: { } function buildCompleteSlackBackfillConversationProof(input: { - teamId: string; - accountKey: string; conversation: SlackConversation; - family: SlackConversationFamily; - scanMode: SlackScanMode; - scanStartedAt: number; knownConversationCount: number; observedAt: number; messages: SlackMessage[]; @@ -783,7 +768,6 @@ function buildCompleteSlackBackfillConversationProof(input: { repliesError?: SlackBackfillProofError | null; threadRootCount?: number; completedThreadCount?: number; - membersError?: SlackBackfillProofError | null; }): SlackBackfillConversationProof { const range = summarizeMessageRange(input.messages); const threadRootCount = @@ -791,14 +775,8 @@ function buildCompleteSlackBackfillConversationProof(input: { input.messages.filter((message) => Number(message.reply_count ?? 0) > 0).length; const completedThreadCount = input.completedThreadCount ?? threadRootCount; return { - teamId: input.teamId, - accountKey: input.accountKey, - syncMode: input.scanMode, - scanStartedAt: input.scanStartedAt, knownConversationCount: input.knownConversationCount, conversationId: input.conversation.id, - conversationName: input.conversation.name, - conversationFamily: input.family, conversationPhase: "complete", historyComplete: true, historyCursor: null, @@ -812,7 +790,6 @@ function buildCompleteSlackBackfillConversationProof(input: { observedAt: input.observedAt, historyError: input.historyError ?? null, repliesError: input.repliesError ?? null, - membersError: input.membersError ?? null, }; } @@ -939,10 +916,7 @@ export async function buildSlackSyncBundle(options?: { conversation, observedBase, ); - const { memberIds, error: membersError } = await listConversationMembers( - client, - conversation, - ); + const { memberIds } = await listConversationMembers(client, conversation); const messageBatch = await listConversationMessagesBatch( client, conversation.id, @@ -965,20 +939,12 @@ export async function buildSlackSyncBundle(options?: { knownConversationIds.add(conversation.id); slackConversationProofs.push( buildSlackBackfillConversationProof({ - teamId, - accountKey, conversation, - family, - scan, knownConversationCount: knownConversationIds.size, observedAt: observedBase, messageBatch, }), ); - const proof = slackConversationProofs.at(-1); - if (proof) { - proof.membersError = membersError; - } rawEvents.push( buildSlackConversationEvent({ teamId, @@ -1142,10 +1108,7 @@ export async function buildSlackSyncBundle(options?: { conversation, observedBase, ); - const { memberIds, error: membersError } = await listConversationMembers( - client, - conversation, - ); + const { memberIds } = await listConversationMembers(client, conversation); const messageBatch = await listConversationMessages( client, conversation.id, @@ -1179,12 +1142,7 @@ export async function buildSlackSyncBundle(options?: { slackConversationProofs.push( buildCompleteSlackBackfillConversationProof({ - teamId, - accountKey, conversation, - family, - scanMode: scan.mode, - scanStartedAt: scan.startedAt, knownConversationCount: knownConversationIds.size, observedAt: observedBase, messages: messageBatch.messages, @@ -1192,7 +1150,6 @@ export async function buildSlackSyncBundle(options?: { repliesError: messageBatch.repliesError, threadRootCount: messageBatch.threadRootCount, completedThreadCount: messageBatch.completedThreadCount, - membersError, }), ); } diff --git a/src/platforms/slack/sync/events.ts b/src/platforms/slack/sync/events.ts index 182e79e6..7dce2671 100644 --- a/src/platforms/slack/sync/events.ts +++ b/src/platforms/slack/sync/events.ts @@ -47,14 +47,6 @@ function toAttachmentMetadata(message: SlackMessage): Array ({ sourceEntityKey: slackSourceKey(input.teamId, memberId), isSelf: memberId === input.selfUserId, })), } satisfies ConversationObservationPayload, - sourceVersion: "slack-v1", }; } @@ -246,19 +222,11 @@ export function buildSlackMessageEvents(input: { ) .filter(Boolean) .join("\n"), - service: "slack", status: null, isFromMe: senderUserId === input.selfUserId, - editedAt: slackTimestampMs(message.edited?.ts), - isEdited: Boolean(message.edited?.ts), isDeleted: false, - replyToSourceMessageKey: - message.thread_ts && message.thread_ts !== message.ts - ? slackMessageKey(input.teamId, input.conversationId, message.thread_ts) - : null, attachments, } satisfies MessagePayload, - sourceVersion: "slack-v1", }); for (const reaction of message.reactions ?? []) { @@ -288,7 +256,6 @@ export function buildSlackMessageEvents(input: { timestamp: messageTsMs, isActive: true, } satisfies ReactionPayload, - sourceVersion: "slack-v1", }); } } diff --git a/src/platforms/slack/sync/proof.ts b/src/platforms/slack/sync/proof.ts index ee02726f..42a79dd7 100644 --- a/src/platforms/slack/sync/proof.ts +++ b/src/platforms/slack/sync/proof.ts @@ -10,14 +10,8 @@ export interface SlackBackfillProofError { } export interface SlackBackfillConversationProof { - teamId: string; - accountKey: string; - syncMode: "full" | "incremental"; - scanStartedAt: number; knownConversationCount: number; conversationId: string; - conversationName?: string; - conversationFamily: "direct" | "channels"; conversationPhase: SlackBackfillConversationPhase; historyComplete: boolean; historyCursor: string | null; @@ -32,7 +26,6 @@ export interface SlackBackfillConversationProof { observedAt: number; historyError?: SlackBackfillProofError | null; repliesError?: SlackBackfillProofError | null; - membersError?: SlackBackfillProofError | null; } export function buildSlackBackfillSyncProofs( @@ -41,11 +34,6 @@ export function buildSlackBackfillSyncProofs( const scope = { kind: "conversation" as const, key: proof.conversationId, - displayName: proof.conversationName ?? null, - metadata: { - teamId: proof.teamId, - conversationFamily: proof.conversationFamily, - }, }; const messageStatus = getMessagesProofStatus(proof); const proofs: SyncProofInput[] = [ @@ -53,11 +41,7 @@ export function buildSlackBackfillSyncProofs( scope, proofKind: "messages", status: messageStatus, - syncMode: proof.syncMode, observedAt: proof.observedAt, - runStartedAt: proof.scanStartedAt, - completedAt: - messageStatus === "complete" || messageStatus === "partial" ? proof.observedAt : null, resumeCursor: messageStatus === "complete" || proof.historyError ? null @@ -70,7 +54,6 @@ export function buildSlackBackfillSyncProofs( knownConversationCount: proof.knownConversationCount, ...(proof.threadRootCount > 0 ? { threadRootCount: proof.threadRootCount } : {}), }, - error: proof.historyError ?? proof.membersError ?? undefined, }, ]; @@ -88,11 +71,7 @@ export function buildSlackBackfillSyncProofs( scope, proofKind: "replies", status: repliesStatus, - syncMode: proof.syncMode, observedAt: proof.observedAt, - runStartedAt: proof.scanStartedAt, - completedAt: - repliesStatus === "complete" || repliesStatus === "partial" ? proof.observedAt : null, resumeCursor: repliesStatus === "complete" || proof.repliesError || proof.historyError ? null @@ -112,7 +91,6 @@ export function buildSlackBackfillSyncProofs( completedThreadCount: proof.completedThreadCount, pendingThreadCount: proof.pendingThreadCount, }, - error: proof.repliesError ?? proof.historyError ?? undefined, }); } diff --git a/src/platforms/whatsapp/desktop.test.ts b/src/platforms/whatsapp/desktop.test.ts index 363c270b..de1729ed 100644 --- a/src/platforms/whatsapp/desktop.test.ts +++ b/src/platforms/whatsapp/desktop.test.ts @@ -236,19 +236,15 @@ describe("whatsapp desktop import", () => { ]); expect(bundle.rawEvents.find((event) => event.entityKind === "message")).toMatchObject({ platform: "whatsapp", - sourceVersion: "whatsapp-v1", provenance: { acquisitionMode: "sync", - adapterVersion: "whatsapp-desktop-db", }, }); expect(bundle.proofs?.[0]).toEqual( expect.objectContaining({ proofKind: "messages", status: "complete", - scope: expect.objectContaining({ - metadata: { source: "desktop_db" }, - }), + scope: { kind: "account", key: "default" }, coverage: expect.objectContaining({ source: "desktop_db", newestMessageAt: 1_700_000_040_000, diff --git a/src/platforms/whatsapp/desktop.ts b/src/platforms/whatsapp/desktop.ts index 7e766520..65457405 100644 --- a/src/platforms/whatsapp/desktop.ts +++ b/src/platforms/whatsapp/desktop.ts @@ -19,8 +19,6 @@ const APPLE_EPOCH_SECONDS = 978_307_200; type DesktopChatRow = { jid: string; name: string; - last_message_date: number | null; - unread_count: number; raw_session_type: number; }; @@ -154,7 +152,6 @@ export function buildWhatsAppDesktopSyncBundle( provenance: { ...event.provenance, acquisitionMode: "sync" as const, - adapterVersion: "whatsapp-desktop-db", }, })); @@ -172,17 +169,10 @@ export function buildWhatsAppDesktopSyncBundle( scope: { kind: "account", key: accountKey, - displayName: "WhatsApp Desktop", - metadata: { - source: "desktop_db", - }, }, proofKind: "messages", status: "complete", - syncMode: "full", observedAt: observedBase, - runStartedAt: observedBase, - completedAt, coverage: { source: "desktop_db", oldestMessageAt: inspected.oldestMessageAt, @@ -304,8 +294,6 @@ function readChats( SELECT COALESCE(ZCONTACTJID, '') AS jid, COALESCE(ZPARTNERNAME, '') AS name, - ZLASTMESSAGEDATE AS last_message_date, - COALESCE(ZUNREADCOUNT, 0) AS unread_count, COALESCE(ZSESSIONTYPE, 0) AS raw_session_type FROM ZWACHATSESSION `) @@ -417,11 +405,6 @@ function readMessages( local_path: mediaPath, remote_url: row.media_url || null, size_bytes: row.media_size || null, - availability_status: mediaPath ? "available" : "metadata_only", - provider_metadata: { - source: "desktop_db", - rawType: row.raw_type, - }, }, ] : [], diff --git a/src/platforms/whatsapp/realtime/session.test.ts b/src/platforms/whatsapp/realtime/session.test.ts index 72bed2a7..971aae89 100644 --- a/src/platforms/whatsapp/realtime/session.test.ts +++ b/src/platforms/whatsapp/realtime/session.test.ts @@ -91,31 +91,6 @@ describe("whatsapp realtime", () => { }), ); - const sendPromise = session.sendText("12016824050@s.whatsapp.net", "ping"); - expect(JSON.parse(child.stdinWrites[0]!.trim())).toEqual({ - id: 1, - command: "sendText", - target: "12016824050@s.whatsapp.net", - text: "ping", - }); - - child.stdout.write( - `${JSON.stringify({ - id: 1, - ok: true, - result: { - messageID: "wamid-2", - chatJID: "12016824050@s.whatsapp.net", - timestamp: 123, - }, - })}\n`, - ); - - await expect(sendPromise).resolves.toEqual({ - messageID: "wamid-2", - chatJID: "12016824050@s.whatsapp.net", - timestamp: 123, - }); expect(parseWhatsAppHelperLine('{"event":"connected","data":{"accountJid":"x"}}')).toEqual({ event: "connected", data: { @@ -142,10 +117,6 @@ describe("whatsapp realtime", () => { session.start(); child.emit("spawn"); expect(session.getStatus().state).toBe("connecting"); - await expect(session.sendText("12016824050@s.whatsapp.net", "ping")).rejects.toThrowError( - "WhatsApp realtime session is not connected", - ); - child.stdout.write( `${JSON.stringify({ event: "connected", @@ -299,13 +270,6 @@ describe("whatsapp realtime", () => { isConnected() { return true; }, - async sendText() { - return { - messageID: "wamid-1", - chatJID: "12016824050@s.whatsapp.net", - timestamp: 1, - }; - }, async downloadMedia() { return { dataBase64: Buffer.from("hello").toString("base64"), diff --git a/src/platforms/whatsapp/realtime/session.ts b/src/platforms/whatsapp/realtime/session.ts index c2ab3f51..5dd7cc47 100644 --- a/src/platforms/whatsapp/realtime/session.ts +++ b/src/platforms/whatsapp/realtime/session.ts @@ -6,7 +6,6 @@ import type { WhatsAppHelperDownloadResult, WhatsAppHelperEventEnvelope, WhatsAppHelperResponseEnvelope, - WhatsAppHelperSendResult, WhatsAppHistoryBackfillAnchor, WhatsAppHistoryBackfillResult, WhatsAppResyncPage, @@ -67,7 +66,6 @@ export interface WhatsAppRealtimeSessionLike { stop(): void; getStatus(): WhatsAppRealtimeStatus; isConnected(): boolean; - sendText(target: string, text: string): Promise; downloadMedia( chatJID: string, messageID: string, @@ -245,22 +243,6 @@ export class WhatsAppRealtimeSession implements WhatsAppRealtimeSessionLike { return this.status.state === "connected" && Boolean(this.child?.stdin?.writable); } - async sendText(target: string, text: string): Promise { - const trimmedTarget = target.trim(); - const trimmedText = text.trim(); - if (!trimmedTarget || !trimmedText) { - throw new Error("WhatsApp send requires a target and text"); - } - - const result = await this.request({ - id: this.nextRequestId++, - command: "sendText", - target: trimmedTarget, - text: trimmedText, - }); - return result; - } - async resync( input: { cursor?: string | null; sinceMs?: number | null; limit?: number } = {}, ): Promise { diff --git a/src/platforms/whatsapp/sync/events.test.ts b/src/platforms/whatsapp/sync/events.test.ts index ebe04810..214b8f5f 100644 --- a/src/platforms/whatsapp/sync/events.test.ts +++ b/src/platforms/whatsapp/sync/events.test.ts @@ -57,7 +57,6 @@ describe("whatsapp events", () => { sourceMessageKey: "12016824050@s.whatsapp.net:wamid-1", senderSourceKey: "whatsapp:12016824050@s.whatsapp.net", content: "hello", - service: "whatsapp", }), ); expect( @@ -121,7 +120,6 @@ describe("whatsapp events", () => { durationSeconds: 61, status: "completed", medium: "audio", - providerCallType: "regular", }, ], }, @@ -138,17 +136,14 @@ describe("whatsapp events", () => { platform: "whatsapp", entityKind: "call", eventKind: "observed", - sourceVersion: "whatsapp-v1", payload: { sourceCallKey: "12016824050@s.whatsapp.net:call-1", sourceConversationKey: "whatsapp:12016824050@s.whatsapp.net", provider: "whatsapp", - providerCallType: "regular", direction: "incoming", medium: "audio", status: "completed", durationSeconds: 61, - endedAt: 1_710_000_062_000, }, }); }); @@ -176,7 +171,7 @@ describe("whatsapp events", () => { }); }); - it("maps WhatsApp reply, forward, and revoke metadata onto message payloads", () => { + it("maps WhatsApp reply and revoke state onto message payloads", () => { const rawEvents = buildWhatsAppRawEventsFromSnapshot({ accountKey: "default", snapshot: { @@ -188,10 +183,6 @@ describe("whatsapp events", () => { fromMe: false, timestamp: 1_710_000_002_000, text: "reply", - replyToMessageID: "wamid-parent", - replyToSenderJID: "15551234567@s.whatsapp.net", - isForwarded: true, - forwardingScore: 2, status: "delivered", }, { @@ -211,20 +202,11 @@ describe("whatsapp events", () => { const messageEvents = rawEvents.filter((event) => event.entityKind === "message"); expect(messageEvents[0]?.payload).toMatchObject({ sourceMessageKey: "12016824050@s.whatsapp.net:wamid-reply", - replyToSourceMessageKey: "12016824050@s.whatsapp.net:wamid-parent", isDeleted: false, - providerMetadata: { - whatsapp: { - isForwarded: true, - forwardingScore: 2, - replyToSenderJID: "15551234567@s.whatsapp.net", - }, - }, }); expect(messageEvents[1]?.payload).toMatchObject({ sourceMessageKey: "12016824050@s.whatsapp.net:wamid-revoked", isDeleted: true, - deletedAt: 1_710_000_003_000, }); expect(messageEvents[1]?.eventKind).toBe("deleted"); }); diff --git a/src/platforms/whatsapp/sync/events.ts b/src/platforms/whatsapp/sync/events.ts index 0afd36aa..2f5ffeb0 100644 --- a/src/platforms/whatsapp/sync/events.ts +++ b/src/platforms/whatsapp/sync/events.ts @@ -101,42 +101,6 @@ function normalizeWhatsAppAttachments( }); } -function whatsappReplySourceKey(message: WhatsAppMessageSnapshot): string | null { - const replyID = message.replyToMessageID?.trim(); - if (!replyID) { - return null; - } - return `${normalizeWhatsAppJid(message.chatJID)}:${replyID}`; -} - -type WhatsAppMessageProviderMetadata = { - providerMetadata?: { - whatsapp?: { - isForwarded?: boolean; - forwardingScore?: number; - replyToSenderJID?: string; - }; - }; -}; - -function whatsappMessageProviderMetadata( - message: WhatsAppMessageSnapshot, -): WhatsAppMessageProviderMetadata { - const whatsapp: NonNullable< - NonNullable["whatsapp"] - > = {}; - if (message.isForwarded) { - whatsapp.isForwarded = true; - } - if (typeof message.forwardingScore === "number") { - whatsapp.forwardingScore = message.forwardingScore; - } - if (message.replyToSenderJID?.trim()) { - whatsapp.replyToSenderJID = normalizeWhatsAppJid(message.replyToSenderJID); - } - return Object.keys(whatsapp).length > 0 ? { providerMetadata: { whatsapp } } : {}; -} - function bestContactName(contact: WhatsAppContactSnapshot): string { return ( contact.name?.trim() || @@ -184,7 +148,6 @@ export function buildWhatsAppContactEvent( }, handles, } satisfies ContactObservationPayload, - sourceVersion: "whatsapp-v1", }; } @@ -207,14 +170,11 @@ export function buildWhatsAppChatEvent( sourceConversationKey: whatsappSourceConversationKey(chatJID), conversationType: chat.isGroup ? "group" : "dm", displayName: chat.name?.trim() || null, - nativeConversationKey: chatJID, - service: "whatsapp", participants: (chat.participants ?? []) .map((jid) => normalizeWhatsAppJid(jid)) .filter((jid) => jid.length > 0) .map((jid) => ({ sourceEntityKey: whatsappSourceEntityKey(jid) })), } satisfies ConversationObservationPayload, - sourceVersion: "whatsapp-v1", }; } @@ -283,7 +243,6 @@ export function buildWhatsAppMessageEvent( : null; const normalizedChatJID = normalizeWhatsAppJid(message.chatJID); const sourceMessageKey = whatsappMessageSourceKey(message); - const replyToSourceMessageKey = whatsappReplySourceKey(message); return { id: stableId(`whatsapp:message:${accountKey}:${sourceMessageKey}`), platform: "whatsapp", @@ -301,18 +260,13 @@ export function buildWhatsAppMessageEvent( senderSourceKey: senderJID ? whatsappSourceEntityKey(senderJID) : null, sentAt: message.timestamp, content: message.text, - service: "whatsapp", status: message.status ?? (message.fromMe ? "sent" : "delivered"), isFromMe: message.fromMe, deliveredAt: message.deliveredAt ?? null, readAt: message.readAt ?? null, - deletedAt: message.revoked ? message.timestamp : null, - replyToSourceMessageKey, isDeleted: message.revoked ?? false, attachments: normalizeWhatsAppAttachments(message.attachments), - ...whatsappMessageProviderMetadata(message), - } satisfies MessagePayload & WhatsAppMessageProviderMetadata, - sourceVersion: "whatsapp-v1", + } satisfies MessagePayload, }; } @@ -365,23 +319,14 @@ export function buildWhatsAppCallEvent( sourceCallKey, sourceConversationKey: whatsappSourceConversationKey(normalizedChatJID), provider: "whatsapp", - providerCallType: call.providerCallType ?? null, direction: normalizeCallDirection(call), medium: normalizeCallMedium(call.medium), status: normalizeCallStatus(call.status), startedAt: call.timestamp, durationSeconds: call.durationSeconds ?? null, - endedAt: - typeof call.durationSeconds === "number" && Number.isFinite(call.durationSeconds) - ? call.timestamp + call.durationSeconds * 1000 - : null, initiatorSourceKey, primaryRemoteSourceKey: remoteSourceKey, - metadata: { - isFromMe: call.fromMe, - }, } satisfies CallPayload, - sourceVersion: "whatsapp-v1", }; } @@ -411,7 +356,6 @@ export function buildWhatsAppCallDeleteEvent( provider: "whatsapp", direction, }, - sourceVersion: "whatsapp-v1", }; } diff --git a/src/platforms/whatsapp/sync/proof.test.ts b/src/platforms/whatsapp/sync/proof.test.ts index 17317e82..a8a873c5 100644 --- a/src/platforms/whatsapp/sync/proof.test.ts +++ b/src/platforms/whatsapp/sync/proof.test.ts @@ -143,9 +143,7 @@ describe("WhatsApp sync proof helpers", () => { const proof = buildWhatsAppMessagesProof({ accountKey: "default", - syncMode: "full", observedAt: 500, - runStartedAt: 100, hasMore: false, nextCursor: null, sinceMs: null, @@ -161,7 +159,6 @@ describe("WhatsApp sync proof helpers", () => { }); expect(proof.status).toBe("complete"); - expect(proof.syncMode).toBe("full"); expect(proof.resumeCursor).toBeNull(); expect(proof.coverage).toMatchObject({ oldestMessageAt: 10, diff --git a/src/platforms/whatsapp/sync/proof.ts b/src/platforms/whatsapp/sync/proof.ts index 4ba1b8f2..a0346d60 100644 --- a/src/platforms/whatsapp/sync/proof.ts +++ b/src/platforms/whatsapp/sync/proof.ts @@ -202,9 +202,7 @@ export function mergeWhatsAppResyncCoverage( export function buildWhatsAppMessagesProof(input: { accountKey: string; - syncMode: SyncMode; observedAt: number; - runStartedAt: number; hasMore: boolean; nextCursor: string | null; sinceMs: number | null; @@ -216,17 +214,10 @@ export function buildWhatsAppMessagesProof(input: { scope: { kind: "account", key: input.accountKey, - displayName: "WhatsApp", - metadata: { - source: "whatsmeow_history_cache", - }, }, proofKind: "messages", status: input.hasMore ? "running" : "complete", - syncMode: input.syncMode, observedAt: input.observedAt, - runStartedAt: input.runStartedAt, - completedAt: input.hasMore ? null : input.completedAt, resumeCursor: input.hasMore ? { cursor: input.nextCursor, diff --git a/src/platforms/whatsapp/types.ts b/src/platforms/whatsapp/types.ts index b0973d33..1b4086d1 100644 --- a/src/platforms/whatsapp/types.ts +++ b/src/platforms/whatsapp/types.ts @@ -24,10 +24,6 @@ export interface WhatsAppMessageSnapshot { status?: string | null; deliveredAt?: number | null; readAt?: number | null; - replyToMessageID?: string | null; - replyToSenderJID?: string | null; - isForwarded?: boolean; - forwardingScore?: number | null; revoked?: boolean; attachments?: Array>; } @@ -51,7 +47,6 @@ export interface WhatsAppCallSnapshot { durationSeconds?: number | null; status: string; medium: string; - providerCallType?: string | null; } export interface WhatsAppCallDeleteSnapshot { @@ -138,12 +133,6 @@ export interface WhatsAppHelperEventEnvelope< } export type WhatsAppHelperCommand = - | { - id: number; - command: "sendText"; - target: string; - text: string; - } | { id: number; command: "resync"; @@ -184,12 +173,6 @@ export interface WhatsAppHelperStatusResult { lastHistoryNotificationAt?: number | null; } -export interface WhatsAppHelperSendResult { - messageID: string; - chatJID: string; - timestamp: number; -} - export interface WhatsAppHelperDownloadResult { dataBase64: string; mimeType?: string | null; diff --git a/src/runtime/attachments.test.ts b/src/runtime/attachments.test.ts index ce8e7157..3bca4552 100644 --- a/src/runtime/attachments.test.ts +++ b/src/runtime/attachments.test.ts @@ -118,10 +118,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-1", "source-conversation-1", "Thread", timestamp, timestamp); @@ -130,10 +129,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run("message-1", "platform-message-1", "conversation-1", timestamp, timestamp, timestamp); @@ -143,8 +142,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'note.txt', 'Note', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'note.txt', 'Note', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( @@ -201,10 +200,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'gmail', 'default', ?, NULL, 'dm', 1, 'Gmail', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'gmail', 'default', ?, 'dm', 1, ?, '', ?, ?) `, ) .run( @@ -219,10 +217,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'gmail', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'Gmail', NULL, 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'gmail', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, NULL, 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -239,8 +237,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'gmail', 'default', ?, 'file', 'application/vnd.google-apps.document', 'remote-doc.gdoc', 'Remote Doc', NULL, NULL, ?, NULL, 'provider_fetch', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'gmail', 'default', ?, 'file', 'application/vnd.google-apps.document', 'remote-doc.gdoc', 'Remote Doc', NULL, NULL, ?, NULL, 'provider_fetch', ?, ?, ?) `, ) .run( @@ -297,10 +295,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-variant", "source-conversation-variant", "Thread", timestamp, timestamp); @@ -309,10 +306,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -329,8 +326,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'variant.txt', 'Variant', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'variant.txt', 'Variant', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( @@ -382,10 +379,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-2", "source-conversation-2", "Thread", timestamp, timestamp); @@ -394,10 +390,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run("message-2", "platform-message-2", "conversation-2", timestamp, timestamp, timestamp); @@ -407,8 +403,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'note.txt', 'Note', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'note.txt', 'Note', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( @@ -447,10 +443,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-3", "source-conversation-3", "Thread", timestamp, timestamp); @@ -459,10 +454,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run("message-3", "platform-message-3", "conversation-3", timestamp, timestamp, timestamp); @@ -472,8 +467,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'missing.txt', 'Missing', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'missing.txt', 'Missing', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( @@ -512,10 +507,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, NULL, 'dm', 1, 'Discord', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-remote", "source-conversation-remote", "Thread", timestamp, timestamp); @@ -524,10 +518,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'Discord', NULL, 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, NULL, 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -544,8 +538,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, ?, ?) `, ) .run( @@ -577,10 +571,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, NULL, 'dm', 1, 'Discord', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-private", "source-conversation-private", "Thread", timestamp, timestamp); @@ -589,10 +582,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'Discord', NULL, 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, NULL, 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -609,8 +602,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, ?, ?) `, ) .run( @@ -638,10 +631,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, NULL, 'dm', 1, 'Discord', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, 'dm', 1, ?, '', ?, ?) `, ) .run( @@ -656,10 +648,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'Discord', NULL, 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, NULL, 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -676,8 +668,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, ?, ?) `, ) .run( @@ -706,10 +698,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, NULL, 'dm', 1, 'Discord', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, 'dm', 1, ?, '', ?, ?) `, ) .run( @@ -724,10 +715,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'Discord', NULL, 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'discord', 'default', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, NULL, 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -744,8 +735,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'discord', 'default', ?, 'file', 'text/plain', 'note.txt', 'Note', NULL, ?, NULL, NULL, 'remote_url', ?, ?, ?) `, ) .run( @@ -786,10 +777,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run("conversation-shared", "source-conversation-shared", "Thread", timestamp, timestamp); @@ -800,10 +790,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -820,8 +810,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'shared.txt', 'Shared', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'shared.txt', 'Shared', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( @@ -881,10 +871,9 @@ describe("attachment service", () => { .prepare( ` INSERT INTO conversations ( - id, platform, account_key, source_conversation_key, native_conversation_key, type, is_active, - service, name, topic, participant_names, last_message_id, last_message_at, last_message_preview, - unread_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, NULL, 'dm', 1, 'iMessage', ?, NULL, '', NULL, NULL, NULL, 0, ?, ?) + id, platform, account_key, source_conversation_key, type, is_active, + name, participant_names, created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, 'dm', 1, ?, '', ?, ?) `, ) .run( @@ -899,10 +888,10 @@ describe("attachment service", () => { ` INSERT INTO messages ( id, platform, account_key, platform_message_id, conversation_id, sender_contact_id, - sender_source_key, sender_name, conversation_name, sent_at, service, status, is_from_me, - content, delivered_at, read_at, edited_at, deleted_at, reply_to_message_id, is_deleted, - is_edited, attachment_count, reaction_count, created_at, updated_at - ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'iMessage', 'delivered', 0, 'hello', NULL, NULL, NULL, NULL, NULL, 0, 0, 1, 0, ?, ?) + sender_source_key, sender_name, conversation_name, sent_at, status, is_from_me, + content, delivered_at, read_at, is_deleted, + created_at, updated_at + ) VALUES (?, 'imessage', 'local', ?, ?, NULL, NULL, 'Ben', 'Thread', ?, 'delivered', 0, 'hello', NULL, NULL, 0, ?, ?) `, ) .run( @@ -919,8 +908,8 @@ describe("attachment service", () => { INSERT INTO message_attachments ( id, message_id, platform, account_key, source_attachment_key, kind, mime_type, filename, title, local_path, remote_url, size_bytes, text_content, access_kind, access_ref_json, - preview_ref_json, availability_status, provider_metadata_json, metadata_json, created_at, updated_at - ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'large.txt', 'Large', ?, NULL, ?, NULL, 'local_path', ?, NULL, 'available', '{}', '{}', ?, ?) + created_at, updated_at + ) VALUES (?, ?, 'imessage', 'local', ?, 'file', 'text/plain', 'large.txt', 'Large', ?, NULL, ?, NULL, 'local_path', ?, ?, ?) `, ) .run( diff --git a/src/runtime/daemon/server.test.ts b/src/runtime/daemon/server.test.ts index 192a2a98..27da2eea 100644 --- a/src/runtime/daemon/server.test.ts +++ b/src/runtime/daemon/server.test.ts @@ -9,7 +9,6 @@ import { isDisconnectedSocketError, requestWhatsAppHistoryBackfillOnce, shouldDeferContinuationProjection, - shouldDrainOutboundQueue, shouldProjectIngestRunInline, shouldSkipConnectedDiscordSchedulerSync, } from "./server.js"; @@ -51,44 +50,6 @@ describe("discord scheduler pacing", () => { }); }); -describe("outbound send gate", () => { - it("does not drain queued outbound rows unless outbound send is enabled", () => { - expect( - shouldDrainOutboundQueue({ - outboundSendEnabled: false, - isUpdateShutdownRequested: false, - activeOutboundSend: null, - }), - ).toBe(false); - - expect( - shouldDrainOutboundQueue({ - outboundSendEnabled: true, - isUpdateShutdownRequested: false, - activeOutboundSend: null, - }), - ).toBe(true); - }); - - it("does not drain while shutdown or another outbound send is active", () => { - expect( - shouldDrainOutboundQueue({ - outboundSendEnabled: true, - isUpdateShutdownRequested: true, - activeOutboundSend: null, - }), - ).toBe(false); - - expect( - shouldDrainOutboundQueue({ - outboundSendEnabled: true, - isUpdateShutdownRequested: false, - activeOutboundSend: Promise.resolve(), - }), - ).toBe(false); - }); -}); - describe("ingest projection strategy", () => { it("defers discord sync projection so run completion is not coupled to projection", () => { expect( diff --git a/src/runtime/daemon/server.ts b/src/runtime/daemon/server.ts index 28286d63..0c98688b 100644 --- a/src/runtime/daemon/server.ts +++ b/src/runtime/daemon/server.ts @@ -21,9 +21,10 @@ import { type Platform, type ProviderRawEventInput, type RawEventAcquisitionMode, + type SourceAccountInput, } from "../../core/types/provider.js"; import { safeParseJsonRecord, safeParseJsonStringArray } from "../../db/codecs.js"; -import { type CuedDatabase, type OutboundMessageRow, openCuedDatabase } from "../../db/database.js"; +import { type CuedDatabase, openCuedDatabase } from "../../db/database.js"; import { buildAdapterInvocationEnv, selectAdapterInvocationProofs, @@ -35,10 +36,7 @@ import { refreshLocalIntegrationStates } from "../../platforms/core/state/local- import { refreshManagedIntegrationStates } from "../../platforms/core/state/refresh.js"; import { getIntegrationSummary } from "../../platforms/core/state/status.js"; import type { SyncContinuation } from "../../platforms/core/sync.js"; -import { - DiscordApiClient, - isDiscordAuthInvalidationError, -} from "../../platforms/discord/api/client.js"; +import { isDiscordAuthInvalidationError } from "../../platforms/discord/api/client.js"; import { type DiscordRealtimeEventEnvelope, type DiscordRealtimeStatus, @@ -49,7 +47,6 @@ import { buildDiscordConversationEvent, buildDiscordMessageEvent, } from "../../platforms/discord/sync/events.js"; -import { isDiscordDmChannel } from "../../platforms/discord/types.js"; import { GmailClient } from "../../platforms/gmail/api/client.js"; import { DEFAULT_CALL_HISTORY_DB_PATH } from "../../platforms/imessage/call-history.js"; import { DEFAULT_CHAT_DB_PATH } from "../../platforms/imessage/reader.js"; @@ -65,7 +62,6 @@ import { isSignalCliVersionSupported, readSignalLinkedAccount, } from "../../platforms/signal/cli/binary.js"; -import { SignalCliClient } from "../../platforms/signal/cli/client.js"; import { type SignalRealtimeStatus, SignalRealtimeSupervisor, @@ -199,9 +195,7 @@ const QUEUE_DRAIN_RETRY_DELAY_MS = 1_000; const INGEST_RUN_LEASE_MS = 15 * 60_000; const PROJECTION_WORKER_TIMEOUT_MS = 30 * 60_000; const PROJECTION_RUN_LEASE_MS = PROJECTION_WORKER_TIMEOUT_MS + 5 * 60_000; -const SIGNAL_SEND_SESSION_WAIT_MS = 3_000; -const SIGNAL_SEND_ECHO_TIMEOUT_MS = 5_000; -const WHATSAPP_SEND_SESSION_WAIT_MS = 3_000; +const WHATSAPP_MEDIA_SESSION_WAIT_MS = 3_000; const daemonLogger = createLogger("daemon"); const hooksLogger = createLogger("hooks"); const nativeWatchLogger = createLogger("native-watch"); @@ -241,10 +235,6 @@ function getFallbackAppStatusMetadata(): { }; } -function isOutboundSendEnabled(): boolean { - return false; -} - function writeMenuBarStatusSnapshot( db: CuedDatabase, options: { @@ -292,7 +282,6 @@ function getDaemonIdentity(): { type QueueSchedulers = { wakeIngest: (delayMs?: number) => void; - wakeOutbound: () => void; wakeProjection: (delayMs?: number) => void; wakeSearchIndex: (delayMs?: number) => void; }; @@ -351,15 +340,6 @@ type LinkedInDesiredSession = { realtimeRecipeMap: string; }; -type PendingSignalEcho = { - accountKey: string; - threadId: string | null; - text: string; - timestamp: number; - timeout: NodeJS.Timeout; - outboundMessageId: string; -}; - function getConfiguredAutoSyncPlatforms(): AdapterPlatform[] | null { const raw = process.env.CUED_AUTOSYNC_PLATFORMS?.trim(); if (raw == null) { @@ -522,14 +502,6 @@ export function shouldSkipConnectedDiscordSchedulerSync( return targetPlatform === "discord" && trigger === "scheduler" && status?.state === "connected"; } -export function shouldDrainOutboundQueue(input: { - outboundSendEnabled: boolean; - isUpdateShutdownRequested: boolean; - activeOutboundSend: unknown; -}): boolean { - return input.outboundSendEnabled && !input.isUpdateShutdownRequested && !input.activeOutboundSend; -} - export function shouldProjectIngestRunInline(input: { platform: Platform; runType: "sync" | "sync_resume"; @@ -997,12 +969,7 @@ function getWhatsAppResyncPageBudget(): number { } async function safeEmitHookEvent( - event: - | "integration.authenticated" - | "sync.completed" - | "sync.failed" - | "message.sent" - | "message.received", + event: "integration.authenticated" | "sync.completed" | "sync.failed" | "message.received", payload: Record, ): Promise { try { @@ -1022,34 +989,6 @@ async function emitAuthenticatedHook( }); } -async function emitMessageSentHook( - message: OutboundMessageRow, - details: { - transport: string; - sentAt: number; - providerMessageId?: string | null; - conversationExternalId?: string | null; - }, -): Promise { - await safeEmitHookEvent("message.sent", { - outboundMessage: { - id: message.id, - platform: message.platform, - accountKey: message.account_key, - target: message.target, - threadId: message.thread_id, - text: message.text, - createdAt: message.created_at, - }, - delivery: { - transport: details.transport, - sentAt: details.sentAt, - providerMessageId: details.providerMessageId ?? null, - conversationExternalId: details.conversationExternalId ?? null, - }, - }); -} - function queueNativeTriggeredSync( db: ReturnType, platform: AdapterPlatform, @@ -1296,88 +1235,6 @@ function isInboundMessageEvent(rawEvent: Record): boolean { ); } -function resolveSignalTarget(message: OutboundMessageRow): { - recipient?: string; - groupId?: string; -} { - const threadId = message.thread_id ?? ""; - const target = message.target; - if (threadId.startsWith("group:")) { - return { groupId: threadId.slice("group:".length) }; - } - if (target.startsWith("group:")) { - return { groupId: target.slice("group:".length) }; - } - return { recipient: target }; -} - -function isRetryableSignalSendError(message: string): boolean { - const normalized = message.toLowerCase(); - if ( - normalized.includes("invalid") || - normalized.includes("unregistered") || - normalized.includes("not found") || - normalized.includes("malformed") - ) { - return false; - } - return true; -} - -async function sendSignalOutboundMessage( - message: OutboundMessageRow, - signalRealtime: SignalRealtimeSupervisor, -): Promise<{ transport: "session" | "fallback"; timestamp: number }> { - const target = resolveSignalTarget(message); - const session = signalRealtime.getSession(message.account_key); - if (session?.isConnected()) { - const result = await session.sendMessage(message.text, target); - return { - transport: "session", - timestamp: result.timestamp, - }; - } - - const waitedSession = await signalRealtime.waitForConnected( - message.account_key, - SIGNAL_SEND_SESSION_WAIT_MS, - ); - if (waitedSession?.isConnected()) { - const result = await waitedSession.sendMessage(message.text, target); - return { - transport: "session", - timestamp: result.timestamp, - }; - } - - const inspected = await inspectSignalCli(); - if (!inspected.cliPath) { - throw new Error("Bundled Signal helper was not found"); - } - if (!isSignalCliVersionSupported(inspected.version)) { - throw new Error( - `Bundled Signal helper is too old or invalid (${inspected.version?.raw ?? "unknown"})`, - ); - } - - const configDir = getSignalConfigDir(message.account_key); - const account = readSignalLinkedAccount(configDir); - if (!account) { - throw new Error(`Signal account is not linked for '${message.account_key}'`); - } - - const client = new SignalCliClient({ - account, - cliPath: inspected.cliPath, - configDir, - }); - const result = await client.sendMessage(message.text, target); - return { - transport: "fallback", - timestamp: result.timestamp, - }; -} - async function collectDesiredSignalSessions(db: ReturnType): Promise<{ desired: SignalDesiredSession[]; degraded: Array>; @@ -1873,12 +1730,10 @@ export async function runDaemon(): Promise { { child: ChildProcess; platform: Platform; accountKey: string } >(); const activeIngestRuns = new Map>(); - let activeOutboundSend: Promise | null = null; let isProcessingProjection = false; let isProcessingSearchIndex = false; let authProjectionPausedUntil = 0; let ingestDrainScheduled = false; - let outboundDrainScheduled = false; let projectionDrainScheduled = false; let searchIndexDrainScheduled = false; let ingestDrainTimer: NodeJS.Timeout | null = null; @@ -1890,7 +1745,6 @@ export async function runDaemon(): Promise { const lastAutoSyncQueuedAt = new Map(); const lastSignalReconnectSyncQueuedAt = new Map(); const lastContinuationProjectionQueuedAt = new Map(); - const pendingSignalSendEchoes = new Map(); const projectionMessageHooks = new ProjectionMessageHookBarrier(); const suppressNextSignalReconnectSync = new Set(); const nativeWatchers = new Map< @@ -1919,31 +1773,6 @@ export async function runDaemon(): Promise { error: null, }; - const clearSignalSendEcho = ( - accountKey: string, - matcher: (echo: PendingSignalEcho) => boolean, - ) => { - const echoes = pendingSignalSendEchoes.get(accountKey); - if (!echoes || echoes.length === 0) { - return; - } - - const remaining: PendingSignalEcho[] = []; - for (const echo of echoes) { - if (matcher(echo)) { - clearTimeout(echo.timeout); - continue; - } - remaining.push(echo); - } - - if (remaining.length === 0) { - pendingSignalSendEchoes.delete(accountKey); - return; - } - pendingSignalSendEchoes.set(accountKey, remaining); - }; - const queueMessageReceivedHooks = ( range: { startRowId: number; endRowId: number } | null, inboundMessages: ProjectionMessageHookPayload[], @@ -2033,42 +1862,6 @@ export async function runDaemon(): Promise { }); }; - const scheduleSignalSendEchoCatchup = (message: OutboundMessageRow, timestamp: number) => { - const threadId = message.thread_id ?? null; - const pending: PendingSignalEcho = { - accountKey: message.account_key, - threadId, - text: message.text, - timestamp, - outboundMessageId: message.id, - timeout: setTimeout(() => { - clearSignalSendEcho( - message.account_key, - (candidate) => candidate.outboundMessageId === message.id, - ); - if (!db.hasQueuedOrRunningRun(message.platform, message.account_key)) { - db.queueSyncRun({ - platform: message.platform, - accountKey: message.account_key, - runType: "sync", - trigger: "signal_send_echo_timeout", - details: { - source: message.platform, - accountKey: message.account_key, - trigger: "signal_send_echo_timeout", - outboundMessageId: message.id, - }, - }); - schedulers.wakeIngest(); - } - }, SIGNAL_SEND_ECHO_TIMEOUT_MS), - }; - - const existing = pendingSignalSendEchoes.get(message.account_key) ?? []; - existing.push(pending); - pendingSignalSendEchoes.set(message.account_key, existing); - }; - const updateSlackCheckpointFromRealtime = (accountKey: string) => { const checkpoint = db.getCheckpoint("slack", accountKey); const projection = db.getProjectionBacklog(); @@ -2580,19 +2373,6 @@ export async function runDaemon(): Promise { updateSignalCheckpointFromRealtime(accountKey); } - for (const message of messages) { - if (message.isFromMe) { - const normalizedThreadId = message.threadId; - const normalizedText = message.text.trim(); - clearSignalSendEcho(accountKey, (echo) => { - const sameThread = !echo.threadId || echo.threadId === normalizedThreadId; - const sameText = echo.text.trim() === normalizedText; - const nearTimestamp = Math.abs(echo.timestamp - message.sentAt) < 30_000; - return sameThread && sameText && nearTimestamp; - }); - } - } - const inboundMessages = collectInboundMessageHookPayloads( `signal_realtime:${accountKey}`, insertResult.insertedRows, @@ -2832,62 +2612,6 @@ export async function runDaemon(): Promise { } }; - const sendWhatsAppOutboundMessage = async ( - message: OutboundMessageRow, - realtime: WhatsAppRealtimeSupervisor, - ): Promise<{ - transport: "session"; - result: { messageID: string; chatJID: string; timestamp: number }; - }> => { - const session = - realtime.getSession(message.account_key) ?? - (await realtime.waitForConnected(message.account_key, WHATSAPP_SEND_SESSION_WAIT_MS)); - if (!session?.isConnected()) { - throw new Error(`WhatsApp session is not connected for '${message.account_key}'`); - } - - return { - transport: "session", - result: await session.sendText(message.target, message.text), - }; - }; - - const sendDiscordOutboundMessage = async ( - message: OutboundMessageRow, - realtime: DiscordRealtimeSupervisor, - ): Promise<{ - transport: "session" | "fallback"; - result: Awaited>; - currentUser: Awaited>; - channel: Awaited>; - }> => { - const secret = loadIntegrationSecret("discord", message.account_key).secret; - if (typeof secret.token !== "string" || secret.token.trim().length === 0) { - throw new Error(`Discord integration '${message.account_key}' is missing a token`); - } - - const client = new DiscordApiClient({ token: secret.token }); - const session = realtime.getSession(message.account_key); - const transport = session?.isConnected() ? "session" : "fallback"; - const [currentUser, channel] = await Promise.all([ - client.getCurrentUser(), - client.getChannel(message.target), - ]); - if (!isDiscordDmChannel(channel)) { - throw new Error(`Discord DM-only mode cannot send to non-DM target '${message.target}'`); - } - const result = session?.isConnected() - ? await session.sendMessage(message.target, message.text) - : await client.sendMessage(message.target, message.text); - - return { - transport, - result, - currentUser, - channel, - }; - }; - const drainIngestQueue = () => { ingestDrainScheduled = false; ingestDrainDueAt = null; @@ -2956,149 +2680,6 @@ export async function runDaemon(): Promise { }, normalizedDelayMs); }; - const drainOutboundQueue = () => { - outboundDrainScheduled = false; - if ( - !shouldDrainOutboundQueue({ - outboundSendEnabled: isOutboundSendEnabled(), - isUpdateShutdownRequested, - activeOutboundSend, - }) - ) { - return; - } - - const message = db.claimNextOutboundMessage(); - if (!message) { - return; - } - - activeOutboundSend = (async () => { - try { - if (message.platform === "discord") { - const sendResult = await sendDiscordOutboundMessage(message, discordRealtime); - await ingestDiscordRealtimeRawEvents( - message.account_key, - [ - buildDiscordConversationEvent({ - accountKey: message.account_key, - observedAt: now(), - channel: sendResult.channel, - currentUser: sendResult.currentUser, - }), - buildDiscordMessageEvent({ - accountKey: message.account_key, - observedAt: now(), - channel: sendResult.channel, - message: sendResult.result, - currentUserId: sendResult.currentUser.id, - }), - ], - `discord_send:${message.id}`, - sendResult.currentUser.global_name?.trim() || sendResult.currentUser.username, - ); - db.completeOutboundMessage(message.id); - await emitMessageSentHook(message, { - transport: sendResult.transport, - sentAt: Date.parse(sendResult.result.timestamp), - providerMessageId: sendResult.result.id, - conversationExternalId: sendResult.result.channel_id, - }); - return; - } - - if (message.platform === "signal") { - const sendResult = await sendSignalOutboundMessage(message, signalRealtime); - db.completeOutboundMessage(message.id); - await emitMessageSentHook(message, { - transport: sendResult.transport, - sentAt: sendResult.timestamp, - }); - if (sendResult.transport === "session") { - scheduleSignalSendEchoCatchup(message, sendResult.timestamp); - return; - } - if (!db.hasQueuedOrRunningRun(message.platform, message.account_key)) { - db.queueSyncRun({ - platform: message.platform, - accountKey: message.account_key, - runType: "sync", - trigger: "outbound_send_completed", - details: { - source: message.platform, - accountKey: message.account_key, - trigger: "outbound_send_completed", - outboundMessageId: message.id, - }, - }); - scheduleIngestDrain(); - } - return; - } - - if (message.platform === "whatsapp") { - const sendResult = await sendWhatsAppOutboundMessage(message, whatsAppRealtime); - db.completeOutboundMessage(message.id); - await emitMessageSentHook(message, { - transport: sendResult.transport, - sentAt: sendResult.result.timestamp, - providerMessageId: sendResult.result.messageID, - conversationExternalId: sendResult.result.chatJID, - }); - return; - } - db.failOutboundMessage({ - id: message.id, - retryable: false, - error: `Unsupported outbound platform: ${message.platform}`, - }); - return; - } catch (error) { - const messageText = error instanceof Error ? error.message : String(error); - const resolvedLogger = - message.platform === "discord" - ? discordLogger - : message.platform === "signal" - ? signalLogger - : whatsAppLogger; - resolvedLogger.warn("outbound send failed", { - accountKey: message.account_key, - outboundMessageId: message.id, - error: messageText, - }); - if (message.platform === "discord" && isDiscordAuthInvalidationError(error)) { - blockDiscordIntegration(db, message.account_key, messageText); - requestDiscordRealtimeReconcile(); - } - db.failOutboundMessage({ - id: message.id, - retryable: - message.platform === "signal" - ? isRetryableSignalSendError(messageText) - : message.platform === "discord" - ? !isDiscordAuthInvalidationError(error) - : true, - error: messageText, - }); - } finally { - activeOutboundSend = null; - scheduleOutboundDrain(); - maybeFinishUpdateShutdown(); - } - })(); - }; - - const scheduleOutboundDrain = () => { - if (!isOutboundSendEnabled()) { - return; - } - if (outboundDrainScheduled) { - return; - } - outboundDrainScheduled = true; - setImmediate(drainOutboundQueue); - }; - const drainProjectionQueue = () => { projectionDrainScheduled = false; projectionDrainDueAt = null; @@ -3281,7 +2862,6 @@ export async function runDaemon(): Promise { const schedulers = { wakeIngest: scheduleIngestDrain, - wakeOutbound: scheduleOutboundDrain, wakeProjection: scheduleProjectionDrain, wakeSearchIndex: scheduleSearchIndexDrain, }; @@ -3816,8 +3396,6 @@ export async function runDaemon(): Promise { return null; } }; - scheduleOutboundDrain(); - const stopRealtimeAndWatchers = () => { for (const watcher of nativeWatchers.values()) { stopNativeWatcher(watcher); @@ -3837,10 +3415,7 @@ export async function runDaemon(): Promise { const deadlineReached = updateShutdownRequestedAt != null && now() - updateShutdownRequestedAt >= UPDATE_SHUTDOWN_GRACE_MS; - if ( - deadlineReached || - (activeIngestRuns.size === 0 && !activeOutboundSend && !isProcessingProjection) - ) { + if (deadlineReached || (activeIngestRuns.size === 0 && !isProcessingProjection)) { shutdown(); } }; @@ -3857,7 +3432,6 @@ export async function runDaemon(): Promise { updateShutdownRequestedAt = now(); daemonLogger.info("daemon entering update shutdown", { activeIngestRuns: activeIngestRuns.size, - activeOutboundSend: Boolean(activeOutboundSend), isProcessingProjection, }); stopRealtimeAndWatchers(); @@ -3943,7 +3517,6 @@ export async function runDaemon(): Promise { currentRun.id, `Unsupported ingest run target: ${currentRun.run_type}:${currentRun.platform ?? "none"}`, undefined, - null, runClaim, ); return; @@ -3953,7 +3526,6 @@ export async function runDaemon(): Promise { currentRun.id, `No adapter registered for platform: ${currentRun.platform}`, undefined, - null, runClaim, ); await sendTelemetryEventSafe(db, "sync_failed", { @@ -4007,11 +3579,7 @@ export async function runDaemon(): Promise { let bundleContinuation: SyncContinuation | null = null; let runDiagnostics: Record | null = null; let checkpointLastSuccessAt = now(); - let sourceAccounts: Array<{ - platform: Platform; - accountKey: string; - displayName?: string | null; - }> = []; + let sourceAccounts: SourceAccountInput[] = []; let insertResult: ReturnType = { insertedCount: 0, insertedEvents: [], @@ -4185,9 +3753,7 @@ export async function runDaemon(): Promise { accountKey, proof: buildWhatsAppMessagesProof({ accountKey, - syncMode: bundleSyncMode, observedAt: now(), - runStartedAt: resyncStartedAt, hasMore, nextCursor: cursor, sinceMs, @@ -4284,29 +3850,6 @@ export async function runDaemon(): Promise { : null, inboundMessages, ); - if (platform === "signal") { - for (const rawEvent of insertResult.insertedEvents) { - if (rawEvent.entityKind !== "message" || rawEvent.eventKind !== "created") { - continue; - } - const payload = rawEvent.payload as Record; - if (payload.isFromMe !== true || typeof payload.content !== "string") { - continue; - } - const sourceConversationKey = - typeof payload.sourceConversationKey === "string" - ? payload.sourceConversationKey.replace(/^signal:/, "") - : null; - const sentAt = typeof payload.sentAt === "number" ? payload.sentAt : 0; - clearSignalSendEcho(accountKey, (echo) => { - const sameThread = !echo.threadId || echo.threadId === sourceConversationKey; - const sameText = echo.text.trim() === payload.content; - const nearTimestamp = sentAt > 0 ? Math.abs(echo.timestamp - sentAt) < 30_000 : true; - return sameThread && sameText && nearTimestamp; - }); - } - } - const projection = db.getProjectionBacklog(); const checkpointSyncMode = resolveCheckpointSyncMode( currentRun.run_type, @@ -4467,7 +4010,7 @@ export async function runDaemon(): Promise { ); requestDiscordRealtimeReconcile(); } - db.failRun(currentRun.id, errorMessage, undefined, null, runClaim); + db.failRun(currentRun.id, errorMessage, undefined, runClaim); if (currentRun.platform === "signal") { requestSignalRealtimeReconcile(); } @@ -4627,7 +4170,6 @@ export async function runDaemon(): Promise { currentRun.id, error instanceof Error ? error.message : String(error), undefined, - null, runClaim, ); await sendTelemetryEventSafe(db, "sync_failed", { @@ -4682,7 +4224,7 @@ export async function runDaemon(): Promise { }, reconcileLocalWatchers, requestUpdateShutdown, - () => isProcessingProjection || activeIngestRuns.size > 0 || activeOutboundSend !== null, + () => isProcessingProjection || activeIngestRuns.size > 0, () => { authProjectionPausedUntil = now() + PROJECTION_AUTH_GRACE_MS; schedulers.wakeProjection(PROJECTION_AUTH_RETRY_DELAY_MS); @@ -4748,11 +4290,6 @@ export async function runDaemon(): Promise { session.child.kill("SIGTERM"); } activeAuthSessions.clear(); - for (const echoes of pendingSignalSendEchoes.values()) { - for (const echo of echoes) { - clearTimeout(echo.timeout); - } - } projectionMessageHooks.clear(); db.upsertDaemonState({ pid: null, @@ -5481,7 +5018,7 @@ async function dispatchRequest( whatsAppRealtime.getSession(attachment.account_key) ?? (await whatsAppRealtime.waitForConnected( attachment.account_key, - WHATSAPP_SEND_SESSION_WAIT_MS, + WHATSAPP_MEDIA_SESSION_WAIT_MS, )); if (!session?.isConnected()) { throw new Error( diff --git a/src/runtime/hooks.test.ts b/src/runtime/hooks.test.ts index 38770a66..45ab706d 100644 --- a/src/runtime/hooks.test.ts +++ b/src/runtime/hooks.test.ts @@ -49,7 +49,7 @@ describe("hooks service", () => { expect(typeof doctor.openClawPath === "string" || doctor.openClawPath === null).toBe(true); expect(contents).toContain("sync.completed"); expect(contents).toContain("sync.failed"); - expect(contents).toContain("message.sent"); + expect(contents).toContain("message.received"); }); it("runs enabled subprocess hooks with JSON stdin", async () => { @@ -78,4 +78,35 @@ args = ["-lc", "cat > ${join(home, "hook-payload.json")}"] expect(payload).toContain('"event": "sync.completed"'); expect(payload).toContain('"runId": "123"'); }); + + it("ignores deprecated message.sent hooks from existing configs", async () => { + const home = setTempHome(); + const { doctorHooksConfig, loadHooksConfig } = await loadHooksService(); + const hooksDir = join(home, ".cued"); + mkdirSync(hooksDir, { recursive: true }); + + writeFileSync( + join(hooksDir, "hooks.toml"), + `version = 1 + +[[hooks]] +event = "message.sent" +enabled = false +command = "/bin/sh" + +[[hooks]] +event = "message.received" +enabled = true +command = "/bin/sh" +`, + "utf8", + ); + + const config = loadHooksConfig(); + expect(config.hooks.map((hook) => hook.event)).toEqual(["message.received"]); + + const doctor = doctorHooksConfig(); + expect(doctor.valid).toBe(true); + expect(doctor.hooks.map((hook) => hook.event)).toEqual(["message.received"]); + }); }); diff --git a/src/runtime/hooks.ts b/src/runtime/hooks.ts index 7e4a401e..d89dbf72 100644 --- a/src/runtime/hooks.ts +++ b/src/runtime/hooks.ts @@ -8,12 +8,13 @@ export const HOOK_EVENT_NAMES = [ "integration.authenticated", "sync.completed", "sync.failed", - "message.sent", "message.received", ] as const; export type HookEventName = (typeof HOOK_EVENT_NAMES)[number]; +const DEPRECATED_HOOK_EVENT_NAMES = ["message.sent"] as const; + export interface HookDefinition { event: HookEventName; enabled: boolean; @@ -57,6 +58,10 @@ function isHookEventName(value: string): value is HookEventName { return (HOOK_EVENT_NAMES as readonly string[]).includes(value); } +function isDeprecatedHookEventName(value: string): boolean { + return (DEPRECATED_HOOK_EVENT_NAMES as readonly string[]).includes(value); +} + function resolveCommandPath(command: string): string | null { try { return execFileSync("sh", ["-lc", `command -v ${shellQuote(command)}`], { @@ -81,7 +86,11 @@ function detectOpenClaw(): { detected: boolean; path: string | null } { return { detected: path !== null, path }; } -function normalizeHook(raw: Record): HookDefinition { +function normalizeHook(raw: Record): HookDefinition | null { + if (typeof raw.event === "string" && isDeprecatedHookEventName(raw.event)) { + return null; + } + const event = typeof raw.event === "string" && isHookEventName(raw.event) ? raw.event : null; if (!event) { throw new Error(`Unsupported hook event: ${String(raw.event)}`); @@ -117,7 +126,10 @@ export function loadHooksConfig(): { path: string; exists: boolean; hooks: HookD const raw = parse(readFileSync(CUED_HOOKS_PATH, "utf8")) as HookConfigFile; const hooks = Array.isArray(raw.hooks) - ? raw.hooks.map((hook) => normalizeHook(hook as unknown as Record)) + ? raw.hooks.flatMap((hook) => { + const normalized = normalizeHook(hook as unknown as Record); + return normalized ? [normalized] : []; + }) : []; return { path: CUED_HOOKS_PATH, exists: true, hooks }; } @@ -144,12 +156,6 @@ function buildSampleHooks(openClaw: { detected: boolean; path: string | null }): command: "/bin/sh", args: ["-lc", "cat >/tmp/cued-hook-sync-failed.json"], }, - { - event: "message.sent", - enabled: false, - command: "/bin/sh", - args: ["-lc", "cat >/tmp/cued-hook-message-sent.json"], - }, { event: "message.received", enabled: false, diff --git a/src/runtime/perf/run.ts b/src/runtime/perf/run.ts index 37629769..5cc19508 100644 --- a/src/runtime/perf/run.ts +++ b/src/runtime/perf/run.ts @@ -42,9 +42,6 @@ function perfRawEvent(input: ProviderRawEventInput): ProviderRawEventInput { return { ...input, normalizedSchema: buildNormalizedRawEventSchema(input.entityKind, input.eventKind), - provenance: { - adapterVersion: "perf-harness@1", - }, }; } @@ -389,7 +386,6 @@ function createLinkedInClientFixture(conversationCount: number, messagesPerConve groupChat: false, read: true, categories: ["PRIMARY_INBOX"], - unreadCount: 0, conversationParticipants: [ { entityURN: "urn:li:fsd_profile:SELF", @@ -502,7 +498,6 @@ function buildProjectionReplayEvents(): ProviderRawEventInput[] { { type: "email", value: `perf-${conversationIndex}@example.com`, deterministic: true }, ], }, - sourceVersion: "perf-v1", }), ); rawEvents.push( @@ -517,10 +512,8 @@ function buildProjectionReplayEvents(): ProviderRawEventInput[] { payload: { sourceConversationKey: `perf-conversation-${conversationIndex}`, conversationType: "dm", - service: "linkedin", participants: [{ sourceEntityKey: `contacts:${conversationIndex}` }], }, - sourceVersion: "perf-v1", }), ); @@ -540,10 +533,8 @@ function buildProjectionReplayEvents(): ProviderRawEventInput[] { senderSourceKey: `contacts:${conversationIndex}`, sentAt: baseObservedAt + conversationIndex * 100 + messageIndex, content: `Projection perf message ${conversationIndex}-${messageIndex}`, - service: "linkedin", isFromMe: false, }, - sourceVersion: "perf-v1", }), ); } @@ -581,7 +572,6 @@ function buildIncrementalProjectionEvents( }, ], }, - sourceVersion: "perf-v1", }), ); rawEvents.push( @@ -598,7 +588,6 @@ function buildIncrementalProjectionEvents( conversationType: "dm", participants: [{ sourceEntityKey: `contacts:${conversationIndex}` }], }, - sourceVersion: "perf-v1", }), ); @@ -618,10 +607,8 @@ function buildIncrementalProjectionEvents( senderSourceKey: `contacts:${conversationIndex}`, sentAt: baseObservedAt + conversationIndex * 100 + messageIndex, content: `Incremental message ${conversationIndex}-${messageIndex}`, - service: "linkedin", isFromMe: false, }, - sourceVersion: "perf-v1", }), ); } diff --git a/src/runtime/projection/events.ts b/src/runtime/projection/events.ts index 92a64095..3b8fbcd3 100644 --- a/src/runtime/projection/events.ts +++ b/src/runtime/projection/events.ts @@ -148,10 +148,6 @@ function buildSystemMessagePayload( return { ...timelinePayload, eventKind: "system_message", - metadata: { - ...(timelinePayload.metadata ?? {}), - legacyEventKind, - }, }; } @@ -172,10 +168,6 @@ function buildSystemMessagePayload( eventKind: "system_message", eventAt: Number(messagePayload.sentAt), text: typeof messagePayload.content === "string" ? messagePayload.content : null, - metadata: { - legacyEventKind, - sourceMessageKey: messagePayload.sourceMessageKey, - }, }; } @@ -211,11 +203,6 @@ function buildParticipantPayload( return { sourceConversationKey: timelinePayload.sourceConversationKey, participantSourceKey, - eventAt: timelinePayload.eventAt, - metadata: { - ...(timelinePayload.metadata ?? {}), - legacyTimelineEventKind: timelinePayload.eventKind, - }, }; } diff --git a/src/runtime/projection/projector.test.ts b/src/runtime/projection/projector.test.ts index f90ae4af..43b4e925 100644 --- a/src/runtime/projection/projector.test.ts +++ b/src/runtime/projection/projector.test.ts @@ -59,13 +59,6 @@ describe("projector", () => { SET actor_name = 'Ava' WHERE actor_contact_id IN ('contact-a') `); - const reactionPlan = db.orm().all<{ detail: string }>(sql` - EXPLAIN QUERY PLAN - UPDATE message_reactions - SET reactor_name = 'Ava' - WHERE reactor_contact_id IN ('contact-a') - `); - expect(messagePlan.map((row) => row.detail).join("\n")).toContain("idx_messages_sender"); expect(participantPlan.map((row) => row.detail).join("\n")).toContain( "idx_conversation_participants_contact", @@ -73,9 +66,6 @@ describe("projector", () => { expect(timelinePlan.map((row) => row.detail).join("\n")).toContain( "idx_timeline_events_actor_contact", ); - expect(reactionPlan.map((row) => row.detail).join("\n")).toContain( - "idx_message_reactions_reactor_contact", - ); }); it("quarantines unsupported normalized schemas with raw-event context", () => { @@ -91,9 +81,7 @@ describe("projector", () => { observed_at, dedupe_key, payload_json, - normalized_schema, - provenance_json, - source_version + normalized_schema ) VALUES ( 'unsupported-schema', 'linkedin', @@ -108,15 +96,9 @@ describe("projector", () => { senderSourceKey: "contacts:test", sentAt: 1_710_000_000_000, content: "unsupported", - service: "linkedin", isFromMe: false, })}, - 'message.created@99', - ${JSON.stringify({ - acquisitionMode: "realtime", - providerApiVersion: "2026-03", - })}, - 'linkedin-v99' + 'message.created@99' ) `); @@ -130,16 +112,13 @@ describe("projector", () => { }); expect( db.orm().get(sql` - SELECT raw_event_rowid, raw_event_id, platform, account_key, error_message + SELECT raw_event_rowid, error_message FROM raw_event_projection_failures `), ).toEqual({ raw_event_rowid: 1, - raw_event_id: "unsupported-schema", - platform: "linkedin", - account_key: "default", error_message: - "Failed to normalize raw event (row 1, event unsupported-schema, linkedin/default, message:created, schema message.created@99, sourceVersion linkedin-v99, providerApiVersion 2026-03, acquisitionMode realtime): Unsupported normalized raw event schema 'message.created@99'", + "Failed to normalize raw event (row 1, event unsupported-schema, linkedin/default, message:created, schema message.created@99): Unsupported normalized raw event schema 'message.created@99'", }); db.close(); @@ -158,8 +137,7 @@ describe("projector", () => { observed_at, dedupe_key, payload_json, - normalized_schema, - source_version + normalized_schema ) VALUES ( 'malformed-json', 'linkedin', @@ -169,25 +147,25 @@ describe("projector", () => { 1710000000000, 'linkedin:malformed-json', '{', - 'message.created@1', - 'linkedin-v1' + 'message.created@1' ) `); - db.insertRawEvent({ - id: "contact-after-malformed", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1_710_000_000_001, - dedupeKey: "contacts:after-malformed", - payload: { - sourceEntityKey: "contacts:after-malformed", - fields: { display_name: "After Malformed" }, - handles: [{ type: "email", value: "after@example.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); + db.insertRawEvents([ + { + id: "contact-after-malformed", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1_710_000_000_001, + dedupeKey: "contacts:after-malformed", + payload: { + sourceEntityKey: "contacts:after-malformed", + fields: { display_name: "After Malformed" }, + handles: [{ type: "email", value: "after@example.com", deterministic: true }], + }, + }, + ]); expect(projectPendingRawEvents(db)).toEqual({ contacts: 1, @@ -199,12 +177,11 @@ describe("projector", () => { }); expect( db.orm().get(sql` - SELECT raw_event_rowid, raw_event_id, error_message + SELECT raw_event_rowid, error_message FROM raw_event_projection_failures `), ).toEqual({ raw_event_rowid: 1, - raw_event_id: "malformed-json", error_message: expect.stringContaining( "Failed to normalize raw event (row 1, event malformed-json", ), @@ -226,9 +203,7 @@ describe("projector", () => { observed_at, dedupe_key, payload_json, - normalized_schema, - provenance_json, - source_version + normalized_schema ) VALUES ( 'legacy-message-created', 'linkedin', @@ -243,12 +218,9 @@ describe("projector", () => { senderSourceKey: "contacts:test", sentAt: 1_710_000_000_000, content: "legacy message", - service: "linkedin", isFromMe: false, })}, - NULL, - NULL, - 'linkedin-v1' + NULL ) `); @@ -266,48 +238,47 @@ describe("projector", () => { it("marks deleted messages without overwriting original content", () => { const db = createDb(); - db.insertRawEvent({ - id: "message-create", - platform: "whatsapp", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 1_710_000_000_000, - dedupeKey: "whatsapp:message:create", - payload: { - sourceMessageKey: "chat-1:msg-1", - sourceConversationKey: "chat-1", - senderSourceKey: "whatsapp:15551234567@s.whatsapp.net", - sentAt: 1_710_000_000_000, - content: "keep this content", - service: "whatsapp", - status: "delivered", - isFromMe: false, - }, - sourceVersion: "whatsapp-v1", - }); - db.insertRawEvent({ - id: "message-delete", - platform: "whatsapp", - accountKey: "default", - entityKind: "message", - eventKind: "deleted", - observedAt: 1_710_000_001_000, - dedupeKey: "whatsapp:message:delete", - payload: { - sourceMessageKey: "chat-1:msg-1", - sourceConversationKey: "chat-1", - senderSourceKey: null, - sentAt: 1_710_000_001_000, - content: "", - service: "whatsapp", - status: "sent", - isFromMe: true, - deletedAt: 1_710_000_001_000, - isDeleted: true, - }, - sourceVersion: "whatsapp-v1", - }); + db.insertRawEvents([ + { + id: "message-create", + platform: "whatsapp", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 1_710_000_000_000, + dedupeKey: "whatsapp:message:create", + payload: { + sourceMessageKey: "chat-1:msg-1", + sourceConversationKey: "chat-1", + senderSourceKey: "whatsapp:15551234567@s.whatsapp.net", + sentAt: 1_710_000_000_000, + content: "keep this content", + status: "delivered", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-delete", + platform: "whatsapp", + accountKey: "default", + entityKind: "message", + eventKind: "deleted", + observedAt: 1_710_000_001_000, + dedupeKey: "whatsapp:message:delete", + payload: { + sourceMessageKey: "chat-1:msg-1", + sourceConversationKey: "chat-1", + senderSourceKey: null, + sentAt: 1_710_000_001_000, + content: "", + status: "sent", + isFromMe: true, + isDeleted: true, + }, + }, + ]); rebuildProjectedState(db); @@ -315,10 +286,9 @@ describe("projector", () => { content: string | null; status: string | null; is_deleted: number; - deleted_at: number | null; is_from_me: number; }>(sql` - SELECT content, status, is_deleted, deleted_at, is_from_me + SELECT content, status, is_deleted, is_from_me FROM messages WHERE platform_message_id = 'chat-1:msg-1' `); @@ -326,7 +296,6 @@ describe("projector", () => { content: "keep this content", status: "delivered", is_deleted: 1, - deleted_at: 1_710_000_001_000, is_from_me: 0, }); @@ -336,82 +305,84 @@ describe("projector", () => { it("rebuilds projected state and preserves agent-facing views", () => { const db = createDb(); - db.insertRawEvent({ - id: randomUUID(), - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1_710_000_000_000, - dedupeKey: "contacts:ava", - payload: { - sourceEntityKey: "contacts:ava", - fields: { - display_name: "Ava Chen", - photo_url: "https://example.com/ava.png", - company: "Cued", - }, - handles: [ - { type: "email", value: "ava@cued.com", deterministic: true }, - { type: "phone", value: "+1 (555) 123-4567", deterministic: true }, - ], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: randomUUID(), - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 1_710_000_000_100, - dedupeKey: "linkedin:thread-1", - payload: { - sourceConversationKey: "thread-1", - conversationType: "dm", - service: "linkedin", - participants: [{ sourceEntityKey: "contacts:ava" }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: randomUUID(), - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 1_710_000_000_200, - dedupeKey: "linkedin:msg-1", - payload: { - sourceMessageKey: "msg-1", - sourceConversationKey: "thread-1", - senderSourceKey: "contacts:ava", - sentAt: 1_710_000_000_150, - content: "Founder update tomorrow?", - service: "linkedin", - status: "delivered", - isFromMe: false, + db.insertRawEvents([ + { + id: randomUUID(), + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1_710_000_000_000, + dedupeKey: "contacts:ava", + payload: { + sourceEntityKey: "contacts:ava", + fields: { + display_name: "Ava Chen", + photo_url: "https://example.com/ava.png", + company: "Cued", + }, + handles: [ + { type: "email", value: "ava@cued.com", deterministic: true }, + { type: "phone", value: "+1 (555) 123-4567", deterministic: true }, + ], + }, }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: randomUUID(), - platform: "linkedin", - accountKey: "default", - entityKind: "reaction", - eventKind: "added", - observedAt: 1_710_000_000_300, - dedupeKey: "linkedin:msg-1:thumbs-up", - payload: { - sourceMessageKey: "msg-1", - sourceConversationKey: "thread-1", - reactorSourceKey: "contacts:ava", - emoji: "👍", - timestamp: 1_710_000_000_250, - isActive: true, - }, - sourceVersion: "linkedin-v1", - }); + ]); + db.insertRawEvents([ + { + id: randomUUID(), + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 1_710_000_000_100, + dedupeKey: "linkedin:thread-1", + payload: { + sourceConversationKey: "thread-1", + conversationType: "dm", + participants: [{ sourceEntityKey: "contacts:ava" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: randomUUID(), + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 1_710_000_000_200, + dedupeKey: "linkedin:msg-1", + payload: { + sourceMessageKey: "msg-1", + sourceConversationKey: "thread-1", + senderSourceKey: "contacts:ava", + sentAt: 1_710_000_000_150, + content: "Founder update tomorrow?", + status: "delivered", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: randomUUID(), + platform: "linkedin", + accountKey: "default", + entityKind: "reaction", + eventKind: "added", + observedAt: 1_710_000_000_300, + dedupeKey: "linkedin:msg-1:thumbs-up", + payload: { + sourceMessageKey: "msg-1", + sourceConversationKey: "thread-1", + reactorSourceKey: "contacts:ava", + emoji: "👍", + timestamp: 1_710_000_000_250, + isActive: true, + }, + }, + ]); expect(rebuildProjectedState(db, { limit: 2 })).toEqual({ contacts: 1, @@ -464,11 +435,11 @@ describe("projector", () => { }, ]); - const reactionRows = db.orm().all<{ reaction_count: number }>(sql` - SELECT reaction_count - FROM messages + const reactionRows = db.orm().all<{ emoji: string }>(sql` + SELECT emoji + FROM message_reactions `); - expect(reactionRows).toEqual([{ reaction_count: 1 }]); + expect(reactionRows).toEqual([{ emoji: "👍" }]); db.close(); }); @@ -487,7 +458,6 @@ describe("projector", () => { payload: { sourceConversationKey: "thread-chunked", conversationType: "dm", - service: "linkedin", participants: [{ sourceEntityKey: "linkedin:casey" }], }, }, @@ -505,7 +475,6 @@ describe("projector", () => { senderSourceKey: "linkedin:casey", sentAt: 1_710_000_000_050, content: "identity arrives later", - service: "linkedin", isFromMe: false, }, }, @@ -549,76 +518,78 @@ describe("projector", () => { const db = createDb(); const observedAt = 1_710_100_000_000; - db.insertRawEvent({ - id: "conversation-before-contact", - platform: "slack", - accountKey: "T0A9C9RHZ9T", - entityKind: "conversation", - eventKind: "observed", - observedAt, - dedupeKey: "slack:conversation:C123", - payload: { - sourceConversationKey: "slack:T0A9C9RHZ9T:C123", - conversationType: "dm", - service: "slack", - participants: [{ sourceEntityKey: "slack:T0A9C9RHZ9T:U123" }], - }, - sourceVersion: "slack-v1", - }); - db.insertRawEvent({ - id: "message-before-contact", - platform: "slack", - accountKey: "T0A9C9RHZ9T", - entityKind: "message", - eventKind: "created", - observedAt, - dedupeKey: "slack:message:C123:1", - payload: { - sourceMessageKey: "slack:T0A9C9RHZ9T:C123:1710100000.000100", - sourceConversationKey: "slack:T0A9C9RHZ9T:C123", - senderSourceKey: "slack:T0A9C9RHZ9T:U123", - sentAt: observedAt - 500, - content: "hello from slack", - service: "slack", - isFromMe: false, - }, - sourceVersion: "slack-v1", - }); - db.insertRawEvent({ - id: "reaction-before-contact", - platform: "slack", - accountKey: "T0A9C9RHZ9T", - entityKind: "reaction", - eventKind: "added", - observedAt, - dedupeKey: "slack:reaction:C123:1:thumbsup", - payload: { - sourceMessageKey: "slack:T0A9C9RHZ9T:C123:1710100000.000100", - sourceConversationKey: "slack:T0A9C9RHZ9T:C123", - reactorSourceKey: "slack:T0A9C9RHZ9T:U123", - emoji: ":thumbsup:", - timestamp: observedAt - 400, - isActive: true, - }, - sourceVersion: "slack-v1", - }); - db.insertRawEvent({ - id: "contact-after", - platform: "slack", - accountKey: "T0A9C9RHZ9T", - entityKind: "contact", - eventKind: "observed", - observedAt, - dedupeKey: "slack:contact:U123", - payload: { - sourceEntityKey: "slack:T0A9C9RHZ9T:U123", - fields: { - display_name: "Avery Example", - }, - handles: [{ type: "slack_user_id", value: "T0A9C9RHZ9T:U123", deterministic: true }], - }, - sourceVersion: "slack-v1", - }); + db.insertRawEvents([ + { + id: "conversation-before-contact", + platform: "slack", + accountKey: "T0A9C9RHZ9T", + entityKind: "conversation", + eventKind: "observed", + observedAt, + dedupeKey: "slack:conversation:C123", + payload: { + sourceConversationKey: "slack:T0A9C9RHZ9T:C123", + conversationType: "dm", + participants: [{ sourceEntityKey: "slack:T0A9C9RHZ9T:U123" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-before-contact", + platform: "slack", + accountKey: "T0A9C9RHZ9T", + entityKind: "message", + eventKind: "created", + observedAt, + dedupeKey: "slack:message:C123:1", + payload: { + sourceMessageKey: "slack:T0A9C9RHZ9T:C123:1710100000.000100", + sourceConversationKey: "slack:T0A9C9RHZ9T:C123", + senderSourceKey: "slack:T0A9C9RHZ9T:U123", + sentAt: observedAt - 500, + content: "hello from slack", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: "reaction-before-contact", + platform: "slack", + accountKey: "T0A9C9RHZ9T", + entityKind: "reaction", + eventKind: "added", + observedAt, + dedupeKey: "slack:reaction:C123:1:thumbsup", + payload: { + sourceMessageKey: "slack:T0A9C9RHZ9T:C123:1710100000.000100", + sourceConversationKey: "slack:T0A9C9RHZ9T:C123", + reactorSourceKey: "slack:T0A9C9RHZ9T:U123", + emoji: ":thumbsup:", + timestamp: observedAt - 400, + isActive: true, + }, + }, + ]); + db.insertRawEvents([ + { + id: "contact-after", + platform: "slack", + accountKey: "T0A9C9RHZ9T", + entityKind: "contact", + eventKind: "observed", + observedAt, + dedupeKey: "slack:contact:U123", + payload: { + sourceEntityKey: "slack:T0A9C9RHZ9T:U123", + fields: { + display_name: "Avery Example", + }, + handles: [{ type: "slack_user_id", value: "T0A9C9RHZ9T:U123", deterministic: true }], + }, + }, + ]); expect(rebuildProjectedState(db)).toEqual({ contacts: 1, @@ -635,11 +606,11 @@ describe("projector", () => { `); expect(participantRows).toEqual([{ participant_name: "Avery Example" }]); - const messageRows = db.orm().all<{ sender_name: string | null; reaction_count: number }>(sql` - SELECT sender_name, reaction_count + const messageRows = db.orm().all<{ sender_name: string | null }>(sql` + SELECT sender_name FROM messages `); - expect(messageRows).toEqual([{ sender_name: "Avery Example", reaction_count: 1 }]); + expect(messageRows).toEqual([{ sender_name: "Avery Example" }]); db.close(); }); @@ -662,7 +633,6 @@ describe("projector", () => { fields: { display_name: "Ava Chen" }, handles: [], }, - sourceVersion: "contacts-v1", }, { id: "timeline-initial", @@ -679,7 +649,6 @@ describe("projector", () => { eventAt: observedAt + 1, text: "Ava joined", }, - sourceVersion: "linkedin-v1", }, { id: "timeline-updated", @@ -697,7 +666,6 @@ describe("projector", () => { text: "Ava joined", subjectSourceKey: "contacts:ava", }, - sourceVersion: "linkedin-v1", }, ]); @@ -732,7 +700,6 @@ describe("projector", () => { displayName: "Ava Chen", participants: [{ sourceEntityKey: "linkedin:urn:li:member:ACoAAA1" }], }, - sourceVersion: "test-v1", }, { id: "message-system-summary", @@ -750,7 +717,6 @@ describe("projector", () => { content: "older human message", isFromMe: false, }, - sourceVersion: "test-v1", }, { id: "timeline-system-summary", @@ -767,32 +733,29 @@ describe("projector", () => { eventAt: observedAt + 5, text: "Ava renamed the conversation", }, - sourceVersion: "test-v1", }, ]); projectPendingRawEvents(db); - const conversationRow = db.orm().get<{ - last_message_id: string | null; - last_message_at: number | null; - last_message_preview: string | null; + const timelineRow = db.orm().get<{ + text: string | null; + event_at: number; }>(sql` - SELECT last_message_id, last_message_at, last_message_preview - FROM conversations - WHERE source_conversation_key = 'thread-system-summary' + SELECT text, event_at + FROM timeline_events + WHERE source_event_key = 'timeline:system-summary' `); - expect(conversationRow).toEqual({ - last_message_id: null, - last_message_at: observedAt + 5, - last_message_preview: "Ava renamed the conversation", + expect(timelineRow).toEqual({ + text: "Ava renamed the conversation", + event_at: observedAt + 5, }); db.close(); }); - it("projects call raw events into typed timeline rows and conversation summaries", () => { + it("projects call raw events into typed timeline rows", () => { const db = createDb(); const observedAt = 1_710_060_000_000; @@ -810,7 +773,6 @@ describe("projector", () => { fields: { display_name: "Ava Chen" }, handles: [{ type: "phone", value: "+14155550123", deterministic: true }], }, - sourceVersion: "imessage-v1", }, { id: "imessage-conversation-call", @@ -826,7 +788,6 @@ describe("projector", () => { displayName: "Ava Chen", participants: [{ sourceEntityKey: "imessage:+14155550123" }], }, - sourceVersion: "imessage-v1", }, { id: "imessage-call", @@ -840,20 +801,14 @@ describe("projector", () => { sourceCallKey: "call-1", sourceConversationKey: "1", provider: "facetime", - providerCallType: "8", direction: "incoming", medium: "video", status: "declined", startedAt: observedAt + 10, - endedAt: observedAt + 10, durationSeconds: 0, initiatorSourceKey: "imessage:+14155550123", primaryRemoteSourceKey: "imessage:+14155550123", - remoteAddress: "+14155550123", - remoteDisplayName: "Ava Chen", - disconnectedCause: "21", }, - sourceVersion: "imessage-v1", }, ]); @@ -863,12 +818,6 @@ describe("projector", () => { system_kind: string | null; call_provider: string | null; call_direction: string | null; - call_status: string | null; - call_medium: string | null; - call_started_at: number | null; - call_ended_at: number | null; - call_duration_seconds: number | null; - call_disconnected_cause: string | null; subject_source_key: string | null; text: string | null; }>(sql` @@ -876,12 +825,6 @@ describe("projector", () => { system_kind, call_provider, call_direction, - call_status, - call_medium, - call_started_at, - call_ended_at, - call_duration_seconds, - call_disconnected_cause, subject_source_key, text FROM timeline_events @@ -891,29 +834,10 @@ describe("projector", () => { system_kind: "call", call_provider: "facetime", call_direction: "incoming", - call_status: "declined", - call_medium: "video", - call_started_at: observedAt + 10, - call_ended_at: observedAt + 10, - call_duration_seconds: 0, - call_disconnected_cause: "21", subject_source_key: "imessage:+14155550123", text: "Declined FaceTime video call", }); - const conversationRow = db.orm().get<{ - last_message_preview: string | null; - last_message_at: number | null; - }>(sql` - SELECT last_message_preview, last_message_at - FROM conversations - WHERE source_conversation_key = '1' - `); - expect(conversationRow).toEqual({ - last_message_preview: "Declined FaceTime video call", - last_message_at: observedAt + 10, - }); - db.close(); }); @@ -936,7 +860,6 @@ describe("projector", () => { displayName: null, participants: [], }, - sourceVersion: "whatsapp-v1", }, { id: "whatsapp-call-delete-outgoing", @@ -957,7 +880,6 @@ describe("projector", () => { durationSeconds: 61, primaryRemoteSourceKey: "whatsapp:12016824050@s.whatsapp.net", }, - sourceVersion: "whatsapp-v1", }, { id: "whatsapp-call-delete-incoming", @@ -977,7 +899,6 @@ describe("projector", () => { startedAt: observedAt + 20, primaryRemoteSourceKey: "whatsapp:12016824050@s.whatsapp.net", }, - sourceVersion: "whatsapp-v1", }, { id: "whatsapp-call-delete-event", @@ -992,7 +913,6 @@ describe("projector", () => { provider: "whatsapp", direction: "outgoing", }, - sourceVersion: "whatsapp-v1", }, ]); @@ -1013,15 +933,6 @@ describe("projector", () => { }, ]); - const conversationRow = db.orm().get<{ last_message_preview: string | null }>(sql` - SELECT last_message_preview - FROM conversations - WHERE source_conversation_key = 'whatsapp:12016824050@s.whatsapp.net' - `); - expect(conversationRow).toEqual({ - last_message_preview: "Missed WhatsApp call", - }); - db.close(); }); @@ -1043,7 +954,6 @@ describe("projector", () => { fields: { display_name: "Ava Chen" }, handles: [{ type: "phone", value: "+14155550124", deterministic: true }], }, - sourceVersion: "imessage-v1", }, { id: "imessage-conversation-call-unknown", @@ -1059,7 +969,6 @@ describe("projector", () => { displayName: "Ava Chen", participants: [{ sourceEntityKey: "imessage:+14155550124" }], }, - sourceVersion: "imessage-v1", }, { id: "imessage-call-unknown-provider", @@ -1077,14 +986,10 @@ describe("projector", () => { medium: "audio", status: "missed", startedAt: observedAt + 10, - endedAt: observedAt + 10, durationSeconds: 0, initiatorSourceKey: "imessage:+14155550124", primaryRemoteSourceKey: "imessage:+14155550124", - remoteAddress: "+14155550124", - remoteDisplayName: "Ava Chen", }, - sourceVersion: "imessage-v1", }, ]); @@ -1106,80 +1011,79 @@ describe("projector", () => { const db = createDb(); const observedAt = 1_710_200_000_000; - db.insertRawEvent({ - id: "contact", - platform: "linkedin", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt, - dedupeKey: "linkedin:contact:ava", - payload: { - sourceEntityKey: "linkedin:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "conversation", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: observedAt + 1, - dedupeKey: "linkedin:conversation:thread-1", - payload: { - sourceConversationKey: "thread-1", - conversationType: "dm", - service: "linkedin", - participants: [{ sourceEntityKey: "linkedin:ava" }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "message-1", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: observedAt + 2, - dedupeKey: "linkedin:message:1", - payload: { - sourceMessageKey: "msg-1", - sourceConversationKey: "thread-1", - senderSourceKey: "linkedin:ava", - sentAt: observedAt + 2, - content: "before refresh", - service: "linkedin", - isFromMe: false, - }, - sourceVersion: "linkedin-v1", - }); + db.insertRawEvents([ + { + id: "contact", + platform: "linkedin", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt, + dedupeKey: "linkedin:contact:ava", + payload: { + sourceEntityKey: "linkedin:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "conversation", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: observedAt + 1, + dedupeKey: "linkedin:conversation:thread-1", + payload: { + sourceConversationKey: "thread-1", + conversationType: "dm", + participants: [{ sourceEntityKey: "linkedin:ava" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-1", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: observedAt + 2, + dedupeKey: "linkedin:message:1", + payload: { + sourceMessageKey: "msg-1", + sourceConversationKey: "thread-1", + senderSourceKey: "linkedin:ava", + sentAt: observedAt + 2, + content: "before refresh", + isFromMe: false, + }, + }, + ]); projectPendingRawEvents(db); - db.insertRawEvent({ - id: "message-1-refresh", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "updated", - observedAt: observedAt + 3, - dedupeKey: "linkedin:message:1:refresh", - payload: { - sourceMessageKey: "msg-1", - sourceConversationKey: "thread-1", - senderSourceKey: "linkedin:ava", - sentAt: observedAt + 2, - content: "after refresh", - service: "linkedin", - isFromMe: false, - isEdited: true, - editedAt: observedAt + 3, - }, - sourceVersion: "linkedin-v2", - }); + db.insertRawEvents([ + { + id: "message-1-refresh", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "updated", + observedAt: observedAt + 3, + dedupeKey: "linkedin:message:1:refresh", + payload: { + sourceMessageKey: "msg-1", + sourceConversationKey: "thread-1", + senderSourceKey: "linkedin:ava", + sentAt: observedAt + 2, + content: "after refresh", + isFromMe: false, + }, + }, + ]); projectPendingRawEvents(db); drainSearchIndex(db); @@ -1220,158 +1124,161 @@ describe("projector", () => { it("does not let later raw handle observations clobber a resolved contact name", () => { const db = createDb(); - db.insertRawEvent({ - id: "contacts-ava", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:ava", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "phone", value: "(555) 123-4567", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "imessage-contact-ava", - platform: "imessage", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 2, - dedupeKey: "imessage:contact:ava", - payload: { - sourceEntityKey: "imessage:+15551234567", - fields: { display_name: "+15551234567" }, - handles: [ - { type: "phone", value: "+15551234567", deterministic: true }, - { type: "imessage_handle", value: "+15551234567", deterministic: true }, - ], - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "imessage-conversation-ava", - platform: "imessage", - accountKey: "local", - entityKind: "conversation", - eventKind: "observed", - observedAt: 3, - dedupeKey: "imessage:conversation:ava", - payload: { - sourceConversationKey: "chat-ava", - conversationType: "dm", - service: "iMessage", - displayName: "+15551234567", - participants: [{ sourceEntityKey: "imessage:+15551234567" }], - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "imessage-message-ava", - platform: "imessage", - accountKey: "local", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "imessage:message:ava", - payload: { - sourceMessageKey: "message-ava", - sourceConversationKey: "chat-ava", - senderSourceKey: "imessage:+15551234567", - sentAt: 4, - content: "hello from imessage", - service: "iMessage", - isFromMe: false, - }, - sourceVersion: "imessage-v1", - }); - - projectPendingRawEvents(db); - - const row = db.orm().get<{ - contact_name: string | null; - participant_name: string | null; - sender_name: string | null; - conversation_name: string | null; - }>(sql` - SELECT - c.name AS contact_name, - cp.participant_name, - m.sender_name, - m.conversation_name - FROM contacts c - JOIN conversation_participants cp ON cp.contact_id = c.id - JOIN messages m ON m.sender_contact_id = c.id - WHERE cp.source_participant_key = 'imessage:+15551234567' - `); - expect(row).toEqual({ - contact_name: "Ava Chen", - participant_name: "Ava Chen", - sender_name: "Ava Chen", - conversation_name: "Ava Chen", - }); - + db.insertRawEvents([ + { + id: "contacts-ava", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:ava", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "phone", value: "(555) 123-4567", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-contact-ava", + platform: "imessage", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 2, + dedupeKey: "imessage:contact:ava", + payload: { + sourceEntityKey: "imessage:+15551234567", + fields: { display_name: "+15551234567" }, + handles: [ + { type: "phone", value: "+15551234567", deterministic: true }, + { type: "imessage_handle", value: "+15551234567", deterministic: true }, + ], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-conversation-ava", + platform: "imessage", + accountKey: "local", + entityKind: "conversation", + eventKind: "observed", + observedAt: 3, + dedupeKey: "imessage:conversation:ava", + payload: { + sourceConversationKey: "chat-ava", + conversationType: "dm", + displayName: "+15551234567", + participants: [{ sourceEntityKey: "imessage:+15551234567" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-message-ava", + platform: "imessage", + accountKey: "local", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "imessage:message:ava", + payload: { + sourceMessageKey: "message-ava", + sourceConversationKey: "chat-ava", + senderSourceKey: "imessage:+15551234567", + sentAt: 4, + content: "hello from imessage", + isFromMe: false, + }, + }, + ]); + + projectPendingRawEvents(db); + + const row = db.orm().get<{ + contact_name: string | null; + participant_name: string | null; + sender_name: string | null; + conversation_name: string | null; + }>(sql` + SELECT + c.name AS contact_name, + cp.participant_name, + m.sender_name, + m.conversation_name + FROM contacts c + JOIN conversation_participants cp ON cp.contact_id = c.id + JOIN messages m ON m.sender_contact_id = c.id + WHERE cp.source_participant_key = 'imessage:+15551234567' + `); + expect(row).toEqual({ + contact_name: "Ava Chen", + participant_name: "Ava Chen", + sender_name: "Ava Chen", + conversation_name: "Ava Chen", + }); + db.close(); }); it("merges iMessage phone stubs into later Contacts observations when formats differ", () => { const db = createDb(); - db.insertRawEvent({ - id: "imessage-conversation-parent", - platform: "imessage", - accountKey: "local", - entityKind: "conversation", - eventKind: "observed", - observedAt: 1, - dedupeKey: "imessage:conversation:parent", - payload: { - sourceConversationKey: "chat-parent", - conversationType: "dm", - service: "iMessage", - displayName: "+17737441662", - participants: [{ sourceEntityKey: "imessage:+17737441662" }], - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "imessage-message-parent", - platform: "imessage", - accountKey: "local", - entityKind: "message", - eventKind: "created", - observedAt: 2, - dedupeKey: "imessage:message:parent", - payload: { - sourceMessageKey: "message-parent", - sourceConversationKey: "chat-parent", - senderSourceKey: "imessage:+17737441662", - sentAt: 2, - content: "hello from parent", - service: "iMessage", - isFromMe: false, - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "contacts-parent", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 3, - dedupeKey: "contacts:parent", - payload: { - sourceEntityKey: "contacts:parent", - fields: { display_name: "Parent" }, - handles: [{ type: "phone", value: "773 744 1662", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); + db.insertRawEvents([ + { + id: "imessage-conversation-parent", + platform: "imessage", + accountKey: "local", + entityKind: "conversation", + eventKind: "observed", + observedAt: 1, + dedupeKey: "imessage:conversation:parent", + payload: { + sourceConversationKey: "chat-parent", + conversationType: "dm", + displayName: "+17737441662", + participants: [{ sourceEntityKey: "imessage:+17737441662" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-message-parent", + platform: "imessage", + accountKey: "local", + entityKind: "message", + eventKind: "created", + observedAt: 2, + dedupeKey: "imessage:message:parent", + payload: { + sourceMessageKey: "message-parent", + sourceConversationKey: "chat-parent", + senderSourceKey: "imessage:+17737441662", + sentAt: 2, + content: "hello from parent", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: "contacts-parent", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 3, + dedupeKey: "contacts:parent", + payload: { + sourceEntityKey: "contacts:parent", + fields: { display_name: "Parent" }, + handles: [{ type: "phone", value: "773 744 1662", deterministic: true }], + }, + }, + ]); projectPendingRawEvents(db); @@ -1408,57 +1315,58 @@ describe("projector", () => { it("matches iMessage email aliases case-insensitively when contacts land later", () => { const db = createDb(); - db.insertRawEvent({ - id: "imessage-conversation-email", - platform: "imessage", - accountKey: "local", - entityKind: "conversation", - eventKind: "observed", - observedAt: 1, - dedupeKey: "imessage:conversation:email", - payload: { - sourceConversationKey: "chat-email", - conversationType: "dm", - service: "iMessage", - displayName: "Casey@Example.com", - participants: [{ sourceEntityKey: "imessage:Casey@Example.com" }], - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "imessage-message-email", - platform: "imessage", - accountKey: "local", - entityKind: "message", - eventKind: "created", - observedAt: 2, - dedupeKey: "imessage:message:email", - payload: { - sourceMessageKey: "message-email", - sourceConversationKey: "chat-email", - senderSourceKey: "imessage:Casey@Example.com", - sentAt: 2, - content: "hello from email", - service: "iMessage", - isFromMe: false, - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "contacts-email", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 3, - dedupeKey: "contacts:email", - payload: { - sourceEntityKey: "contacts:email", - fields: { display_name: "Casey Contact" }, - handles: [{ type: "email", value: "casey@example.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); + db.insertRawEvents([ + { + id: "imessage-conversation-email", + platform: "imessage", + accountKey: "local", + entityKind: "conversation", + eventKind: "observed", + observedAt: 1, + dedupeKey: "imessage:conversation:email", + payload: { + sourceConversationKey: "chat-email", + conversationType: "dm", + displayName: "Casey@Example.com", + participants: [{ sourceEntityKey: "imessage:Casey@Example.com" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-message-email", + platform: "imessage", + accountKey: "local", + entityKind: "message", + eventKind: "created", + observedAt: 2, + dedupeKey: "imessage:message:email", + payload: { + sourceMessageKey: "message-email", + sourceConversationKey: "chat-email", + senderSourceKey: "imessage:Casey@Example.com", + sentAt: 2, + content: "hello from email", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: "contacts-email", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 3, + dedupeKey: "contacts:email", + payload: { + sourceEntityKey: "contacts:email", + fields: { display_name: "Casey Contact" }, + handles: [{ type: "email", value: "casey@example.com", deterministic: true }], + }, + }, + ]); projectPendingRawEvents(db); @@ -1482,57 +1390,58 @@ describe("projector", () => { it("uses a DM conversation name to resolve sender identities when contact records stay raw", () => { const db = createDb(); - db.insertRawEvent({ - id: "signal-contact", - platform: "signal", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "signal:contact:+14155550123", - payload: { - sourceEntityKey: "signal:+14155550123", - fields: { display_name: "+14155550123" }, - handles: [{ type: "phone", value: "+14155550123", deterministic: true }], - }, - sourceVersion: "signal-v1", - }); - db.insertRawEvent({ - id: "signal-conversation", - platform: "signal", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "signal:conversation:dm-ava", - payload: { - sourceConversationKey: "signal:dm-ava", - conversationType: "dm", - displayName: "Ava Chen", - service: "signal", - participants: [{ sourceEntityKey: "signal:+14155550123" }], - }, - sourceVersion: "signal-v1", - }); - db.insertRawEvent({ - id: "signal-message", - platform: "signal", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "signal:message:1", - payload: { - sourceMessageKey: "signal:message:1", - sourceConversationKey: "signal:dm-ava", - senderSourceKey: "signal:+14155550123", - sentAt: 3, - content: "hello from signal", - service: "signal", - isFromMe: false, - }, - sourceVersion: "signal-v1", - }); + db.insertRawEvents([ + { + id: "signal-contact", + platform: "signal", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "signal:contact:+14155550123", + payload: { + sourceEntityKey: "signal:+14155550123", + fields: { display_name: "+14155550123" }, + handles: [{ type: "phone", value: "+14155550123", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "signal-conversation", + platform: "signal", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "signal:conversation:dm-ava", + payload: { + sourceConversationKey: "signal:dm-ava", + conversationType: "dm", + displayName: "Ava Chen", + participants: [{ sourceEntityKey: "signal:+14155550123" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "signal-message", + platform: "signal", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "signal:message:1", + payload: { + sourceMessageKey: "signal:message:1", + sourceConversationKey: "signal:dm-ava", + senderSourceKey: "signal:+14155550123", + sentAt: 3, + content: "hello from signal", + isFromMe: false, + }, + }, + ]); projectPendingRawEvents(db); @@ -1569,63 +1478,64 @@ describe("projector", () => { it("treats Signal UUID contact names as raw identifiers in SQL-backed sender resolution", () => { const db = createDb(); - db.insertRawEvent({ - id: "signal-contact-uuid", - platform: "signal", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "signal:contact:uuid", - payload: { - sourceEntityKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678", - fields: { display_name: "a1b2c3d4-e5f6-1234-9abc-def012345678" }, - handles: [ - { - type: "signal_id", - value: "a1b2c3d4-e5f6-1234-9abc-def012345678", - deterministic: true, - }, - ], + db.insertRawEvents([ + { + id: "signal-contact-uuid", + platform: "signal", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "signal:contact:uuid", + payload: { + sourceEntityKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678", + fields: { display_name: "a1b2c3d4-e5f6-1234-9abc-def012345678" }, + handles: [ + { + type: "signal_id", + value: "a1b2c3d4-e5f6-1234-9abc-def012345678", + deterministic: true, + }, + ], + }, }, - sourceVersion: "signal-v1", - }); - db.insertRawEvent({ - id: "signal-conversation-uuid", - platform: "signal", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "signal:conversation:uuid", - payload: { - sourceConversationKey: "signal:dm-uuid", - conversationType: "dm", - displayName: "Ava Chen", - service: "signal", - participants: [{ sourceEntityKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678" }], - }, - sourceVersion: "signal-v1", - }); - db.insertRawEvent({ - id: "signal-message-uuid", - platform: "signal", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "signal:message:uuid", - payload: { - sourceMessageKey: "signal:message:uuid", - sourceConversationKey: "signal:dm-uuid", - senderSourceKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678", - sentAt: 3, - content: "hello from signal uuid", - service: "signal", - isFromMe: false, - }, - sourceVersion: "signal-v1", - }); + ]); + db.insertRawEvents([ + { + id: "signal-conversation-uuid", + platform: "signal", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "signal:conversation:uuid", + payload: { + sourceConversationKey: "signal:dm-uuid", + conversationType: "dm", + displayName: "Ava Chen", + participants: [{ sourceEntityKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "signal-message-uuid", + platform: "signal", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "signal:message:uuid", + payload: { + sourceMessageKey: "signal:message:uuid", + sourceConversationKey: "signal:dm-uuid", + senderSourceKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678", + sentAt: 3, + content: "hello from signal uuid", + isFromMe: false, + }, + }, + ]); projectPendingRawEvents(db); @@ -1662,21 +1572,22 @@ describe("projector", () => { it("projects new raw events incrementally without clearing canonical tables", () => { const db = createDb(); - db.insertRawEvent({ - id: "contacts-ava", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 10, - dedupeKey: "contacts:ava", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); + db.insertRawEvents([ + { + id: "contacts-ava", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 10, + dedupeKey: "contacts:ava", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], + }, + }, + ]); expect(projectPendingRawEvents(db)).toEqual({ contacts: 1, @@ -1687,41 +1598,41 @@ describe("projector", () => { projectionWatermark: 1, }); - db.insertRawEvent({ - id: "conversation-1", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 20, - dedupeKey: "linkedin:thread-1", - payload: { - sourceConversationKey: "thread-1", - conversationType: "dm", - service: "linkedin", - participants: [{ sourceEntityKey: "contacts:ava" }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "message-1", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 30, - dedupeKey: "linkedin:msg-1", - payload: { - sourceMessageKey: "msg-1", - sourceConversationKey: "thread-1", - senderSourceKey: "contacts:ava", - sentAt: 25, - content: "incremental projection", - service: "linkedin", - isFromMe: false, - }, - sourceVersion: "linkedin-v1", - }); + db.insertRawEvents([ + { + id: "conversation-1", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 20, + dedupeKey: "linkedin:thread-1", + payload: { + sourceConversationKey: "thread-1", + conversationType: "dm", + participants: [{ sourceEntityKey: "contacts:ava" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-1", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 30, + dedupeKey: "linkedin:msg-1", + payload: { + sourceMessageKey: "msg-1", + sourceConversationKey: "thread-1", + senderSourceKey: "contacts:ava", + sentAt: 25, + content: "incremental projection", + isFromMe: false, + }, + }, + ]); expect(projectPendingRawEvents(db)).toEqual({ contacts: 1, @@ -1750,36 +1661,38 @@ describe("projector", () => { it("supports batch-limited incremental projection", () => { const db = createDb(); - db.insertRawEvent({ - id: "contact-1", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:one", - payload: { - sourceEntityKey: "contacts:one", - fields: { display_name: "One" }, - handles: [{ type: "email", value: "one@example.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "contact-2", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 2, - dedupeKey: "contacts:two", - payload: { - sourceEntityKey: "contacts:two", - fields: { display_name: "Two" }, - handles: [{ type: "email", value: "two@example.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); + db.insertRawEvents([ + { + id: "contact-1", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:one", + payload: { + sourceEntityKey: "contacts:one", + fields: { display_name: "One" }, + handles: [{ type: "email", value: "one@example.com", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "contact-2", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 2, + dedupeKey: "contacts:two", + payload: { + sourceEntityKey: "contacts:two", + fields: { display_name: "Two" }, + handles: [{ type: "email", value: "two@example.com", deterministic: true }], + }, + }, + ]); expect(projectPendingRawEvents(db, { limit: 1 })).toEqual({ contacts: 1, @@ -1819,7 +1732,6 @@ describe("projector", () => { fields: { display_name: "Ava Chen" }, handles: [{ type: "email", value: "ava@example.com", deterministic: true }], }, - sourceVersion: "test-v1", }, { id: "conversation-hot-path", @@ -1834,7 +1746,6 @@ describe("projector", () => { conversationType: "dm", participants: [{ sourceEntityKey: "contacts:ava" }], }, - sourceVersion: "test-v1", }, { id: "message-hot-path", @@ -1852,7 +1763,6 @@ describe("projector", () => { content: "hot path preview", isFromMe: false, }, - sourceVersion: "test-v1", }, ]); @@ -1863,17 +1773,13 @@ describe("projector", () => { }); const hotConversation = db.orm().get<{ - last_message_preview: string | null; - unread_count: number; participant_names: string | null; }>(sql` - SELECT last_message_preview, unread_count, participant_names + SELECT participant_names FROM conversations WHERE source_conversation_key = 'thread-hot-path' `); expect(hotConversation).toEqual({ - last_message_preview: "hot path preview", - unread_count: 1, participant_names: "Ava Chen", }); @@ -1955,7 +1861,6 @@ describe("projector", () => { conversationType: "dm", participants: [], }, - sourceVersion: "test-v1", }, { id: "message-realtime-attachment", @@ -1981,7 +1886,6 @@ describe("projector", () => { }, ], }, - sourceVersion: "test-v1", }, { id: "reaction-realtime-attachment", @@ -1999,7 +1903,6 @@ describe("projector", () => { timestamp: 3, isActive: true, }, - sourceVersion: "test-v1", }, ]); @@ -2011,26 +1914,14 @@ describe("projector", () => { const messageRow = db.orm().get<{ content: string | null; - attachment_count: number; - reaction_count: number; }>(sql` - SELECT content, attachment_count, reaction_count + SELECT content FROM messages WHERE platform_message_id = 'imessage:message:attachment' `); - const conversationRow = db.orm().get<{ last_message_preview: string | null }>(sql` - SELECT last_message_preview - FROM conversations - WHERE source_conversation_key = 'imessage:chat:attachment' - `); expect(messageRow).toEqual({ content: "[attachment: deck.pdf]", - attachment_count: 1, - reaction_count: 1, - }); - expect(conversationRow).toEqual({ - last_message_preview: "[attachment: deck.pdf]", }); db.close(); @@ -2055,7 +1946,6 @@ describe("projector", () => { { type: "imessage_handle", value: "+15559876543", deterministic: true }, ], }, - sourceVersion: "imessage-v1", }, { id: "rt-imessage-conv", @@ -2068,11 +1958,9 @@ describe("projector", () => { payload: { sourceConversationKey: "rt-imessage-chat", conversationType: "dm", - service: "iMessage", displayName: "+15559876543", participants: [{ sourceEntityKey: "imessage:+15559876543" }], }, - sourceVersion: "imessage-v1", }, { id: "rt-imessage-msg", @@ -2088,10 +1976,8 @@ describe("projector", () => { senderSourceKey: "imessage:+15559876543", sentAt: 3, content: "hi", - service: "iMessage", isFromMe: false, }, - sourceVersion: "imessage-v1", }, ]); projectRealtimeRange(db, { @@ -2128,11 +2014,9 @@ describe("projector", () => { payload: { sourceConversationKey: "order-imessage-chat", conversationType: "dm", - service: "iMessage", displayName: "+15551110000", participants: [{ sourceEntityKey: "imessage:+15551110000" }], }, - sourceVersion: "imessage-v1", }, { id: "order-imessage-msg", @@ -2148,10 +2032,8 @@ describe("projector", () => { senderSourceKey: "imessage:+15551110000", sentAt: 2, content: "out of order", - service: "iMessage", isFromMe: false, }, - sourceVersion: "imessage-v1", }, { id: "order-imessage-contact", @@ -2169,7 +2051,6 @@ describe("projector", () => { { type: "imessage_handle", value: "+15551110000", deterministic: true }, ], }, - sourceVersion: "imessage-v1", }, ]); projectRealtimeRange(db, { @@ -2195,57 +2076,58 @@ describe("projector", () => { it("replays older contacts observations onto existing imessage stubs without rewinding the watermark", () => { const db = createDb(); - db.insertRawEvent({ - id: "contacts-ava-phone", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:ava-phone", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "imessage-conversation-ava", - platform: "imessage", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "imessage:conversation:ava", - payload: { - sourceConversationKey: "chat-ava", - conversationType: "dm", - service: "iMessage", - displayName: "+15551234567", - participants: [{ sourceEntityKey: "imessage:+15551234567" }], - }, - sourceVersion: "imessage-v1", - }); - db.insertRawEvent({ - id: "imessage-message-ava", - platform: "imessage", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "imessage:message:ava", - payload: { - sourceMessageKey: "message-ava", - sourceConversationKey: "chat-ava", - senderSourceKey: "imessage:+15551234567", - sentAt: 3, - content: "hello from imessage", - service: "iMessage", - isFromMe: false, - }, - sourceVersion: "imessage-v1", - }); + db.insertRawEvents([ + { + id: "contacts-ava-phone", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:ava-phone", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-conversation-ava", + platform: "imessage", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "imessage:conversation:ava", + payload: { + sourceConversationKey: "chat-ava", + conversationType: "dm", + displayName: "+15551234567", + participants: [{ sourceEntityKey: "imessage:+15551234567" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "imessage-message-ava", + platform: "imessage", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "imessage:message:ava", + payload: { + sourceMessageKey: "message-ava", + sourceConversationKey: "chat-ava", + senderSourceKey: "imessage:+15551234567", + sentAt: 3, + content: "hello from imessage", + isFromMe: false, + }, + }, + ]); expect( projectDeferredRange(db, { @@ -2346,109 +2228,98 @@ describe("projector", () => { it("resolves replies, projects attachments, and propagates renamed names", () => { const db = createDb(); - db.insertRawEvent({ - id: "contact-ava", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:ava", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "conversation-ava", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "linkedin:thread-ava", - payload: { - sourceConversationKey: "thread-ava", - conversationType: "dm", - service: "linkedin", - participants: [{ sourceEntityKey: "contacts:ava" }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "reply-message", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "linkedin:reply", - payload: { - sourceMessageKey: "reply", - sourceConversationKey: "thread-ava", - senderSourceKey: "contacts:ava", - sentAt: 30, - content: "reply first", - service: "linkedin", - isFromMe: false, - replyToSourceMessageKey: "parent", - attachments: [ - { - id: "att-1", - kind: "file", - name: "agenda.pdf", - title: "Agenda", - text: "Board agenda", - }, - ], + db.insertRawEvents([ + { + id: "contact-ava", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:ava", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], + }, }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "parent-message", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "linkedin:parent", - payload: { - sourceMessageKey: "parent", - sourceConversationKey: "thread-ava", - senderSourceKey: "contacts:ava", - sentAt: 20, - content: "parent later", - service: "linkedin", - isFromMe: false, - }, - sourceVersion: "linkedin-v1", - }); + ]); + db.insertRawEvents([ + { + id: "conversation-ava", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "linkedin:thread-ava", + payload: { + sourceConversationKey: "thread-ava", + conversationType: "dm", + participants: [{ sourceEntityKey: "contacts:ava" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "reply-message", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "linkedin:reply", + payload: { + sourceMessageKey: "reply", + sourceConversationKey: "thread-ava", + senderSourceKey: "contacts:ava", + sentAt: 30, + content: "reply first", + isFromMe: false, + attachments: [ + { + id: "att-1", + kind: "file", + name: "agenda.pdf", + title: "Agenda", + text: "Board agenda", + }, + ], + }, + }, + ]); + db.insertRawEvents([ + { + id: "parent-message", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "linkedin:parent", + payload: { + sourceMessageKey: "parent", + sourceConversationKey: "thread-ava", + senderSourceKey: "contacts:ava", + sentAt: 20, + content: "parent later", + isFromMe: false, + }, + }, + ]); rebuildProjectedState(db); const replyRows = db.orm().all<{ platform_message_id: string; - reply_to_message_id: string | null; - attachment_count: number; sender_name: string | null; conversation_name: string | null; }>(sql` - SELECT platform_message_id, reply_to_message_id, attachment_count, sender_name, conversation_name + SELECT platform_message_id, sender_name, conversation_name FROM messages ORDER BY sent_at ASC `); - const parent = replyRows.find((row) => row.platform_message_id === "parent"); const reply = replyRows.find((row) => row.platform_message_id === "reply"); - const parentIdRow = db.orm().get<{ id: string }>(sql` - SELECT id - FROM messages - WHERE platform_message_id = 'parent' - `); - expect(parent?.reply_to_message_id).toBeNull(); - expect(reply?.reply_to_message_id).toBe(parentIdRow?.id ?? null); - expect(reply?.attachment_count).toBe(1); expect(reply?.sender_name).toBe("Ava Chen"); expect(reply?.conversation_name).toBe("Ava Chen"); @@ -2458,37 +2329,39 @@ describe("projector", () => { `); expect(attachmentRows).toEqual([{ filename: "agenda.pdf", title: "Agenda" }]); - db.insertRawEvent({ - id: "contact-ava-rename", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 5, - dedupeKey: "contacts:ava:rename", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Zhang" }, - handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], - }, - sourceVersion: "contacts-v2", - }); - db.insertRawEvent({ - id: "conversation-ava-rename", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 6, - dedupeKey: "linkedin:thread-ava:rename", - payload: { - sourceConversationKey: "thread-ava", - conversationType: "dm", - displayName: "Investor thread", - participants: [{ sourceEntityKey: "contacts:ava" }], - }, - sourceVersion: "linkedin-v2", - }); + db.insertRawEvents([ + { + id: "contact-ava-rename", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 5, + dedupeKey: "contacts:ava:rename", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Zhang" }, + handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "conversation-ava-rename", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 6, + dedupeKey: "linkedin:thread-ava:rename", + payload: { + sourceConversationKey: "thread-ava", + conversationType: "dm", + displayName: "Investor thread", + participants: [{ sourceEntityKey: "contacts:ava" }], + }, + }, + ]); projectPendingRawEvents(db); @@ -2529,92 +2402,97 @@ describe("projector", () => { it("normalizes deferred attachment-only placeholders and preserves real text", () => { const db = createDb(); - db.insertRawEvent({ - id: "conversation-placeholder-policy", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 1, - dedupeKey: "conversation-placeholder-policy", - payload: { - sourceConversationKey: "thread-placeholder-policy", - conversationType: "dm", - participants: [], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-placeholder-mime", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 2, - dedupeKey: "message-placeholder-mime", - payload: { - sourceMessageKey: "message-placeholder-mime", - sourceConversationKey: "thread-placeholder-policy", - sentAt: 2, - content: "[attachment]", - attachments: [{ id: "att-pdf", kind: "file", mime_type: "application/pdf" }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-placeholder-text", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "message-placeholder-text", - payload: { - sourceMessageKey: "message-placeholder-text", - sourceConversationKey: "thread-placeholder-policy", - sentAt: 3, - content: "Quarterly memo", - attachments: [{ id: "att-caption", filename: "memo.pdf" }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-placeholder-multi", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "message-placeholder-multi", - payload: { - sourceMessageKey: "message-placeholder-multi", - sourceConversationKey: "thread-placeholder-policy", - sentAt: 4, - content: "", - attachments: [ - { id: "att-1", filename: "one.txt" }, - { id: "att-2", filename: "two.txt" }, - ], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-placeholder-filename", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 5, - dedupeKey: "message-placeholder-filename", - payload: { - sourceMessageKey: "message-placeholder-filename", - sourceConversationKey: "thread-placeholder-policy", - sentAt: 5, - content: "", - attachments: [{ id: "att-deck", filename: "deck.pdf" }], - }, - sourceVersion: "test-v1", - }); + db.insertRawEvents([ + { + id: "conversation-placeholder-policy", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 1, + dedupeKey: "conversation-placeholder-policy", + payload: { + sourceConversationKey: "thread-placeholder-policy", + conversationType: "dm", + participants: [], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-placeholder-mime", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 2, + dedupeKey: "message-placeholder-mime", + payload: { + sourceMessageKey: "message-placeholder-mime", + sourceConversationKey: "thread-placeholder-policy", + sentAt: 2, + content: "[attachment]", + attachments: [{ id: "att-pdf", kind: "file", mime_type: "application/pdf" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-placeholder-text", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "message-placeholder-text", + payload: { + sourceMessageKey: "message-placeholder-text", + sourceConversationKey: "thread-placeholder-policy", + sentAt: 3, + content: "Quarterly memo", + attachments: [{ id: "att-caption", filename: "memo.pdf" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-placeholder-multi", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "message-placeholder-multi", + payload: { + sourceMessageKey: "message-placeholder-multi", + sourceConversationKey: "thread-placeholder-policy", + sentAt: 4, + content: "", + attachments: [ + { id: "att-1", filename: "one.txt" }, + { id: "att-2", filename: "two.txt" }, + ], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-placeholder-filename", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 5, + dedupeKey: "message-placeholder-filename", + payload: { + sourceMessageKey: "message-placeholder-filename", + sourceConversationKey: "thread-placeholder-policy", + sentAt: 5, + content: "", + attachments: [{ id: "att-deck", filename: "deck.pdf" }], + }, + }, + ]); projectPendingRawEvents(db); @@ -2626,12 +2504,6 @@ describe("projector", () => { ) ORDER BY sent_at ASC `); - const conversationRow = db.orm().get<{ last_message_preview: string | null }>(sql` - SELECT last_message_preview - FROM conversations - WHERE source_conversation_key = 'thread-placeholder-policy' - `); - expect(messageRows).toEqual([ { platform_message_id: "message-placeholder-mime", content: "[pdf attachment]" }, { platform_message_id: "message-placeholder-text", content: "Quarterly memo" }, @@ -2641,9 +2513,6 @@ describe("projector", () => { content: "[attachment: deck.pdf]", }, ]); - expect(conversationRow).toEqual({ - last_message_preview: "[attachment: deck.pdf]", - }); db.close(); }); @@ -2651,78 +2520,81 @@ describe("projector", () => { it("applies manual contact merge decisions during rebuild", () => { const db = createDb(); - db.insertRawEvent({ - id: "contacts-ava-primary", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:ava-primary", - payload: { - sourceEntityKey: "contacts:ava-primary", - fields: { - display_name: "Ava Chen", - company: "Prime Ventures", - }, - handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], - }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "linkedin-ava-duplicate", - platform: "linkedin", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 2, - dedupeKey: "linkedin:ava-duplicate", - payload: { - sourceEntityKey: "linkedin:ava-duplicate", - sourceProfileUrl: "https://www.linkedin.com/in/ava-chen/", - fields: { - display_name: "Ava Chen", - company: "Acme Ventures", - photo_url: "https://example.com/linkedin-ava.jpg", - }, - handles: [{ type: "linkedin", value: "urn:li:person:ava-chen", deterministic: true }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "linkedin-conversation-ava", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 3, - dedupeKey: "linkedin:conversation:ava", - payload: { - sourceConversationKey: "thread-ava", - conversationType: "dm", - displayName: "Ava Chen", - participants: [{ sourceEntityKey: "linkedin:ava-duplicate" }], - }, - sourceVersion: "linkedin-v1", - }); - db.insertRawEvent({ - id: "linkedin-message-ava", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "linkedin:message:ava", - payload: { - sourceMessageKey: "msg-ava", - sourceConversationKey: "thread-ava", - senderSourceKey: "linkedin:ava-duplicate", - sentAt: 4, - content: "hello from linkedin", - isFromMe: false, - }, - sourceVersion: "linkedin-v1", - }); + db.insertRawEvents([ + { + id: "contacts-ava-primary", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:ava-primary", + payload: { + sourceEntityKey: "contacts:ava-primary", + fields: { + display_name: "Ava Chen", + company: "Prime Ventures", + }, + handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "linkedin-ava-duplicate", + platform: "linkedin", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 2, + dedupeKey: "linkedin:ava-duplicate", + payload: { + sourceEntityKey: "linkedin:ava-duplicate", + fields: { + display_name: "Ava Chen", + company: "Acme Ventures", + photo_url: "https://example.com/linkedin-ava.jpg", + }, + handles: [{ type: "linkedin", value: "urn:li:person:ava-chen", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "linkedin-conversation-ava", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 3, + dedupeKey: "linkedin:conversation:ava", + payload: { + sourceConversationKey: "thread-ava", + conversationType: "dm", + displayName: "Ava Chen", + participants: [{ sourceEntityKey: "linkedin:ava-duplicate" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "linkedin-message-ava", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "linkedin:message:ava", + payload: { + sourceMessageKey: "msg-ava", + sourceConversationKey: "thread-ava", + senderSourceKey: "linkedin:ava-duplicate", + sentAt: 4, + content: "hello from linkedin", + isFromMe: false, + }, + }, + ]); rebuildProjectedState(db); @@ -2796,57 +2668,60 @@ describe("projector", () => { it("updates attachments incrementally without leaving orphan rows", () => { const db = createDb(); - db.insertRawEvent({ - id: "contact-attachments", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contact-attachments", - payload: { - sourceEntityKey: "contacts:ava", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "email", value: "ava@example.com", deterministic: true }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "conversation-attachments", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "conversation-attachments", - payload: { - sourceConversationKey: "thread-attachments", - conversationType: "dm", - participants: [{ sourceEntityKey: "contacts:ava" }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-attachments-v1", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "message-attachments-v1", - payload: { - sourceMessageKey: "message-attachments", - sourceConversationKey: "thread-attachments", - senderSourceKey: "contacts:ava", - sentAt: 3, - content: "first attachment set", - attachments: [ - { id: "att-1", filename: "one.txt" }, - { id: "att-2", filename: "two.txt" }, - ], - }, - sourceVersion: "test-v1", - }); + db.insertRawEvents([ + { + id: "contact-attachments", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contact-attachments", + payload: { + sourceEntityKey: "contacts:ava", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "email", value: "ava@example.com", deterministic: true }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "conversation-attachments", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "conversation-attachments", + payload: { + sourceConversationKey: "thread-attachments", + conversationType: "dm", + participants: [{ sourceEntityKey: "contacts:ava" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-attachments-v1", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "message-attachments-v1", + payload: { + sourceMessageKey: "message-attachments", + sourceConversationKey: "thread-attachments", + senderSourceKey: "contacts:ava", + sentAt: 3, + content: "first attachment set", + attachments: [ + { id: "att-1", filename: "one.txt" }, + { id: "att-2", filename: "two.txt" }, + ], + }, + }, + ]); projectPendingRawEvents(db); @@ -2869,27 +2744,28 @@ describe("projector", () => { title: "One", }); - db.insertRawEvent({ - id: "message-attachments-v2", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "message-attachments-v2", - payload: { - sourceMessageKey: "message-attachments", - sourceConversationKey: "thread-attachments", - senderSourceKey: "contacts:ava", - sentAt: 4, - content: "second attachment set", - attachments: [ - { id: "att-2", filename: "two-updated.txt" }, - { id: "att-3", filename: "three.txt" }, - ], - }, - sourceVersion: "test-v1", - }); + db.insertRawEvents([ + { + id: "message-attachments-v2", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "message-attachments-v2", + payload: { + sourceMessageKey: "message-attachments", + sourceConversationKey: "thread-attachments", + senderSourceKey: "contacts:ava", + sentAt: 4, + content: "second attachment set", + attachments: [ + { id: "att-2", filename: "two-updated.txt" }, + { id: "att-3", filename: "three.txt" }, + ], + }, + }, + ]); projectPendingRawEvents(db); @@ -2922,72 +2798,76 @@ describe("projector", () => { it("scopes fallback attachment ids to the message so reused Slack file ids do not collide", () => { const db = createDb(); - db.insertRawEvent({ - id: "slack-conversation-a", - platform: "slack", - accountKey: "workspace-a", - entityKind: "conversation", - eventKind: "observed", - observedAt: 1, - dedupeKey: "slack-conversation-a", - payload: { - sourceConversationKey: "slack:T1:C_A", - conversationType: "group", - displayName: "alpha", - participants: [], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "slack-conversation-b", - platform: "slack", - accountKey: "workspace-a", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "slack-conversation-b", - payload: { - sourceConversationKey: "slack:T1:C_B", - conversationType: "group", - displayName: "beta", - participants: [], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "slack-message-a", - platform: "slack", - accountKey: "workspace-a", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "slack-message-a", - payload: { - sourceMessageKey: "slack:T1:C_A:1000.000001", - sourceConversationKey: "slack:T1:C_A", - sentAt: 3, - content: "alpha attachment", - attachments: [{ id: "F_SHARED", name: "shared.pdf" }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "slack-message-b", - platform: "slack", - accountKey: "workspace-a", - entityKind: "message", - eventKind: "created", - observedAt: 4, - dedupeKey: "slack-message-b", - payload: { - sourceMessageKey: "slack:T1:C_B:1000.000002", - sourceConversationKey: "slack:T1:C_B", - sentAt: 4, - content: "beta attachment", - attachments: [{ id: "F_SHARED", name: "shared.pdf" }], - }, - sourceVersion: "test-v1", - }); + db.insertRawEvents([ + { + id: "slack-conversation-a", + platform: "slack", + accountKey: "workspace-a", + entityKind: "conversation", + eventKind: "observed", + observedAt: 1, + dedupeKey: "slack-conversation-a", + payload: { + sourceConversationKey: "slack:T1:C_A", + conversationType: "group", + displayName: "alpha", + participants: [], + }, + }, + ]); + db.insertRawEvents([ + { + id: "slack-conversation-b", + platform: "slack", + accountKey: "workspace-a", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "slack-conversation-b", + payload: { + sourceConversationKey: "slack:T1:C_B", + conversationType: "group", + displayName: "beta", + participants: [], + }, + }, + ]); + db.insertRawEvents([ + { + id: "slack-message-a", + platform: "slack", + accountKey: "workspace-a", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "slack-message-a", + payload: { + sourceMessageKey: "slack:T1:C_A:1000.000001", + sourceConversationKey: "slack:T1:C_A", + sentAt: 3, + content: "alpha attachment", + attachments: [{ id: "F_SHARED", name: "shared.pdf" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "slack-message-b", + platform: "slack", + accountKey: "workspace-a", + entityKind: "message", + eventKind: "created", + observedAt: 4, + dedupeKey: "slack-message-b", + payload: { + sourceMessageKey: "slack:T1:C_B:1000.000002", + sourceConversationKey: "slack:T1:C_B", + sentAt: 4, + content: "beta attachment", + attachments: [{ id: "F_SHARED", name: "shared.pdf" }], + }, + }, + ]); projectPendingRawEvents(db); @@ -3024,88 +2904,87 @@ describe("projector", () => { it("preserves deleted conversations locally while marking participants inactive", () => { const db = createDb(); - db.insertRawEvent({ - id: "contact-ava-delete", - platform: "linkedin", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contact-ava-delete", - payload: { - sourceEntityKey: "linkedin:urn:li:member:ACoAAA1", - fields: { display_name: "Ava Chen" }, - handles: [], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "conversation-delete-observed", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "observed", - observedAt: 2, - dedupeKey: "conversation-delete-observed", - payload: { - sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", - conversationType: "dm", - participants: [{ sourceEntityKey: "linkedin:urn:li:member:ACoAAA1" }], - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "message-delete-observed", - platform: "linkedin", - accountKey: "default", - entityKind: "message", - eventKind: "created", - observedAt: 3, - dedupeKey: "message-delete-observed", - payload: { - sourceMessageKey: "linkedin:urn:li:fsd_message:MSG_DELETE", - sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", - senderSourceKey: "linkedin:urn:li:member:ACoAAA1", - sentAt: 3, - content: "preserve me", - isFromMe: false, - }, - sourceVersion: "test-v1", - }); - db.insertRawEvent({ - id: "conversation-delete-removed", - platform: "linkedin", - accountKey: "default", - entityKind: "conversation", - eventKind: "removed", - observedAt: 4, - dedupeKey: "conversation-delete-removed", - payload: { - sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", - conversationType: "dm", - removalReason: "deleted", - unreadCount: 0, - participants: [{ sourceEntityKey: "linkedin:urn:li:member:ACoAAA1" }], - }, - sourceVersion: "test-v1", - }); + db.insertRawEvents([ + { + id: "contact-ava-delete", + platform: "linkedin", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contact-ava-delete", + payload: { + sourceEntityKey: "linkedin:urn:li:member:ACoAAA1", + fields: { display_name: "Ava Chen" }, + handles: [], + }, + }, + ]); + db.insertRawEvents([ + { + id: "conversation-delete-observed", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "observed", + observedAt: 2, + dedupeKey: "conversation-delete-observed", + payload: { + sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", + conversationType: "dm", + participants: [{ sourceEntityKey: "linkedin:urn:li:member:ACoAAA1" }], + }, + }, + ]); + db.insertRawEvents([ + { + id: "message-delete-observed", + platform: "linkedin", + accountKey: "default", + entityKind: "message", + eventKind: "created", + observedAt: 3, + dedupeKey: "message-delete-observed", + payload: { + sourceMessageKey: "linkedin:urn:li:fsd_message:MSG_DELETE", + sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", + senderSourceKey: "linkedin:urn:li:member:ACoAAA1", + sentAt: 3, + content: "preserve me", + isFromMe: false, + }, + }, + ]); + db.insertRawEvents([ + { + id: "conversation-delete-removed", + platform: "linkedin", + accountKey: "default", + entityKind: "conversation", + eventKind: "removed", + observedAt: 4, + dedupeKey: "conversation-delete-removed", + payload: { + sourceConversationKey: "linkedin:urn:li:fs_conversation:CONV_DELETE", + conversationType: "dm", + participants: [{ sourceEntityKey: "linkedin:urn:li:member:ACoAAA1" }], + }, + }, + ]); projectPendingRawEvents(db); const conversationRow = db.orm().get<{ is_active: number; - removal_reason: string | null; - unread_count: number; }>(sql` - SELECT is_active, removal_reason, unread_count + SELECT is_active FROM conversations WHERE source_conversation_key = 'linkedin:urn:li:fs_conversation:CONV_DELETE' `); const participantRow = db.orm().get<{ is_active: number; - left_at: number | null; }>(sql` - SELECT is_active, left_at + SELECT is_active FROM conversation_participants LIMIT 1 `); @@ -3117,19 +2996,16 @@ describe("projector", () => { expect(conversationRow).toEqual({ is_active: 0, - removal_reason: "deleted", - unread_count: 0, }); expect(participantRow).toEqual({ is_active: 0, - left_at: 4, }); expect(messageCount?.count).toBe(1); db.close(); }); - it("updates reaction counts on realtime projection", () => { + it("projects reactions on realtime projection", () => { const db = createDb(); const insertResult = db.insertRawEvents([ @@ -3146,7 +3022,6 @@ describe("projector", () => { conversationType: "dm", participants: [], }, - sourceVersion: "test-v1", }, { id: "message-realtime-reaction", @@ -3164,7 +3039,6 @@ describe("projector", () => { content: "react to this", isFromMe: false, }, - sourceVersion: "test-v1", }, { id: "reaction-realtime-reaction", @@ -3182,7 +3056,6 @@ describe("projector", () => { timestamp: 3, isActive: true, }, - sourceVersion: "test-v1", }, ]); @@ -3192,12 +3065,16 @@ describe("projector", () => { batchSize: 10, }); - const messageRow = db.orm().get<{ reaction_count: number }>(sql` - SELECT reaction_count - FROM messages - WHERE platform_message_id = 'linkedin:urn:li:fsd_message:MSG_REACTION' + const reactionRow = db.orm().get<{ emoji: string; reactor_source_key: string | null }>(sql` + SELECT mr.emoji, mr.reactor_source_key + FROM message_reactions mr + JOIN messages m ON m.id = mr.message_id + WHERE m.platform_message_id = 'linkedin:urn:li:fsd_message:MSG_REACTION' `); - expect(messageRow?.reaction_count).toBe(1); + expect(reactionRow).toEqual({ + emoji: "👍", + reactor_source_key: "linkedin:urn:li:member:ACoAAA1", + }); db.close(); }); diff --git a/src/runtime/projection/projector.ts b/src/runtime/projection/projector.ts index 26d4c923..7bd28276 100644 --- a/src/runtime/projection/projector.ts +++ b/src/runtime/projection/projector.ts @@ -41,8 +41,6 @@ type RawEventRow = { entity_kind: string; event_kind: string; normalized_schema: string | null; - provenance_json: string | null; - source_version: string | null; observed_at: number; payload_json: string; }; @@ -62,10 +60,7 @@ type ProjectionCache = { type ProjectionChangeSet = { dirtyContactIds: Set; dirtyConversationIds: Set; - dirtyConversationSummaryIds: Set; dirtyMessageIds: Set; - touchedAttachments: boolean; - touchedReactions: boolean; }; type ProjectableRawEvent = Pick< @@ -81,15 +76,7 @@ type ProjectableRawEvent = Pick< type RawEventNormalizationContext = Pick< RawEventRow, - | "rowid" - | "id" - | "platform" - | "account_key" - | "entity_kind" - | "event_kind" - | "normalized_schema" - | "provenance_json" - | "source_version" + "rowid" | "id" | "platform" | "account_key" | "entity_kind" | "event_kind" | "normalized_schema" >; type ProjectionOverview = { @@ -150,7 +137,7 @@ function inferHandleFromSourceEntityKey( } function inferTimelineSystemKind(payload: TimelineEventPayload): string { - const systemKind = payload.metadata?.systemKind; + const systemKind = payload.systemKind; return typeof systemKind === "string" && systemKind.trim().length > 0 ? systemKind.trim() : "provider_notice"; @@ -414,10 +401,6 @@ function boolToInt(value: boolean | undefined | null): number { return value ? 1 : 0; } -function hasStringValue(value: string | null | undefined): value is string { - return typeof value === "string" && value.length > 0; -} - function normalizeAttachmentObject(value: unknown): Record | null { return typeof value === "object" && value !== null ? (value as Record) : null; } @@ -510,9 +493,6 @@ function inferAttachmentProjection(attachment: Record): { remoteUrl: string | null; accessKind: string | null; accessRefJson: string | null; - previewRefJson: string | null; - availabilityStatus: string | null; - providerMetadataJson: string | null; } { const localPath = normalizeText(attachment.local_path) ?? @@ -524,7 +504,6 @@ function inferAttachmentProjection(attachment: Record): { normalizeText(attachment.download_url); const explicitAccessKind = normalizeText(attachment.access_kind); const explicitAccessRef = normalizeAttachmentObject(attachment.access_ref); - const explicitPreviewRef = normalizeAttachmentObject(attachment.preview_ref); const providerFetchRef = normalizeAttachmentObject(attachment.provider_fetch) ?? normalizeAttachmentObject(attachment.download_ref); @@ -548,33 +527,12 @@ function inferAttachmentProjection(attachment: Record): { : accessKind === "remote_url" && remoteUrl ? JSON.stringify({ url: remoteUrl }) : null; - const previewRefJson = - explicitPreviewRef != null - ? JSON.stringify(explicitPreviewRef) - : (normalizeText(attachment.previewUrl) ?? - normalizeText(attachment.preview_url) ?? - normalizeText(attachment.thumb_url) ?? - normalizeText(attachment.image_url)) - ? JSON.stringify({ - url: - normalizeText(attachment.previewUrl) ?? - normalizeText(attachment.preview_url) ?? - normalizeText(attachment.thumb_url) ?? - normalizeText(attachment.image_url), - }) - : null; - const availabilityStatus = - normalizeText(attachment.availability_status) ?? (accessKind ? "available" : "metadata_only"); - const providerMetadata = normalizeAttachmentObject(attachment.provider_metadata); return { localPath, remoteUrl, accessKind, accessRefJson, - previewRefJson, - availabilityStatus, - providerMetadataJson: JSON.stringify(providerMetadata ?? attachment), }; } @@ -686,26 +644,6 @@ function describeRawEventContext(event: RawEventNormalizationContext): string { if (event.normalized_schema) { parts.push(`schema ${event.normalized_schema}`); } - if (event.source_version) { - parts.push(`sourceVersion ${event.source_version}`); - } - - if (event.provenance_json) { - try { - const provenance = JSON.parse(event.provenance_json) as { - providerApiVersion?: string | null; - acquisitionMode?: string | null; - }; - if (provenance.providerApiVersion) { - parts.push(`providerApiVersion ${provenance.providerApiVersion}`); - } - if (provenance.acquisitionMode) { - parts.push(`acquisitionMode ${provenance.acquisitionMode}`); - } - } catch { - parts.push("invalid provenance"); - } - } return parts.join(", "); } @@ -714,10 +652,7 @@ function createProjectionChangeSet(): ProjectionChangeSet { return { dirtyContactIds: new Set(), dirtyConversationIds: new Set(), - dirtyConversationSummaryIds: new Set(), dirtyMessageIds: new Set(), - touchedAttachments: false, - touchedReactions: false, }; } @@ -750,14 +685,9 @@ function mergeProjectionChangeSet(target: ProjectionChangeSet, source: Projectio for (const id of source.dirtyConversationIds) { target.dirtyConversationIds.add(id); } - for (const id of source.dirtyConversationSummaryIds) { - target.dirtyConversationSummaryIds.add(id); - } for (const id of source.dirtyMessageIds) { target.dirtyMessageIds.add(id); } - target.touchedAttachments ||= source.touchedAttachments; - target.touchedReactions ||= source.touchedReactions; } function buildOverview(db: CuedDatabase): { @@ -944,18 +874,10 @@ function ensureConversationStub( platform, accountKey, sourceConversationKey, - nativeConversationKey: null, type: "dm", isActive: 1, - removalReason: null, - service: null, name: null, - topic: null, participantNames: null, - lastMessageId: null, - lastMessageAt: null, - lastMessagePreview: null, - unreadCount: 0, createdAt: observedAt, updatedAt: observedAt, }) @@ -997,19 +919,12 @@ function ensureMessageStub( senderName: null, conversationName: null, sentAt: observedAt, - service: null, status: null, isFromMe: 0, content: null, deliveredAt: null, readAt: null, - editedAt: null, - deletedAt: null, - replyToMessageId: null, isDeleted: 0, - isEdited: 0, - attachmentCount: 0, - reactionCount: 0, createdAt: observedAt, updatedAt: observedAt, }) @@ -1069,8 +984,6 @@ function upsertProjectionState( conn: LocalDbExecutor, input: { projectionWatermark: number; - lastProjectedAt: number | null; - lastRebuildAt?: number | null; }, ): void { conn @@ -1078,16 +991,12 @@ function upsertProjectionState( .values({ singletonKey: "global", projectionWatermark: input.projectionWatermark, - lastProjectedAt: input.lastProjectedAt, - lastRebuildAt: input.lastRebuildAt ?? null, updatedAt: Date.now(), }) .onConflictDoUpdate({ target: projectionState.singletonKey, set: { projectionWatermark: sql`MAX(${projectionState.projectionWatermark}, ${input.projectionWatermark})`, - lastProjectedAt: input.lastProjectedAt, - lastRebuildAt: input.lastRebuildAt ?? sql`${projectionState.lastRebuildAt}`, updatedAt: Date.now(), }, }) @@ -1208,18 +1117,11 @@ function projectContactObservation( platform: event.platform, accountKey: event.account_key, sourceEntityKey: payload.sourceEntityKey, - profileUrl: normalizeText(payload.sourceProfileUrl ?? null), - metadataJson: null, - firstSeenAt: event.observed_at, - lastSeenAt: event.observed_at, }) .onConflictDoUpdate({ target: contactSources.id, set: { contactId, - profileUrl: normalizeText(payload.sourceProfileUrl ?? null), - metadataJson: null, - lastSeenAt: event.observed_at, }, }) .run(); @@ -1234,8 +1136,6 @@ function projectContactObservation( type: handle.type, value: handle.value, normalizedValue, - platform: event.platform, - accountKey: event.account_key, isDeterministic: handle.deterministic ? 1 : 0, createdAt: event.observed_at, updatedAt: event.observed_at, @@ -1244,8 +1144,6 @@ function projectContactObservation( target: contactHandles.id, set: { value: handle.value, - platform: event.platform, - accountKey: event.account_key, isDeterministic: handle.deterministic ? 1 : 0, updatedAt: event.observed_at, }, @@ -1274,36 +1172,17 @@ function projectConversationObservation( const conversationSet: { updatedAt: number; - nativeConversationKey?: string | null; type: "dm" | "group"; isActive: number; - removalReason?: string | null; - service?: string | null; name?: string | null; - topic?: string | null; - unreadCount?: number; } = { updatedAt: event.observed_at, type: payload.conversationType, isActive: event.event_kind === "removed" ? 0 : 1, - removalReason: - event.event_kind === "removed" ? normalizeText(payload.removalReason ?? null) : null, }; - if (payload.nativeConversationKey !== undefined) { - conversationSet.nativeConversationKey = normalizeText(payload.nativeConversationKey); - } - if (payload.service !== undefined) { - conversationSet.service = normalizeText(payload.service); - } if (payload.displayName !== undefined) { conversationSet.name = normalizeText(payload.displayName); } - if (payload.topic !== undefined) { - conversationSet.topic = normalizeText(payload.topic); - } - if (payload.unreadCount !== undefined && payload.unreadCount !== null) { - conversationSet.unreadCount = payload.unreadCount; - } conn.update(conversations).set(conversationSet).where(eq(conversations.id, conversationId)).run(); @@ -1318,7 +1197,6 @@ function projectConversationObservation( .update(conversationParticipants) .set({ isActive: 0, - leftAt: event.observed_at, updatedAt: event.observed_at, }) .where(eq(conversationParticipants.conversationId, conversationId)) @@ -1345,11 +1223,8 @@ function projectConversationObservation( contactId, sourceParticipantKey: participant.sourceEntityKey, participantName: cache.contactNameMap.get(contactId) ?? null, - role: null, isSelf: boolToInt(participant.isSelf), isActive: event.event_kind === "removed" ? 0 : 1, - joinedAt: event.observed_at, - leftAt: event.event_kind === "removed" ? event.observed_at : null, updatedAt: event.observed_at, }) .onConflictDoUpdate({ @@ -1362,7 +1237,6 @@ function projectConversationObservation( participantName: cache.contactNameMap.get(contactId) ?? null, isSelf: boolToInt(participant.isSelf), isActive: event.event_kind === "removed" ? 0 : 1, - leftAt: event.event_kind === "removed" ? event.observed_at : null, updatedAt: event.observed_at, }, }) @@ -1393,13 +1267,11 @@ function projectMessageEvent( conn .update(messages) .set({ - deletedAt: payload.deletedAt ?? payload.sentAt ?? event.observed_at, isDeleted: 1, updatedAt: event.observed_at, }) .where(eq(messages.id, messageId)) .run(); - changes.dirtyConversationSummaryIds.add(conversationId); changes.dirtyMessageIds.add(messageId); return; } @@ -1413,17 +1285,6 @@ function projectMessageEvent( event.observed_at, ); const senderName = resolveProjectedSenderName(conn, cache, conversationId, senderContactId); - const replyToMessageId = hasStringValue(payload.replyToSourceMessageKey) - ? ensureMessageStub( - conn, - cache, - event.platform, - event.account_key, - payload.sourceConversationKey, - payload.replyToSourceMessageKey, - event.observed_at, - ).messageId - : null; conn .update(messages) @@ -1437,37 +1298,29 @@ function projectMessageEvent( senderName, conversationName: cache.conversationNameMap.get(conversationId) ?? null, sentAt: payload.sentAt, - service: normalizeText(payload.service ?? null), status: normalizeText(payload.status ?? null), isFromMe: boolToInt(payload.isFromMe), content: resolveProjectedMessageContent(payload), deliveredAt: payload.deliveredAt ?? null, readAt: payload.readAt ?? null, - editedAt: payload.editedAt ?? null, - deletedAt: payload.deletedAt ?? null, - replyToMessageId, isDeleted: boolToInt(payload.isDeleted), - isEdited: boolToInt(payload.isEdited), updatedAt: event.observed_at, }) .where(eq(messages.id, messageId)) .run(); - changes.dirtyConversationSummaryIds.add(conversationId); changes.dirtyMessageIds.add(messageId); - projectMessageAttachments(conn, changes, event, payload, messageId); + projectMessageAttachments(conn, event, payload, messageId); } function projectMessageAttachments( conn: LocalDbExecutor, - changes: ProjectionChangeSet, event: ProjectableRawEvent, payload: MessagePayload, messageId: string, ): void { const desiredAttachmentIds = new Set(); - let removedAttachment = false; for (const [index, attachment] of (payload.attachments ?? []).entries()) { const normalizedAttachment = normalizeAttachmentObject(attachment) ?? {}; const explicitSourceAttachmentKey = normalizeText(normalizedAttachment.sourceAttachmentKey); @@ -1507,10 +1360,6 @@ function projectMessageAttachments( normalizeText(normalizedAttachment.text), accessKind: inferred.accessKind, accessRefJson: inferred.accessRefJson, - previewRefJson: inferred.previewRefJson, - availabilityStatus: inferred.availabilityStatus, - providerMetadataJson: inferred.providerMetadataJson, - metadataJson: JSON.stringify(normalizedAttachment), createdAt: event.observed_at, updatedAt: event.observed_at, }) @@ -1535,10 +1384,6 @@ function projectMessageAttachments( normalizeText(normalizedAttachment.text), accessKind: inferred.accessKind, accessRefJson: inferred.accessRefJson, - previewRefJson: inferred.previewRefJson, - availabilityStatus: inferred.availabilityStatus, - providerMetadataJson: inferred.providerMetadataJson, - metadataJson: JSON.stringify(normalizedAttachment), updatedAt: event.observed_at, }, }) @@ -1559,10 +1404,6 @@ function projectMessageAttachments( WHERE attachment_id = ${existingAttachment.id} `); conn.delete(messageAttachments).where(eq(messageAttachments.id, existingAttachment.id)).run(); - removedAttachment = true; - } - if (payload.attachments !== undefined || removedAttachment) { - changes.touchedAttachments = true; } } @@ -1582,15 +1423,6 @@ function projectReactionEvent( payload.sourceMessageKey, event.observed_at, ); - const reactorContactId = resolveOrEnsureContact( - conn, - cache, - event.platform, - event.account_key, - payload.reactorSourceKey, - event.observed_at, - ); - conn .insert(messageReactions) .values({ @@ -1601,12 +1433,8 @@ function projectReactionEvent( messageId, platform: event.platform, accountKey: event.account_key, - sourceReactionKey: `${payload.sourceMessageKey}:${payload.reactorSourceKey ?? "__me__"}:${payload.emoji}`, - reactorContactId, reactorSourceKey: payload.reactorSourceKey, - reactorName: reactorContactId ? (cache.contactNameMap.get(reactorContactId) ?? null) : null, emoji: payload.emoji, - reactionType: normalizeText(payload.reactionType ?? null), isActive: boolToInt(payload.isActive), createdAt: payload.timestamp, updatedAt: event.observed_at, @@ -1614,17 +1442,13 @@ function projectReactionEvent( .onConflictDoUpdate({ target: messageReactions.id, set: { - reactorContactId, reactorSourceKey: payload.reactorSourceKey, - reactorName: reactorContactId ? (cache.contactNameMap.get(reactorContactId) ?? null) : null, - reactionType: normalizeText(payload.reactionType ?? null), isActive: boolToInt(payload.isActive), updatedAt: event.observed_at, }, }) .run(); - changes.touchedReactions = true; changes.dirtyMessageIds.add(messageId); } @@ -1655,9 +1479,6 @@ function projectParticipantEvent( return; } - const joinedAt = event.event_kind === "joined" ? payload.eventAt : null; - const leftAt = event.event_kind === "left" ? payload.eventAt : null; - conn .insert(conversationParticipants) .values({ @@ -1665,11 +1486,8 @@ function projectParticipantEvent( contactId, sourceParticipantKey: payload.participantSourceKey, participantName: cache.contactNameMap.get(contactId) ?? null, - role: normalizeText(payload.role ?? null), isSelf: boolToInt(payload.isSelf), isActive: event.event_kind === "left" ? 0 : 1, - joinedAt, - leftAt, updatedAt: event.observed_at, }) .onConflictDoUpdate({ @@ -1680,18 +1498,14 @@ function projectParticipantEvent( ], set: { participantName: cache.contactNameMap.get(contactId) ?? null, - role: normalizeText(payload.role ?? null), isSelf: boolToInt(payload.isSelf), isActive: event.event_kind === "left" ? 0 : 1, - joinedAt: joinedAt ?? sql`${conversationParticipants.joinedAt}`, - leftAt, updatedAt: event.observed_at, }, }) .run(); changes.dirtyConversationIds.add(conversationId); - changes.dirtyConversationSummaryIds.add(conversationId); } function projectTimelineEvent( @@ -1748,13 +1562,6 @@ function projectTimelineEvent( systemKind: inferTimelineSystemKind(payload), callProvider: null, callDirection: null, - callStatus: null, - callMedium: null, - callStartedAt: null, - callEndedAt: null, - callDurationSeconds: null, - callDisconnectedCause: null, - metadataJson: payload.metadata ? JSON.stringify(payload.metadata) : null, createdAt: event.observed_at, updatedAt: event.observed_at, }) @@ -1771,20 +1578,12 @@ function projectTimelineEvent( systemKind: inferTimelineSystemKind(payload), callProvider: null, callDirection: null, - callStatus: null, - callMedium: null, - callStartedAt: null, - callEndedAt: null, - callDurationSeconds: null, - callDisconnectedCause: null, - metadataJson: payload.metadata ? JSON.stringify(payload.metadata) : null, updatedAt: event.observed_at, }, }) .run(); changes.dirtyConversationIds.add(conversationId); - changes.dirtyConversationSummaryIds.add(conversationId); } function projectCallEvent( @@ -1819,16 +1618,6 @@ function projectCallEvent( event.observed_at, ); - const metadata = { - ...(payload.metadata ?? {}), - providerCallType: payload.providerCallType ?? null, - answeredAt: payload.answeredAt ?? null, - endedAt: payload.endedAt ?? null, - disconnectedCause: payload.disconnectedCause ?? null, - remoteAddress: payload.remoteAddress ?? null, - remoteDisplayName: payload.remoteDisplayName ?? null, - }; - conn .insert(timelineEvents) .values({ @@ -1851,13 +1640,6 @@ function projectCallEvent( systemKind: "call", callProvider: payload.provider, callDirection: payload.direction, - callStatus: payload.status, - callMedium: payload.medium, - callStartedAt: payload.startedAt, - callEndedAt: payload.endedAt ?? null, - callDurationSeconds: payload.durationSeconds ?? null, - callDisconnectedCause: payload.disconnectedCause ?? null, - metadataJson: JSON.stringify(metadata), createdAt: event.observed_at, updatedAt: event.observed_at, }) @@ -1874,20 +1656,12 @@ function projectCallEvent( systemKind: "call", callProvider: payload.provider, callDirection: payload.direction, - callStatus: payload.status, - callMedium: payload.medium, - callStartedAt: payload.startedAt, - callEndedAt: payload.endedAt ?? null, - callDurationSeconds: payload.durationSeconds ?? null, - callDisconnectedCause: payload.disconnectedCause ?? null, - metadataJson: JSON.stringify(metadata), updatedAt: event.observed_at, }, }) .run(); changes.dirtyConversationIds.add(conversationId); - changes.dirtyConversationSummaryIds.add(conversationId); } function projectCallDeleteEvent( @@ -1920,88 +1694,6 @@ function projectCallDeleteEvent( `); changes.dirtyConversationIds.add(conversationId); - changes.dirtyConversationSummaryIds.add(conversationId); -} - -function refreshConversationSummariesForIds( - conn: LocalDbExecutor, - conversationIds: Set, -): void { - const latestMessageId = sql`( - SELECT m.id - FROM messages m - WHERE m.conversation_id = conversations.id - ORDER BY m.sent_at DESC, m.updated_at DESC, m.id DESC - LIMIT 1 - )`; - const latestMessageAt = sql`( - SELECT m.sent_at - FROM messages m - WHERE m.conversation_id = conversations.id - ORDER BY m.sent_at DESC, m.updated_at DESC, m.id DESC - LIMIT 1 - )`; - const latestMessagePreview = sql`( - SELECT m.content - FROM messages m - WHERE m.conversation_id = conversations.id - ORDER BY m.sent_at DESC, m.updated_at DESC, m.id DESC - LIMIT 1 - )`; - const latestSystemMessageAt = sql`( - SELECT te.event_at - FROM timeline_events te - WHERE te.conversation_id = conversations.id - AND te.event_kind = 'system_message' - ORDER BY te.event_at DESC, te.updated_at DESC, te.id DESC - LIMIT 1 - )`; - const latestSystemMessagePreview = sql`( - SELECT te.text - FROM timeline_events te - WHERE te.conversation_id = conversations.id - AND te.event_kind = 'system_message' - ORDER BY te.event_at DESC, te.updated_at DESC, te.id DESC - LIMIT 1 - )`; - for (const chunk of chunkArray([...conversationIds], SQL_CHUNK_SIZE)) { - conn.run(sql` - UPDATE conversations - SET - last_message_id = CASE - WHEN ${latestSystemMessageAt} IS NOT NULL - AND (${latestMessageAt} IS NULL OR ${latestSystemMessageAt} > ${latestMessageAt}) - THEN NULL - ELSE ${latestMessageId} - END, - last_message_at = CASE - WHEN ${latestSystemMessageAt} IS NOT NULL - AND (${latestMessageAt} IS NULL OR ${latestSystemMessageAt} > ${latestMessageAt}) - THEN ${latestSystemMessageAt} - ELSE ${latestMessageAt} - END, - last_message_preview = CASE - WHEN ${latestSystemMessageAt} IS NOT NULL - AND (${latestMessageAt} IS NULL OR ${latestSystemMessageAt} > ${latestMessageAt}) - THEN ${latestSystemMessagePreview} - ELSE ${latestMessagePreview} - END, - unread_count = ( - CASE - WHEN conversations.is_active = 0 THEN 0 - ELSE ( - SELECT COUNT(*) - FROM messages m - WHERE m.conversation_id = conversations.id - AND m.is_from_me = 0 - AND m.is_deleted = 0 - AND m.read_at IS NULL - ) - END - ) - WHERE id IN (${sqlValueList(chunk)}) - `); - } } function refreshContactFanoutForIds(conn: LocalDbExecutor, contactIds: Set): void { @@ -2027,13 +1719,6 @@ function refreshContactFanoutForIds(conn: LocalDbExecutor, contactIds: Set): void { - for (const chunk of chunkArray([...messageIds], SQL_CHUNK_SIZE)) { - conn.run(sql` - UPDATE messages - SET attachment_count = ( - SELECT COUNT(*) - FROM message_attachments ma - WHERE ma.message_id = messages.id - ) - WHERE id IN (${sqlValueList(chunk)}) - `); - } -} - -function refreshReactionCountsForIds(conn: LocalDbExecutor, messageIds: Set): void { - for (const chunk of chunkArray([...messageIds], SQL_CHUNK_SIZE)) { - conn.run(sql` - UPDATE messages - SET reaction_count = ( - SELECT COUNT(*) - FROM message_reactions mr - WHERE mr.message_id = messages.id - AND mr.is_active = 1 - ) - WHERE id IN (${sqlValueList(chunk)}) - `); - } -} - function finalizeDeferredProjection( conn: LocalDbExecutor, cache: ProjectionCache, @@ -2274,26 +1930,11 @@ function finalizeDeferredProjection( syncConversationNameCache(conn, cache, changes.dirtyConversationIds); refreshMessageSenderNamesForIds(conn, changes.dirtyConversationIds); refreshMessageConversationNamesForIds(conn, changes.dirtyConversationIds); - for (const conversationId of changes.dirtyConversationIds) { - changes.dirtyConversationSummaryIds.add(conversationId); - } - } - - if (changes.dirtyConversationSummaryIds.size > 0) { - refreshConversationSummariesForIds(conn, changes.dirtyConversationSummaryIds); } if (!options?.initialProjection) { expandDirtyMessageIds(conn, changes); } - if (changes.dirtyMessageIds.size > 0) { - if (changes.touchedAttachments) { - refreshAttachmentCountsForIds(conn, changes.dirtyMessageIds); - } - if (changes.touchedReactions) { - refreshReactionCountsForIds(conn, changes.dirtyMessageIds); - } - } } function summarizeResult( @@ -2325,7 +1966,6 @@ function projectEventBatch( mode: ProjectionMode; rawEvents: RawEventRow[]; projectionWatermark: number | null; - lastRebuildAt?: number | null; initialProjection?: boolean; resetCache?: boolean; clearState?: boolean; @@ -2358,10 +1998,7 @@ function projectEventBatch( } catch (error) { const message = error instanceof Error ? error.message : String(error); db.quarantineRawEventProjectionFailure( - { - ...event, - entity_kind: event.entity_kind as RawEventEntityKind, - }, + event, new Error(`Failed to normalize raw event (${describeRawEventContext(event)}): ${message}`), ); continue; @@ -2380,7 +2017,6 @@ function projectEventBatch( const projectionFailures: Array<{ event: RawEventRow; error: unknown }> = []; db.orm().transaction((tx) => { - const projectedAt = Date.now(); if (input.clearState) { clearProjectedState(tx); } @@ -2425,21 +2061,13 @@ function projectEventBatch( if (input.projectionWatermark != null) { upsertProjectionState(tx, { projectionWatermark: input.projectionWatermark, - lastProjectedAt: projectedAt, - lastRebuildAt: input.lastRebuildAt, }); } }); for (const failure of projectionFailures) { - db.quarantineRawEventProjectionFailure( - { - ...failure.event, - entity_kind: failure.event.entity_kind as RawEventEntityKind, - }, - failure.error, - ); + db.quarantineRawEventProjectionFailure(failure.event, failure.error); } - db.enqueueMessageFtsIndex(changes.dirtyMessageIds, `projection:${input.mode}`); + db.enqueueMessageFtsIndex(changes.dirtyMessageIds); return shapedEvents.length - projectionFailures.length; } @@ -2492,7 +2120,6 @@ function projectRangeInternal( mode: input.mode, rawEvents, projectionWatermark: committedProjectionWatermark, - lastRebuildAt: input.mode === "rebuild" ? Date.now() : undefined, initialProjection: input.mode !== "realtime" && currentProjectionState.projection_watermark === 0, }); @@ -2628,7 +2255,6 @@ export function rebuildProjectedState( mode: "rebuild", rawEvents: [], projectionWatermark: 0, - lastRebuildAt: Date.now(), resetCache: true, clearState: true, }); @@ -2643,7 +2269,6 @@ export function rebuildProjectedState( }; } - const lastRebuildAt = Date.now(); let rawEvents = firstRawEvents; let appliedRawEvents = 0; let projectionWatermark = 0; @@ -2654,7 +2279,6 @@ export function rebuildProjectedState( mode: "rebuild", rawEvents, projectionWatermark, - lastRebuildAt, initialProjection: clearState, resetCache: clearState, clearState, diff --git a/src/runtime/projection/replay-fixtures/contacts-linkedin.ts b/src/runtime/projection/replay-fixtures/contacts-linkedin.ts index d75845b1..e0ab7ae1 100644 --- a/src/runtime/projection/replay-fixtures/contacts-linkedin.ts +++ b/src/runtime/projection/replay-fixtures/contacts-linkedin.ts @@ -12,24 +12,19 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations[0]).toMatchObject({ name: "Ava Chen", participantNames: "Ava Chen", - unreadCount: 1, }); expect(snapshot.messages).toHaveLength(1); expect(snapshot.messages[0]).toMatchObject({ senderName: "Ava Chen", conversationName: "Ava Chen", - attachmentCount: 1, - reactionCount: 1, }); expect(snapshot.messageAttachments[0]).toMatchObject({ filename: "update.pdf", title: "Update", remoteUrl: "https://example.com/update.pdf", - availabilityStatus: "available", }); expect(snapshot.messageReactions[0]).toMatchObject({ emoji: "👍", - reactorName: "Ava Chen", isActive: 1, }); expect(snapshot.ftsMessageIds).toEqual([snapshot.messages[0]!.id]); @@ -55,7 +50,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+1 (555) 123-4567", deterministic: true }, ], }, - sourceVersion: "contacts-v1", }), fixtureEvent({ id: "conversation_thread_1", @@ -68,10 +62,8 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ payload: { sourceConversationKey: "thread-1", conversationType: "dm", - service: "linkedin", participants: [{ sourceEntityKey: "contacts:ava" }], }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "message_msg_1", @@ -87,7 +79,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "contacts:ava", sentAt: 1_710_000_000_150, content: "Founder update tomorrow?", - service: "linkedin", status: "delivered", isFromMe: false, attachments: [ @@ -98,11 +89,9 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ filename: "update.pdf", title: "Update", remote_url: "https://example.com/update.pdf", - availability_status: "available", }, ], }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "reaction_msg_1_thumbs_up", @@ -120,7 +109,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ timestamp: 1_710_000_000_250, isActive: true, }, - sourceVersion: "linkedin-v1", }), ], }, @@ -133,8 +121,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations[0]).toMatchObject({ name: "Northwind diligence", isActive: 0, - removalReason: "deleted", - unreadCount: 0, }); expect(snapshot.messages).toHaveLength(1); expect(snapshot.messages[0]).toMatchObject({ @@ -165,7 +151,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ }, handles: [{ type: "email", value: "milo@northwind.example", deterministic: true }], }, - sourceVersion: "contacts-v1", }), fixtureEvent({ id: "conversation_thread_removed", @@ -179,11 +164,8 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "thread-removed", conversationType: "dm", displayName: "Northwind diligence", - nativeConversationKey: "urn:li:fsd_conversation:thread-removed", - service: "linkedin", participants: [{ sourceEntityKey: "contacts:milo" }], }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "message_thread_removed_1", @@ -199,11 +181,9 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "contacts:milo", sentAt: 1_710_000_100_150, content: "We should archive this thread.", - service: "linkedin", status: "delivered", isFromMe: false, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "conversation_thread_removed_deleted", @@ -217,13 +197,8 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "thread-removed", conversationType: "dm", displayName: "Northwind diligence", - nativeConversationKey: "urn:li:fsd_conversation:thread-removed", - service: "linkedin", - unreadCount: 0, - removalReason: "deleted", participants: [{ sourceEntityKey: "contacts:milo" }], }, - sourceVersion: "linkedin-v1", }), ], }, @@ -235,19 +210,12 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations).toHaveLength(1); expect(snapshot.conversations[0]).toMatchObject({ name: "Ava Chen", - lastMessageId: null, - lastMessageAt: 1_710_000_200_275, - lastMessagePreview: "Ava renamed the conversation", - unreadCount: 0, }); expect(snapshot.messages).toHaveLength(1); expect(snapshot.messages[0]).toMatchObject({ content: "final copy", status: "read", readAt: 1_710_000_200_260, - editedAt: 1_710_000_200_210, - isEdited: 1, - reactionCount: 0, }); expect(snapshot.messageReactions).toHaveLength(1); expect(snapshot.messageReactions[0]).toMatchObject({ @@ -277,7 +245,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ }, handles: [{ type: "email", value: "ava@cued.com", deterministic: true }], }, - sourceVersion: "contacts-v1", }), fixtureEvent({ id: "conversation_thread_updated", @@ -290,10 +257,8 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ payload: { sourceConversationKey: "thread-updated", conversationType: "dm", - service: "linkedin", participants: [{ sourceEntityKey: "contacts:ava" }], }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "message_thread_updated_created", @@ -309,11 +274,9 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "contacts:ava", sentAt: 1_710_000_200_090, content: "draft copy", - service: "linkedin", status: "delivered", isFromMe: false, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "message_thread_updated_updated", @@ -329,13 +292,9 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "contacts:ava", sentAt: 1_710_000_200_090, content: "final copy", - service: "linkedin", status: "delivered", isFromMe: false, - editedAt: 1_710_000_200_210, - isEdited: true, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "reaction_thread_updated_added", @@ -353,7 +312,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ timestamp: 1_710_000_200_225, isActive: true, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "reaction_thread_updated_removed", @@ -371,7 +329,6 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ timestamp: 1_710_000_200_240, isActive: false, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "message_thread_updated_read_receipt", @@ -387,14 +344,10 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "contacts:ava", sentAt: 1_710_000_200_090, content: "final copy", - service: "linkedin", status: "read", isFromMe: false, readAt: 1_710_000_200_260, - editedAt: 1_710_000_200_210, - isEdited: true, }, - sourceVersion: "linkedin-v1", }), fixtureEvent({ id: "timeline_thread_updated_system_message", @@ -412,11 +365,8 @@ export const contactsLinkedInReplayFixtures: ProjectionReplayFixture[] = [ subjectSourceKey: "contacts:ava", eventAt: 1_710_000_200_275, text: "Ava renamed the conversation", - metadata: { - systemKind: "provider_notice", - }, + systemKind: "provider_notice", }, - sourceVersion: "linkedin-v1", }), ], }, diff --git a/src/runtime/projection/replay-fixtures/imessage.ts b/src/runtime/projection/replay-fixtures/imessage.ts index 3f955b94..588b785d 100644 --- a/src/runtime/projection/replay-fixtures/imessage.ts +++ b/src/runtime/projection/replay-fixtures/imessage.ts @@ -30,10 +30,8 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "42", conversationType: "dm", displayName: null, - nativeConversationKey: "chat-42", participants: [{ sourceEntityKey: "imessage:+15551230000" }], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_message_dm_late_contact", @@ -52,15 +50,12 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "imessage:+15551230000", sentAt: 1_710_199_999_500, content: "late contact should still resolve", - service: "iMessage", status: "delivered", isFromMe: false, readAt: null, - isEdited: false, isDeleted: false, attachments: [], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_contact_dm_late_contact", @@ -89,7 +84,6 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ }, ], }, - sourceVersion: "imessage-v1", }), ], }, @@ -104,12 +98,10 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ }); expect(snapshot.messages[0]).toMatchObject({ senderName: "Ava Email", - attachmentCount: 1, }); expect(snapshot.messageAttachments[0]).toMatchObject({ filename: "agenda.pdf", localPath: "/Users/test/Library/Messages/Attachments/agenda.pdf", - availabilityStatus: "available", }); }, events: [ @@ -140,7 +132,6 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ }, ], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_conversation_attachment", @@ -154,10 +145,8 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "84", conversationType: "dm", displayName: null, - nativeConversationKey: "chat-84", participants: [{ sourceEntityKey: "imessage:ava@example.com" }], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_message_attachment", @@ -176,11 +165,9 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "imessage:ava@example.com", sentAt: 1_710_210_000_020, content: "see attached", - service: "iMessage", status: "delivered", isFromMe: false, readAt: null, - isEdited: false, isDeleted: false, attachments: [ { @@ -191,21 +178,12 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ mime_type: "application/pdf", size_bytes: 4096, access_kind: "local_path", - availability_status: "available", access_ref: { path: "/Users/test/Library/Messages/Attachments/agenda.pdf", }, - provider_metadata: { - uti: "com.adobe.pdf", - isSticker: false, - hideAttachment: false, - ckRecordId: null, - sourceFilename: "/Users/test/Library/Messages/Attachments/agenda.pdf", - }, }, ], }, - sourceVersion: "imessage-v1", }), ], }, @@ -223,11 +201,9 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.messages[0]).toMatchObject({ senderName: "Avery Example", conversationName: "Family", - reactionCount: 1, }); expect(snapshot.messageReactions[0]).toMatchObject({ emoji: "❤️", - reactorName: "Jordan Example", isActive: 1, }); }, @@ -244,13 +220,11 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "126", conversationType: "group", displayName: "Family", - nativeConversationKey: "chat-126", participants: [ { sourceEntityKey: "imessage:+15550000001" }, { sourceEntityKey: "imessage:+15550000002" }, ], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_message_reaction", @@ -269,15 +243,12 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "imessage:+15550000001", sentAt: 1_710_220_000_010, content: "dinner at 7?", - service: "iMessage", status: "delivered", isFromMe: false, readAt: null, - isEdited: false, isDeleted: false, attachments: [], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_reaction_added", @@ -298,7 +269,6 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ timestamp: 1_710_220_000_015, isActive: true, }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_contact_sender", @@ -319,7 +289,6 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ { type: "imessage_handle", value: "+15550000001", deterministic: true }, ], }, - sourceVersion: "imessage-v1", }), fixtureEvent({ id: "imessage_contact_reactor", @@ -340,7 +309,6 @@ export const imessageReplayFixtures: ProjectionReplayFixture[] = [ { type: "imessage_handle", value: "+15550000002", deterministic: true }, ], }, - sourceVersion: "imessage-v1", }), ], }, diff --git a/src/runtime/projection/replay-fixtures/shared.ts b/src/runtime/projection/replay-fixtures/shared.ts index 82860b0a..25c32530 100644 --- a/src/runtime/projection/replay-fixtures/shared.ts +++ b/src/runtime/projection/replay-fixtures/shared.ts @@ -14,8 +14,5 @@ export function fixtureEvent(input: ProviderRawEventInput): ProviderRawEventInpu return { ...input, normalizedSchema: buildNormalizedRawEventSchema(input.entityKind, input.eventKind), - provenance: { - adapterVersion: "projection-replay-fixture@1", - }, }; } diff --git a/src/runtime/projection/replay-fixtures/signal.ts b/src/runtime/projection/replay-fixtures/signal.ts index 6e1466b2..1f4afaf1 100644 --- a/src/runtime/projection/replay-fixtures/signal.ts +++ b/src/runtime/projection/replay-fixtures/signal.ts @@ -30,10 +30,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "signal:dm-casey", conversationType: "dm", displayName: "Casey Signal", - service: "signal", participants: [{ sourceEntityKey: "signal:+14155550123" }], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-message-before-contact", @@ -49,10 +47,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "signal:+14155550123", sentAt: 1_710_200_000_050, content: "hello from signal", - service: "signal", isFromMe: false, }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-contact-after", @@ -67,7 +63,6 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ fields: { display_name: "Casey Signal" }, handles: [{ type: "phone", value: "+14155550123", deterministic: true }], }, - sourceVersion: "signal-v1", }), ], }, @@ -79,20 +74,16 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations[0]).toMatchObject({ name: "Investor thread", participantNames: "Ava Zhang", - unreadCount: 2, }); expect(snapshot.messages).toHaveLength(2); const reply = snapshot.messages.find((message) => message.content === "reply first"); expect(reply).toMatchObject({ senderName: "Ava Zhang", conversationName: "Investor thread", - attachmentCount: 1, }); - expect(reply?.replyToMessageId).toBeTruthy(); expect(snapshot.messageAttachments[0]).toMatchObject({ filename: "agenda.pdf", title: "Agenda", - availabilityStatus: "metadata_only", }); }, events: [ @@ -109,7 +100,6 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ fields: { display_name: "Ava Chen" }, handles: [{ type: "phone", value: "+14155550000", deterministic: true }], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-conversation-ava", @@ -123,10 +113,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "signal:dm-ava", conversationType: "dm", displayName: "Ava Chen", - service: "signal", participants: [{ sourceEntityKey: "signal:+14155550000" }], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-parent-message", @@ -142,10 +130,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "signal:+14155550000", sentAt: 1_710_300_000_100, content: "parent later", - service: "signal", isFromMe: false, }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-reply-message", @@ -161,9 +147,7 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "signal:+14155550000", sentAt: 1_710_300_000_150, content: "reply first", - service: "signal", isFromMe: false, - replyToSourceMessageKey: "signal:message:parent", attachments: [ { id: "signal-attachment-1", @@ -173,7 +157,6 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ }, ], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-contact-ava-rename", @@ -188,7 +171,6 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ fields: { display_name: "Ava Zhang" }, handles: [{ type: "phone", value: "+14155550000", deterministic: true }], }, - sourceVersion: "signal-v2", }), fixtureEvent({ id: "signal-conversation-ava-rename", @@ -202,10 +184,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "signal:dm-ava", conversationType: "dm", displayName: "Investor thread", - service: "signal", participants: [{ sourceEntityKey: "signal:+14155550000" }], }, - sourceVersion: "signal-v2", }), ], }, @@ -243,7 +223,6 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ }, ], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-conversation-uuid", @@ -257,10 +236,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: "signal:dm-uuid", conversationType: "dm", displayName: "Ava Chen", - service: "signal", participants: [{ sourceEntityKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678" }], }, - sourceVersion: "signal-v1", }), fixtureEvent({ id: "signal-message-uuid", @@ -276,10 +253,8 @@ export const signalReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "signal:a1b2c3d4-e5f6-1234-9abc-def012345678", sentAt: 1_710_400_000_150, content: "hello from signal uuid", - service: "signal", isFromMe: false, }, - sourceVersion: "signal-v1", }), ], }, diff --git a/src/runtime/projection/replay-fixtures/slack.ts b/src/runtime/projection/replay-fixtures/slack.ts index ca23204c..2bbe9263 100644 --- a/src/runtime/projection/replay-fixtures/slack.ts +++ b/src/runtime/projection/replay-fixtures/slack.ts @@ -12,7 +12,6 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations[0]).toMatchObject({ name: "Slack User", participantNames: "Slack User", - unreadCount: 1, }); expect(snapshot.messages).toHaveLength(1); expect(snapshot.messages[0]).toMatchObject({ @@ -32,10 +31,8 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ payload: { sourceConversationKey: "slack:T0A9C9RHZ9T:C123", conversationType: "dm", - service: "slack", participants: [{ sourceEntityKey: "slack:T0A9C9RHZ9T:U123" }], }, - sourceVersion: "slack-v1", }), fixtureEvent({ id: "slack_message_before_contact", @@ -51,10 +48,8 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "slack:T0A9C9RHZ9T:U123", sentAt: 1_710_099_999_500, content: "hello from slack", - service: "slack", isFromMe: false, }, - sourceVersion: "slack-v1", }), fixtureEvent({ id: "slack_contact_after", @@ -71,7 +66,6 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ }, handles: [{ type: "slack_user", value: "U123", deterministic: true }], }, - sourceVersion: "slack-v1", }), ], }, @@ -83,7 +77,6 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.conversations).toHaveLength(1); expect(snapshot.conversations[0]).toMatchObject({ name: "Jordan Slack", - unreadCount: 2, }); expect(snapshot.messages).toHaveLength(2); const reply = snapshot.messages.find( @@ -91,13 +84,10 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ ); expect(reply).toMatchObject({ senderName: "Jordan Slack", - attachmentCount: 1, }); - expect(reply?.replyToMessageId).toBeTruthy(); expect(snapshot.messageAttachments[0]).toMatchObject({ filename: "deck.pdf", remoteUrl: "https://files.example.com/deck.pdf", - availabilityStatus: "available", }); }, events: [ @@ -112,10 +102,8 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ payload: { sourceConversationKey: "slack:T0A9C9RHZ9T:C777", conversationType: "dm", - service: "slack", participants: [{ sourceEntityKey: "slack:T0A9C9RHZ9T:U777" }], }, - sourceVersion: "slack-v1", }), fixtureEvent({ id: "slack_thread_parent", @@ -131,10 +119,8 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "slack:T0A9C9RHZ9T:U777", sentAt: 1_710_200_000_050, content: "Can you review the deck?", - service: "slack", isFromMe: false, }, - sourceVersion: "slack-v1", }), fixtureEvent({ id: "slack_thread_reply", @@ -150,9 +136,7 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: "slack:T0A9C9RHZ9T:U777", sentAt: 1_710_200_000_150, content: "Uploaded the latest file.", - service: "slack", isFromMe: false, - replyToSourceMessageKey: "slack:T0A9C9RHZ9T:C777:1710200000.000100", attachments: [ { kind: "file", @@ -164,16 +148,9 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ previewUrl: "https://files.example.com/deck-thumb.png", access_kind: "remote_url", access_ref: { url: "https://files.example.com/deck.pdf" }, - preview_ref: { url: "https://files.example.com/deck-thumb.png" }, - availability_status: "available", - provider_metadata: { - id: "F777", - prettyType: "PDF", - }, }, ], }, - sourceVersion: "slack-v1", }), fixtureEvent({ id: "slack_thread_contact_after", @@ -190,7 +167,6 @@ export const slackReplayFixtures: ProjectionReplayFixture[] = [ }, handles: [{ type: "slack_user", value: "U777", deterministic: true }], }, - sourceVersion: "slack-v1", }), ], }, diff --git a/src/runtime/projection/replay-fixtures/whatsapp.ts b/src/runtime/projection/replay-fixtures/whatsapp.ts index b3a196f9..cabf2c62 100644 --- a/src/runtime/projection/replay-fixtures/whatsapp.ts +++ b/src/runtime/projection/replay-fixtures/whatsapp.ts @@ -42,11 +42,9 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: whatsappSourceEntityKey(dmPeer), sentAt: 1_713_000_000_000, content: "hello from WhatsApp", - service: "whatsapp", status: "delivered", isFromMe: false, }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_chat", @@ -60,11 +58,8 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: whatsappSourceConversationKey(dmPeer), conversationType: "dm", displayName: null, - nativeConversationKey: dmPeer, - service: "whatsapp", participants: [{ sourceEntityKey: whatsappSourceEntityKey(dmPeer) }], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_contact", @@ -84,7 +79,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+12016824050", deterministic: true }, ], }, - sourceVersion: "whatsapp-v1", }), ], }, @@ -94,7 +88,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.contacts.map((contact) => contact.name).sort()).toEqual(["Alice", "Bob"]); expect(snapshot.conversations[0]).toMatchObject({ name: "Study Group", - unreadCount: 1, }); expect(snapshot.messages).toHaveLength(2); const inbound = snapshot.messages.find((message) => message.content === "agenda attached"); @@ -102,7 +95,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ expect(inbound).toMatchObject({ senderName: "Alice", conversationName: "Study Group", - attachmentCount: 1, isFromMe: 0, }); expect(outbound).toMatchObject({ @@ -114,7 +106,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ filename: "agenda.pdf", title: "Agenda", remoteUrl: "https://example.com/agenda.pdf", - availabilityStatus: "available", }); }, events: [ @@ -130,14 +121,11 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: whatsappSourceConversationKey(studyGroup), conversationType: "group", displayName: "Study Group", - nativeConversationKey: studyGroup, - service: "whatsapp", participants: [ { sourceEntityKey: whatsappSourceEntityKey(studyGroupAlice) }, { sourceEntityKey: whatsappSourceEntityKey(studyGroupBob) }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_group_contact_alice", @@ -157,7 +145,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+15551234567", deterministic: true }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_group_contact_bob", @@ -177,7 +164,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+15557654321", deterministic: true }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_group_message_attachment", @@ -193,7 +179,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: whatsappSourceEntityKey(studyGroupAlice), sentAt: 1_713_010_000_250, content: "agenda attached", - service: "whatsapp", status: "delivered", isFromMe: false, attachments: [ @@ -204,16 +189,11 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ filename: "agenda.pdf", title: "Agenda", remote_url: "https://example.com/agenda.pdf", - availability_status: "available", mime_type: "application/pdf", size_bytes: 0, - provider_metadata: { - note: "meeting agenda", - }, }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_group_message_fromme", @@ -229,11 +209,9 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: null, sentAt: 1_713_010_000_350, content: "thanks, got it", - service: "whatsapp", status: "sent", isFromMe: true, }, - sourceVersion: "whatsapp-v1", }), ], }, @@ -244,7 +222,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ expect(snapshot.contacts[0]?.name).toBe("Irene"); expect(snapshot.conversations[0]).toMatchObject({ name: "Irene", - unreadCount: 2, }); expect(snapshot.messages).toHaveLength(3); const reply = snapshot.messages.find((message) => message.content === "on my way"); @@ -253,7 +230,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ conversationName: "Irene", isFromMe: 1, }); - expect(reply?.replyToMessageId).toBeTruthy(); }, events: [ fixtureEvent({ @@ -268,11 +244,8 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: whatsappSourceConversationKey(irene), conversationType: "dm", displayName: null, - nativeConversationKey: irene, - service: "whatsapp", participants: [{ sourceEntityKey: whatsappSourceEntityKey(irene) }], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_fromme_parent_message", @@ -288,11 +261,9 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: whatsappSourceEntityKey(irene), sentAt: 1_713_020_000_025, content: "where are you?", - service: "whatsapp", status: "delivered", isFromMe: false, }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_fromme_reply_message", @@ -308,12 +279,9 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: null, sentAt: 1_713_020_000_050, content: "on my way", - service: "whatsapp", status: "sent", isFromMe: true, - replyToSourceMessageKey: `${irene}:wamid-dm-parent`, }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_fromme_contact", @@ -333,7 +301,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+14155550123", deterministic: true }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_dm_followup_message", @@ -349,11 +316,9 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ senderSourceKey: whatsappSourceEntityKey(irene), sentAt: 1_713_020_000_250, content: "see you soon", - service: "whatsapp", status: "delivered", isFromMe: false, }, - sourceVersion: "whatsapp-v1", }), ], }, @@ -366,10 +331,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ subjectSourceKey: whatsappSourceEntityKey(irene), text: "WhatsApp call • 1m 1s", }); - expect(snapshot.conversations[0]).toMatchObject({ - lastMessagePreview: "WhatsApp call • 1m 1s", - lastMessageAt: 1_713_030_000_000, - }); }, events: [ fixtureEvent({ @@ -390,7 +351,6 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ { type: "phone", value: "+14155550123", deterministic: true }, ], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_call_chat", @@ -404,11 +364,8 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ sourceConversationKey: whatsappSourceConversationKey(irene), conversationType: "dm", displayName: "Irene", - nativeConversationKey: irene, - service: "whatsapp", participants: [{ sourceEntityKey: whatsappSourceEntityKey(irene) }], }, - sourceVersion: "whatsapp-v1", }), fixtureEvent({ id: "whatsapp_call", @@ -422,17 +379,14 @@ export const whatsappReplayFixtures: ProjectionReplayFixture[] = [ sourceCallKey: `${irene}:call-1`, sourceConversationKey: whatsappSourceConversationKey(irene), provider: "whatsapp", - providerCallType: "regular", direction: "outgoing", medium: "audio", status: "completed", startedAt: 1_713_030_000_000, - endedAt: 1_713_030_061_000, durationSeconds: 61, initiatorSourceKey: null, primaryRemoteSourceKey: whatsappSourceEntityKey(irene), }, - sourceVersion: "whatsapp-v1", }), ], }, diff --git a/src/runtime/projection/replay-snapshot.ts b/src/runtime/projection/replay-snapshot.ts index 7fc1d659..422ca62d 100644 --- a/src/runtime/projection/replay-snapshot.ts +++ b/src/runtime/projection/replay-snapshot.ts @@ -18,7 +18,6 @@ export type CanonicalProjectionSnapshot = { platform: string; accountKey: string; sourceEntityKey: string; - profileUrl: string | null; }>; }>; conversations: Array<{ @@ -28,13 +27,8 @@ export type CanonicalProjectionSnapshot = { sourceConversationKey: string; type: string; isActive: number; - removalReason: string | null; name: string | null; - lastMessageId: string | null; - lastMessageAt: number | null; - lastMessagePreview: string | null; participantNames: string | null; - unreadCount: number; }>; conversationParticipants: Array<{ conversationId: string; @@ -55,12 +49,7 @@ export type CanonicalProjectionSnapshot = { content: string | null; status: string | null; readAt: number | null; - editedAt: number | null; - isEdited: number; isFromMe: number; - attachmentCount: number; - reactionCount: number; - replyToMessageId: string | null; }>; messageAttachments: Array<{ id: string; @@ -70,14 +59,12 @@ export type CanonicalProjectionSnapshot = { title: string | null; localPath: string | null; remoteUrl: string | null; - availabilityStatus: string | null; }>; messageReactions: Array<{ id: string; messageId: string; emoji: string; isActive: number; - reactorName: string | null; }>; timelineEvents: Array<{ id: string; @@ -124,9 +111,8 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj platform: string; account_key: string; source_entity_key: string; - profile_url: string | null; }>(sql` - SELECT contact_id, platform, account_key, source_entity_key, profile_url + SELECT contact_id, platform, account_key, source_entity_key FROM contact_sources ORDER BY contact_id ASC, platform ASC, account_key ASC, source_entity_key ASC `); @@ -138,15 +124,10 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj source_conversation_key: string; type: string; is_active: number; - removal_reason: string | null; name: string | null; - last_message_id: string | null; - last_message_at: number | null; - last_message_preview: string | null; participant_names: string | null; - unread_count: number; }>(sql` - SELECT id, platform, account_key, source_conversation_key, type, is_active, removal_reason, name, last_message_id, last_message_at, last_message_preview, participant_names, unread_count + SELECT id, platform, account_key, source_conversation_key, type, is_active, name, participant_names FROM conversations ORDER BY id ASC `); @@ -163,14 +144,9 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj content: string | null; status: string | null; read_at: number | null; - edited_at: number | null; - is_edited: number; is_from_me: number; - attachment_count: number; - reaction_count: number; - reply_to_message_id: string | null; }>(sql` - SELECT id, platform, account_key, platform_message_id, conversation_id, sender_name, conversation_name, sent_at, content, status, read_at, edited_at, is_edited, is_from_me, attachment_count, reaction_count, reply_to_message_id + SELECT id, platform, account_key, platform_message_id, conversation_id, sender_name, conversation_name, sent_at, content, status, read_at, is_from_me FROM messages ORDER BY id ASC `); @@ -195,9 +171,8 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj title: string | null; local_path: string | null; remote_url: string | null; - availability_status: string | null; }>(sql` - SELECT id, message_id, source_attachment_key, filename, title, local_path, remote_url, availability_status + SELECT id, message_id, source_attachment_key, filename, title, local_path, remote_url FROM message_attachments ORDER BY id ASC `); @@ -207,9 +182,8 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj message_id: string; emoji: string; is_active: number; - reactor_name: string | null; }>(sql` - SELECT id, message_id, emoji, is_active, reactor_name + SELECT id, message_id, emoji, is_active FROM message_reactions ORDER BY id ASC `); @@ -257,7 +231,6 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj platform: source.platform, accountKey: source.account_key, sourceEntityKey: source.source_entity_key, - profileUrl: source.profile_url, })), })), conversations: conversations.map((conversation) => ({ @@ -267,13 +240,8 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj sourceConversationKey: conversation.source_conversation_key, type: conversation.type, isActive: conversation.is_active, - removalReason: conversation.removal_reason, name: conversation.name, - lastMessageId: conversation.last_message_id, - lastMessageAt: conversation.last_message_at, - lastMessagePreview: conversation.last_message_preview, participantNames: conversation.participant_names, - unreadCount: conversation.unread_count, })), conversationParticipants: conversationParticipants.map((participant) => ({ conversationId: participant.conversation_id, @@ -294,12 +262,7 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj content: message.content, status: message.status, readAt: message.read_at, - editedAt: message.edited_at, - isEdited: message.is_edited, isFromMe: message.is_from_me, - attachmentCount: message.attachment_count, - reactionCount: message.reaction_count, - replyToMessageId: message.reply_to_message_id, })), messageAttachments: messageAttachments.map((attachment) => ({ id: attachment.id, @@ -309,14 +272,12 @@ export function readCanonicalProjectionSnapshot(db: CuedDatabase): CanonicalProj title: attachment.title, localPath: attachment.local_path, remoteUrl: attachment.remote_url, - availabilityStatus: attachment.availability_status, })), messageReactions: messageReactions.map((reaction) => ({ id: reaction.id, messageId: reaction.message_id, emoji: reaction.emoji, isActive: reaction.is_active, - reactorName: reaction.reactor_name, })), timelineEvents: timelineEvents.map((event) => ({ id: event.id, diff --git a/src/runtime/projection/worker.ts b/src/runtime/projection/worker.ts index ae7b27b5..c66b1ef4 100644 --- a/src/runtime/projection/worker.ts +++ b/src/runtime/projection/worker.ts @@ -97,13 +97,7 @@ export async function runProjectionWorker(run: QueuedSyncRun): Promise { db.upsertSourceAccounts([ { platform: "imessage", accountKey: "local", displayName: "Messages" }, ]); - db.insertRawEvent({ - id: "imessage-reset-raw", - platform: "imessage", - accountKey: "local", - entityKind: "message", - eventKind: "created", - observedAt: Date.now(), - dedupeKey: "imessage-reset-raw", - payload: { - sourceMessageKey: "imessage:m1", - sourceConversationKey: "imessage:c1", + db.insertRawEvents([ + { + id: "imessage-reset-raw", + platform: "imessage", + accountKey: "local", + entityKind: "message", + eventKind: "created", + observedAt: Date.now(), + dedupeKey: "imessage-reset-raw", + payload: { + sourceMessageKey: "imessage:m1", + sourceConversationKey: "imessage:c1", + }, }, - }); + ]); const queue = new RunQueueService(db); const result = queue.resetSource("imessage"); @@ -222,36 +224,38 @@ describe("RunQueueService", () => { it("records a manual contact merge and rebuilds projected state immediately", () => { const db = createDb(); - db.insertRawEvent({ - id: "contact-primary", - platform: "contacts", - accountKey: "local", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: "contacts:primary", - payload: { - sourceEntityKey: "contacts:primary", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], + db.insertRawEvents([ + { + id: "contact-primary", + platform: "contacts", + accountKey: "local", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: "contacts:primary", + payload: { + sourceEntityKey: "contacts:primary", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "phone", value: "+1 (555) 123-4567", deterministic: true }], + }, }, - sourceVersion: "contacts-v1", - }); - db.insertRawEvent({ - id: "contact-secondary", - platform: "linkedin", - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 2, - dedupeKey: "linkedin:secondary", - payload: { - sourceEntityKey: "linkedin:secondary", - fields: { display_name: "Ava Chen" }, - handles: [{ type: "linkedin", value: "urn:li:person:ava-chen", deterministic: true }], + ]); + db.insertRawEvents([ + { + id: "contact-secondary", + platform: "linkedin", + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 2, + dedupeKey: "linkedin:secondary", + payload: { + sourceEntityKey: "linkedin:secondary", + fields: { display_name: "Ava Chen" }, + handles: [{ type: "linkedin", value: "urn:li:person:ava-chen", deterministic: true }], + }, }, - sourceVersion: "linkedin-v1", - }); + ]); rebuildProjectedState(db); const contacts = db.orm().all<{ id: string }>(sql` @@ -289,27 +293,28 @@ describe("RunQueueService", () => { ["contact-b", "linkedin", "urn:li:person:ava-chen"], ["contact-c", "slack", "ava@example.com"], ] as const) { - db.insertRawEvent({ - id, - platform, - accountKey: "default", - entityKind: "contact", - eventKind: "observed", - observedAt: 1, - dedupeKey: `${platform}:${id}`, - payload: { - sourceEntityKey: `${platform}:${id}`, - fields: { display_name: "Ava Chen" }, - handles: [ - { - type: platform === "contacts" ? "phone" : platform, - value: handle, - deterministic: true, - }, - ], + db.insertRawEvents([ + { + id, + platform, + accountKey: "default", + entityKind: "contact", + eventKind: "observed", + observedAt: 1, + dedupeKey: `${platform}:${id}`, + payload: { + sourceEntityKey: `${platform}:${id}`, + fields: { display_name: "Ava Chen" }, + handles: [ + { + type: platform === "contacts" ? "phone" : platform, + value: handle, + deterministic: true, + }, + ], + }, }, - sourceVersion: `${platform}-v1`, - }); + ]); } rebuildProjectedState(db); @@ -345,7 +350,9 @@ describe("RunQueueService", () => { expect.objectContaining({ canonicalContactId: contacts[0]!.id }), ], }); - expect(db.listContactMergeDecisions()).toEqual([]); + expect( + db.orm().get<{ count: number }>(sql`SELECT COUNT(*) AS count FROM contact_merge_decisions`), + ).toEqual({ count: 0 }); expect(db.getOverview().contacts).toBe(3); const applied = queue.mergeContactsBatch({ @@ -372,7 +379,9 @@ describe("RunQueueService", () => { projectionWatermark: 3, }), }); - expect(db.listContactMergeDecisions()).toHaveLength(2); + expect( + db.orm().get<{ count: number }>(sql`SELECT COUNT(*) AS count FROM contact_merge_decisions`), + ).toEqual({ count: 2 }); expect(db.getOverview().contacts).toBe(1); db.close(); diff --git a/src/runtime/run-queue.ts b/src/runtime/run-queue.ts index d57bdafa..b2ec76db 100644 --- a/src/runtime/run-queue.ts +++ b/src/runtime/run-queue.ts @@ -9,7 +9,6 @@ import { rebuildProjectedState } from "./projection/projector.js"; type RunQueueSchedulers = { wakeIngest?: () => void; - wakeOutbound?: () => void; wakeProjection?: () => void; }; @@ -43,61 +42,6 @@ export class RunQueueService { return [...new Set(authenticatedTargets)]; } - queueMessageSend(input: { - platform: string; - target: string; - text: string; - accountKey?: string; - }): { - queued: true; - messageId: string; - } { - if ( - input.platform !== "signal" && - input.platform !== "whatsapp" && - input.platform !== "discord" - ) { - throw new Error(`Unsupported outbound platform: ${input.platform}`); - } - if (input.target.trim().length === 0 || input.text.trim().length === 0) { - throw new Error(`${input.platform} send requires a target and non-empty text`); - } - - const resolved = - input.platform === "signal" - ? this.db.resolveSignalSendTarget(input.target.trim()) - : input.platform === "whatsapp" - ? this.db.resolveWhatsAppSendTarget(input.target.trim()) - : this.db.resolveDiscordSendTarget(input.target.trim()); - if (!resolved) { - throw new Error(`Unable to resolve ${input.platform} target: ${input.target.trim()}`); - } - - const messageId = this.db.queueOutboundMessage({ - platform: input.platform, - accountKey: input.accountKey ?? getDefaultAccountKeyForPlatform(input.platform), - target: resolved.target, - threadId: resolved.threadId, - text: input.text, - metadata: { - originalTarget: input.target.trim(), - resolvedTarget: resolved.target, - resolvedThreadId: resolved.threadId, - resolution: resolved.resolution, - matchedContactIds: "matchedContactIds" in resolved ? resolved.matchedContactIds : undefined, - matchedConversationId: - "matchedConversationId" in resolved ? resolved.matchedConversationId : undefined, - matchedName: resolved.matchedName, - }, - }); - this.schedulers.wakeOutbound?.(); - - return { - queued: true, - messageId, - }; - } - queueSyncRun(source?: string): { queued: boolean; runId: string | null; diff --git a/src/runtime/updater/service.test.ts b/src/runtime/updater/service.test.ts index d8484a1b..ee43508a 100644 --- a/src/runtime/updater/service.test.ts +++ b/src/runtime/updater/service.test.ts @@ -319,10 +319,8 @@ describe("updater service", () => { dbBackupPath: "/tmp/local.db", releaseUrl: null, }); - db.recordAppMetadata({ - version: "0.1.0", - releaseChannel: "stable", - }); + db.setAppSetting("installed_app_version", "0.1.0"); + db.setAppSetting("release_channel", "stable"); db.close(); expect(runUpdateHelperHealthCheck(dbPath, "0.2.0")).toBe(true); @@ -331,7 +329,7 @@ describe("updater service", () => { const reopened = new CuedDatabase(dbPath); expect(reopened.getPendingRollbackState()).toBeNull(); - expect(reopened.getUpdateLastError()?.message).toBe("restored"); + expect(reopened.getAppMetadata().updateLastError?.message).toBe("restored"); expect(reopened.getAppMetadata().installedAppVersion).toBe("0.1.0"); expect(reopened.getAppMetadata().releaseChannel).toBe("stable"); reopened.close();