Keep failed text sends so their bubble and Retry survive a restart

app/src/main/java/com/pombo/android/AppViewModel.kt

@@ -43,6 +43,7 @@ class AppViewModel(app: Application) : AndroidViewModel(app), PomboBridge.Listen
     private val unreadStore = com.pombo.android.data.UnreadStore(app)
     private val blobStore = com.pombo.android.core.ImageBlobStore(app)
     private val sentDmStore = com.pombo.android.data.SentDmStore(app)
+    private val failedOutbox = com.pombo.android.data.FailedOutboxStore(app)
     private val sentReactionsStore = com.pombo.android.data.SentReactionsStore(app)
     private val epochKeyStore = com.pombo.android.data.EpochKeyStore(app)
     private val adminFloorStore = com.pombo.android.core.AdminFloorStore(app)
@@ -129,6 +130,7 @@ class AppViewModel(app: Application) : AndroidViewModel(app), PomboBridge.Listen
         blobStore = blobStore,
         sentDmStore = sentDmStore,
         sentReactionsStore = sentReactionsStore,
+        failedOutbox = failedOutbox,
         inviteStore = inviteStore,
         unreadStore = unreadStore,
         epochKeyStore = epochKeyStore,
@@ -1625,6 +1627,8 @@ class AppViewModel(app: Application) : AndroidViewModel(app), PomboBridge.Listen
         sentDmStore.scopeAddress = address
         unreadStore.scopeAddress = address
         sentDmStore.memoryOnly = guest
+        failedOutbox.scopeAddress = address
+        failedOutbox.memoryOnly = guest
         sentReactionsStore.scopeAddress = address
         sentReactionsStore.memoryOnly = guest
         pushRegistry.scopeAddress = address
@@ -1672,6 +1676,7 @@ class AppViewModel(app: Application) : AndroidViewModel(app), PomboBridge.Listen
         contactsStore.scopeAddress = store.address
         inviteStore.scopeAddress = store.address
         sentDmStore.scopeAddress = store.address
+        failedOutbox.scopeAddress = store.address
         sentReactionsStore.scopeAddress = store.address
         pushRegistry.scopeAddress = store.address
         settingsStore.scopeAddress = store.address

app/src/main/java/com/pombo/android/ChannelManager.kt

@@ -159,6 +159,8 @@ class ChannelManager(
     internal val sentDmStore: com.pombo.android.data.SentDmStore,
     /** Own DM/write-only reactions — no resend returns them (web sentReactions). */
     private val sentReactionsStore: com.pombo.android.data.SentReactionsStore? = null,
+    /** Failed text sends, so their Retry survives a restart. */
+    private val failedOutbox: com.pombo.android.data.FailedOutboxStore? = null,
     private val inviteStore: com.pombo.android.data.InviteStore,
     private val unreadStore: com.pombo.android.data.UnreadStore,
     /** Persisted epoch keys for gated channels (-4 protocol, N-A). */
@@ -4050,6 +4052,7 @@ class ChannelManager(
             .distinctBy { it.id }
             .sortedBy { it.timestamp }
         if (merged.isNotEmpty()) mergeMessages(merged)
+        restoreFailedOutbox(channel)
 
         // Now that the messages exist, fill any image from the local ledger.
         // Order matters: hydrating before the merge would map over a list that
@@ -4330,9 +4333,11 @@ class ChannelManager(
             }
         } catch (e: Exception) {
             markFailed(id, e.message)
+            keepForRetry(channel.messageStreamId, wire, sender, e.message)
             throw e
         }
         confirmMessage(id)
+        failedOutbox?.remove(channel.messageStreamId, id)
         // Persist only after the publish succeeded, so a failed send does not
         // leave a message in the history that the peer never received. This is
         // the only copy we will ever have of it — and it keeps `sender`,
@@ -5276,6 +5281,7 @@ class ChannelManager(
             // older snapshot taken before the user left.
             store.markLeft(messageStreamId)
             saveChannels()
+            failedOutbox?.clear(messageStreamId)
             // Rotate the channel pseudonym on a GENUINE leave (never on a mere
             // view switch — closeCurrent keeps it: peers mid-transfer know the
             // current publisher). Rejoining gets a fresh key, so yesterday's
@@ -5722,6 +5728,7 @@ class ChannelManager(
             // what the node answered, which is what the empty state shows.
             lastStorageReadError(channel.messageStreamId, StreamConstants.P_MESSAGES)?.let { _historyError.value = it }
         }
+        restoreFailedOutbox(channel)
 
         if (overrides != null) {
             for (i in 0 until overrides.entries.length()) {
@@ -6050,9 +6057,11 @@ class ChannelManager(
             }
         } catch (e: Exception) {
             markFailed(id, e.message)
+            keepForRetry(channel.messageStreamId, content, content.optString("sender"), e.message)
             throw e
         }
         confirmMessage(id)
+        failedOutbox?.remove(channel.messageStreamId, id)
         delivery.track(channel, id, envelopeTs)
         // Without this a message sent from Android never wakes anyone: web
         // clients rely on the relay seeing a wake signal on the push stream.
@@ -6214,6 +6223,7 @@ class ChannelManager(
         deletedIds.add(targetId)
         if (original.isImage) original.imageId?.let { tombstoneImage(it) }
         _messages.value = _messages.value.filterNot { it.id == targetId }
+        failedOutbox?.remove(channel.messageStreamId, targetId)
         if (channel.type == "dm") {
             sentDmStore.delete(channel.messageStreamId, targetId)
             onLocalStateChanged()
@@ -7337,6 +7347,7 @@ class ChannelManager(
         val mine = sender.equals(me, ignoreCase = true)
         if (mine && _messages.value.any { it.id == id }) {
             confirmMessage(id)
+            failedOutbox?.remove(channel.messageStreamId, id)
             return
         }
         val msg = UiMessage(
@@ -7565,6 +7576,73 @@ class ChannelManager(
         it.copy(pending = false, failed = true, undelivered = true, failError = reason)
     }
 
+    /** Keeps a failed text send, as the wire fields its Retry republishes. */
+    private fun keepForRetry(
+        streamId: String, wire: JSONObject, sender: String, error: String?, undelivered: Boolean = false
+    ) {
+        failedOutbox?.put(streamId, JSONObject(wire.toString())
+            .put("sender", sender)
+            .put("failError", error ?: JSONObject.NULL)
+            .put("undelivered", undelivered)
+            .put("failedAt", System.currentTimeMillis()))
+    }
+
+    /** A sent text storage never recorded joins the outbox like one that failed outright. */
+    @Synchronized
+    internal fun keepUndeliveredForRetry(streamId: String, id: String) {
+        val msg = _messages.value.firstOrNull { it.id == id }
+            ?: reopenSnapshots.values.firstNotNullOfOrNull { s -> s.messages.firstOrNull { it.id == id } }
+            ?: return
+        if (msg.isImage || msg.file != null || msg.storageFile != null) return
+        keepForRetry(streamId, textWire(msg.id, msg.text, msg.sender, msg.timestamp, msg.replyTo),
+            msg.sender, msg.failError, undelivered = true)
+    }
+
+    /**
+     * Brings back the failed sends of this conversation that the timeline does
+     * not hold. An id the timeline holds unfailed was delivered after all.
+     */
+    private fun restoreFailedOutbox(channel: Channel) {
+        val outbox = failedOutbox ?: return
+        val me = myAddress() ?: return
+        val shown = _messages.value.associateBy { it.id }
+        val restored = mutableListOf<UiMessage>()
+        for (entry in outbox.load(channel.messageStreamId)) {
+            val id = entry.optString("id")
+            if (id.isEmpty()) continue
+            val existing = shown[id]
+            if (existing != null) {
+                if (!existing.failed) outbox.remove(channel.messageStreamId, id)
+                continue
+            }
+            val sender = entry.optString("sender").ifEmpty { me }
+            restored += UiMessage(
+                id = id,
+                text = entry.optString("text"),
+                sender = sender,
+                senderName = entry.optStringOrNull("senderName"),
+                timestamp = entry.optLong("timestamp"),
+                mine = true,
+                failed = true,
+                failError = entry.optStringOrNull("failError"),
+                undelivered = entry.optBoolean("undelivered", false),
+                verified = true,
+                ensName = ensStore.cachedName(sender),
+                ensAvatar = ensStore.cachedAvatar(sender),
+                replyTo = entry.optJSONObject("replyTo")?.let {
+                    val rid = it.optString("id")
+                    if (rid.isEmpty()) null else ReplyRef(
+                        id = rid,
+                        sender = it.optString("sender"),
+                        senderName = it.optStringOrNull("senderName"),
+                        text = it.optString("text")
+                    )
+                }
+            )
+        }
+        if (restored.isNotEmpty()) mergeMessages(restored)
+    }
+
     /** A send can settle after the user left its channel: the reopen copy must hear it too. */
     @Synchronized
     private fun updateSendState(id: String, transform: (UiMessage) -> UiMessage) {

app/src/main/java/com/pombo/android/core/channels/DeliveryConfirm.kt

@@ -95,6 +95,7 @@ internal class DeliveryConfirm(private val manager: ChannelManager) {
                 if (!remove(streamId, entry.id)) continue
                 if (entry.verified) {
                     manager.markUndelivered(entry.id, UNDELIVERED_REASON)
+                    manager.keepUndeliveredForRetry(streamId, entry.id)
                     Log.w(TAG, "$label: ${entry.id} never reached storage")
                 } else {
                     Log.w(TAG, "$label: ${entry.id} unverifiable, storage never answered")

app/src/main/java/com/pombo/android/data/FailedOutboxStore.kt

@@ -0,0 +1,77 @@
+package com.pombo.android.data
+
+import android.content.Context
+import org.json.JSONArray
+import org.json.JSONObject
+
+/**
+ * Text messages this device failed to send, per conversation: what brings a
+ * "Not sent" bubble and its Retry back after a restart. Each entry holds the
+ * wire fields the Retry republishes (id, text, sender, senderName, timestamp,
+ * replyTo) plus `failError`, `undelivered` and `failedAt`.
+ *
+ * Device-local on purpose: never in the sync or the backup, or every device
+ * of the account would offer a Retry for the same message.
+ *
+ * Encrypted at rest because it holds plaintext message bodies.
+ */
+class FailedOutboxStore(context: Context) {
+
+    private val prefs by lazy {
+        com.pombo.android.core.SecurePrefs.create(context, "pombo_failed_outbox")
+    }
+
+    @Volatile var scopeAddress: String? = null
+    @Volatile var memoryOnly: Boolean = false
+
+    private fun key(streamId: String): String {
+        val scope = scopeAddress?.lowercase() ?: "none"
+        return "failed_${scope}_$streamId"
+    }
+
+    fun load(streamId: String): List<JSONObject> {
+        if (memoryOnly) return emptyList()
+        val raw = prefs.getString(key(streamId), null) ?: return emptyList()
+        return try {
+            val arr = JSONArray(raw)
+            (0 until arr.length()).mapNotNull { arr.optJSONObject(it) }
+        } catch (e: Exception) {
+            emptyList()
+        }
+    }
+
+    /** Adds or replaces the entry with the same id; past the cap the oldest go. */
+    fun put(streamId: String, entry: JSONObject) {
+        if (memoryOnly) return
+        val id = entry.optString("id")
+        if (id.isEmpty()) return
+        val kept = load(streamId).filterNot { it.optString("id") == id } + entry
+        save(streamId, kept.sortedBy { it.optLong("timestamp") }.takeLast(MAX_PER_CONVERSATION))
+    }
+
+    fun remove(streamId: String, messageId: String) {
+        if (memoryOnly) return
+        val entries = load(streamId)
+        val kept = entries.filterNot { it.optString("id") == messageId }
+        if (kept.size != entries.size) save(streamId, kept)
+    }
+
+    fun clear(streamId: String) {
+        if (memoryOnly) return
+        prefs.edit().remove(key(streamId)).apply()
+    }
+
+    private fun save(streamId: String, entries: List<JSONObject>) {
+        if (entries.isEmpty()) {
+            prefs.edit().remove(key(streamId)).apply()
+            return
+        }
+        val arr = JSONArray()
+        entries.forEach { arr.put(it) }
+        prefs.edit().putString(key(streamId), arr.toString()).apply()
+    }
+
+    companion object {
+        const val MAX_PER_CONVERSATION = 20
+    }
+}

app/src/test/java/com/pombo/android/ChannelManagerHarness.kt

@@ -35,7 +35,8 @@ import org.json.JSONObject
  */
 class ChannelManagerHarness(
     channels: List<Channel> = emptyList(),
-    private val trustedContacts: Set<String> = emptySet()
+    private val trustedContacts: Set<String> = emptySet(),
+    failedOutbox: com.pombo.android.data.FailedOutboxStore? = null
 ) {
     /** Throwaway key; the channel pseudonym is minted from it, so it must be real. */
     val myKey = "0x4f3edf983ac636a65a842ce7c78d9aa706d3b113bce9c46f30d7d21715b23b1d"
@@ -86,6 +87,7 @@ class ChannelManagerHarness(
             ensStore = ensStore,
             blobStore = mockk(relaxed = true),
             sentDmStore = mockk(relaxed = true),
+            failedOutbox = failedOutbox,
             inviteStore = mockk(relaxed = true),
             unreadStore = unreadStore,
             epochKeyStore = mockk(relaxed = true),