diff --git a/backend/src/soroban/soroban.service.ts b/backend/src/soroban/soroban.service.ts index 19fd83e..e32cb8f 100644 --- a/backend/src/soroban/soroban.service.ts +++ b/backend/src/soroban/soroban.service.ts @@ -139,7 +139,13 @@ export class SorobanService implements OnModuleInit { try { const result = await this.client.getPayment(invoiceId); return !!result; - } catch { + } catch (error: any) { + if (error?.code === "PaymentArchived" || error?.numericCode === 24) { + this.logger.log( + `Invoice ${invoiceId} is recorded on-chain but archived due to TTL expiration`, + ); + return true; + } return false; } } @@ -150,7 +156,12 @@ export class SorobanService implements OnModuleInit { } try { return await this.client.getPayment(invoiceId); - } catch { + } catch (error: any) { + if (error?.code === "PaymentArchived" || error?.numericCode === 24) { + this.logger.warn( + `Invoice ${invoiceId} payment record is archived on-chain and needs restoration before reading`, + ); + } return null; } } diff --git a/soroban/README.md b/soroban/README.md index 14d5fe1..3ab48cd 100644 --- a/soroban/README.md +++ b/soroban/README.md @@ -579,9 +579,9 @@ event**, giving the Invoisio backend two independent reconciliation paths: | **Admin-gated writes** | Only the backend service account (`admin`) may call `record_payment` | | **Two-step admin handoff** | `propose_admin` (current admin) + `accept_admin` (proposed admin) — no single transaction can change the admin, so a lost/compromised key can never hand off alone | | **One record per `invoice_id`** | Idempotent; prevents double-counting in reconciliation | -| **Persistent storage** | Records survive ledger archival windows | +| **Tiered storage retention** | Instance storage is permanent; persistent storage has a 90-day retention window with bulk extension (`extend_history_ttl`) and archival restoration | | **Minimized events** | Only `schema_version` + `invoice_id` in each event (issue #512); a subscriber that needs the full record must already know `invoice_id` and call `get_payment` | -| **Privacy-by-default reads** | Bulk/volume reads (`payment_history`, `payment_count`, `settlement_ref_history`, `settlement_ref_index_status`, `history_index_status`) are admin-gated; `settlement_ref` is stored as a SHA-256 commitment, not plaintext (issue #512) — see `contracts/invoice-payment/README.md`'s "Disclosure guarantee / threat model" section | +| **Privacy-by-default reads** | Bulk/volume reads (`payment_history`, `payment_count`, `settlement_ref_history`, `settlement_ref_index_status`, `history_index_status`, `extend_history_ttl`) are admin-gated; `settlement_ref` is stored as a SHA-256 commitment, not plaintext (issue #512) — see `contracts/invoice-payment/README.md`'s "Disclosure guarantee / threat model" section | #### Admin transfer flow @@ -755,6 +755,15 @@ The contract uses `#[contracterror]`; these codes are returned as `ScError::Cont | 20 | SettlementRefAlreadyUsed | The settlement reference is already recorded. Ambiguous on its own — call `settlement_ref_owner()` to tell a benign retry (same invoice) from a genuine conflict (different invoice) apart (issue #495). | | 21 | MustBePausedForUpgrade | `upgrade()` was called while the contract is not paused; the contract must stay paused for the whole `upgrade()` → `upgrade_storage()` window. | | 22 | LegacyPaymentMigrationBatchTooLarge | `migrate_legacy_payments()` was called with more invoice_ids than `MAX_LEGACY_MIGRATION_BATCH` (20) in one call; split the batch across multiple calls. | +| 23 | IssuerMigrationIncomplete | `upgrade_storage()` rewrote a bounded batch of Token issuers from String to Address and has more payment-log slots left; call `upgrade_storage()` again while paused. | +| 24 | PaymentArchived | `get_payment()` called for an `invoice_id` that exists in the write log but has expired due to TTL policy. The record can be restored via a Stellar `RestoreFootprint` operation. | + +#### Retention and Archival Management + +The contract implements a 3-tier retention policy: +1. **Hot Tier (Instance Storage):** Contract config and sequence counters are permanent and refreshed on every read/write. +2. **Active Rent Tier (Persistent Storage):** Payment records and history slots are extended to `BUMP_TTL` (~90 days / 1,555,200 ledgers) upon access. Administrators perform bulk sweeps via `extend_history_ttl` (`./invoke-extend-history-ttl.sh`). +3. **Cold Archival Tier (Network Archival):** Expired records return `PaymentArchived` (code 24) rather than `PaymentNotFound` (code 4), and are restorable on demand via `./invoke-restore-record.sh` without re-anchoring or index corruption. See [`docs/retention-and-restore.md`](docs/retention-and-restore.md) for full details. #### Typed error manifest (off-chain reference) diff --git a/soroban/client/src/codec.ts b/soroban/client/src/codec.ts index 60bc892..7debb25 100644 --- a/soroban/client/src/codec.ts +++ b/soroban/client/src/codec.ts @@ -67,6 +67,12 @@ export const MAX_SETTLEMENT_REF_LEN = 128; */ export const MAX_LEGACY_MIGRATION_BATCH = 20; +/** + * Maximum number of payment records/history slots extended in one + * `extendHistoryTtl` call. Mirrors `storage::MAX_TTL_EXTEND_BATCH`. + */ +export const MAX_TTL_EXTEND_BATCH = 20; + const CANONICAL_IDENTIFIER_PATTERN = /^[a-z0-9-]+$/; /** @@ -243,6 +249,8 @@ export function decodePaymentHistoryPage(scVal: xdr.ScVal): PaymentHistoryPage { records: records.map((record) => decodePaymentRecordFromNative(record)), nextCursor: Number(raw['next_cursor']), hasMore: Boolean(raw['has_more']), + gapsSkipped: Number(raw['gaps_skipped'] ?? 0), + archivedSkipped: Number(raw['archived_skipped'] ?? 0), }; } diff --git a/soroban/client/src/error-manifest.test.ts b/soroban/client/src/error-manifest.test.ts index 08ee4dd..593aabb 100644 --- a/soroban/client/src/error-manifest.test.ts +++ b/soroban/client/src/error-manifest.test.ts @@ -39,6 +39,7 @@ const EXPECTED_CONTRACT_ERRORS = [ { code: 21, name: 'MustBePausedForUpgrade' }, { code: 22, name: 'LegacyPaymentMigrationBatchTooLarge' }, { code: 23, name: 'IssuerMigrationIncomplete' }, + { code: 24, name: 'PaymentArchived' }, ] as const; describe('CONTRACT_ERROR_MANIFEST', () => { diff --git a/soroban/client/src/error-manifest.ts b/soroban/client/src/error-manifest.ts index d045859..37b0cfd 100644 --- a/soroban/client/src/error-manifest.ts +++ b/soroban/client/src/error-manifest.ts @@ -177,6 +177,12 @@ export const CONTRACT_ERROR_MANIFEST = [ meaning: 'upgrade_storage() rewrote a bounded batch of Token issuers from String to Address and has more payment-log slots left; call upgrade_storage() again. Stay paused until storage_schema_version is current.', }, + { + code: 24, + name: 'PaymentArchived', + meaning: + 'get_payment() was called for an invoice_id that exists in the on-chain write log but whose persistent record has expired and been archived due to TTL expiration. It can be restored via a RestoreFootprint operation before reading.', + }, ] as const satisfies readonly ContractErrorManifestEntry[]; /** Union of every known contract error name (excludes the `Unknown` fallback). */ diff --git a/soroban/client/src/soroban-invoice-client.ts b/soroban/client/src/soroban-invoice-client.ts index a6a4b5b..7f040a0 100644 --- a/soroban/client/src/soroban-invoice-client.ts +++ b/soroban/client/src/soroban-invoice-client.ts @@ -41,6 +41,7 @@ import { MAX_INVOICE_ID_LEN, MAX_LEGACY_MIGRATION_BATCH, MAX_SETTLEMENT_REF_LEN, + MAX_TTL_EXTEND_BATCH, parseContractError, } from './codec'; @@ -547,6 +548,46 @@ export class SorobanInvoiceClient { return this.submitWrite(tx); } + /** + * Extend persistent storage TTL across a bounded range of payment history records, + * write logs, and settlement references. + * + * @param cursor - zero-based history slot to start from (default 0) + * @param limit - maximum slots to process (default 20, capped at {@link MAX_TTL_EXTEND_BATCH}) + * + * The **contract admin** keypair must be provided via `signerSecretKey`. + * + * @throws {Error} if `limit` exceeds {@link MAX_TTL_EXTEND_BATCH} + * @throws {SorobanContractError} on contract-level rejection (e.g. `Unauthorized`) + */ + async extendHistoryTtl(cursor = 0, limit = MAX_TTL_EXTEND_BATCH): Promise { + this.requireSigner(); + if (limit > MAX_TTL_EXTEND_BATCH) { + throw new Error( + `limit must be at most ${MAX_TTL_EXTEND_BATCH}, got ${limit}`, + ); + } + const account = await this.server.getAccount(this.keypair!.publicKey()); + const caller = this.keypair!.publicKey(); + + const tx = new TransactionBuilder(account, { + fee: BASE_FEE, + networkPassphrase: this.config.networkPassphrase, + }) + .addOperation( + this.contract.call( + 'extend_history_ttl', + encodeAddress(caller), + encodeU32(cursor), + encodeU32(limit), + ), + ) + .setTimeout(TX_TIMEOUT_SECONDS) + .build(); + + return this.submitWrite(tx); + } + // ─── Read operations (permissionless) ────────────────────────────────────── /** @@ -585,7 +626,9 @@ export class SorobanInvoiceClient { /** * Fetch the full `PaymentRecord` for an invoice. * - * @throws {SorobanContractError} with code `PaymentNotFound` if not recorded + * @throws {SorobanContractError} with code `PaymentArchived` if the record exists on-chain + * in the write log but has expired due to TTL policy (restorable via RestoreFootprint) + * @throws {SorobanContractError} with code `PaymentNotFound` if never recorded */ async getPayment(invoiceId: string): Promise { const retval = await this.simulateView('get_payment', encodeString(invoiceId)); diff --git a/soroban/client/src/types.ts b/soroban/client/src/types.ts index 28c07ef..a3ee74f 100644 --- a/soroban/client/src/types.ts +++ b/soroban/client/src/types.ts @@ -89,6 +89,24 @@ export interface PaymentHistoryPage { readonly records: PaymentRecord[]; readonly nextCursor: number; readonly hasMore: boolean; + /** + * Number of index slots in this page's range that represent unindexed + * corruption gaps (where neither PaymentHistory nor PaymentLog hold an entry). + */ + readonly gapsSkipped: number; + /** + * Number of index slots in this page's range that hold a valid entry in + * PaymentLog but whose persistent PaymentHistory record has expired and + * been archived due to TTL expiration. + */ + readonly archivedSkipped: number; +} + +/** Result of extending TTL across a bounded range of history records. */ +export interface ExtendHistoryTtlResult { + readonly extendedCount: number; + readonly nextCursor: number; + readonly hasMore: boolean; } /** diff --git a/soroban/contracts/invoice-payment/generate-schema.sh b/soroban/contracts/invoice-payment/generate-schema.sh index 294032f..33bf68b 100755 --- a/soroban/contracts/invoice-payment/generate-schema.sh +++ b/soroban/contracts/invoice-payment/generate-schema.sh @@ -66,6 +66,7 @@ declare -A ERROR_DESC=( [MustBePausedForUpgrade]="upgrade() was called while the contract is not paused; the contract must stay paused for the whole upgrade() -> upgrade_storage() window." [LegacyPaymentMigrationBatchTooLarge]="migrate_legacy_payments() was called with more invoice_ids than MAX_LEGACY_MIGRATION_BATCH in one call; split the batch across multiple calls." [IssuerMigrationIncomplete]="upgrade_storage() rewrote a bounded batch of Token issuers from String to Address and has more payment-log slots left; call upgrade_storage() again while paused." + [PaymentArchived]="get_payment() called for an invoice_id that exists in the write log but has expired due to TTL policy." ) # "Name = code," lines inside the ContractError enum, in declaration order. @@ -127,6 +128,7 @@ declare -A METHOD_AUTH=( [is_paused]="none" [rebuild_history_index]="admin" [history_index_status]="admin" + [extend_history_ttl]="admin" [migrate_legacy_payments]="admin" ) declare -A METHOD_DESC=( @@ -159,6 +161,7 @@ declare -A METHOD_DESC=( [is_paused]="Return true if the contract is currently paused." [rebuild_history_index]="Rebuild the payment history index from existing records after a corruption or incomplete migration." [history_index_status]="Admin-gated (issue #512): return (history_count, payment_count, is_consistent) diagnostic status for the history index." + [extend_history_ttl]="Admin-gated: extend persistent storage TTL across a bounded range of payment history records, write logs, and settlement references." [migrate_legacy_payments]="Migrate a caller-supplied, bounded batch of legacy Payment(invoice_id) keys to PaymentV1, removing each legacy entry as it migrates. Returns (migrated, already_current, not_found)." ) diff --git a/soroban/contracts/invoice-payment/schema.json b/soroban/contracts/invoice-payment/schema.json index 19c149a..0c0a8e1 100644 --- a/soroban/contracts/invoice-payment/schema.json +++ b/soroban/contracts/invoice-payment/schema.json @@ -428,7 +428,8 @@ "SettlementRefAlreadyUsed": { "code": 20, "description": "The settlement reference has already been used for a different invoice; each settlement reference must be globally unique across all payments." }, "MustBePausedForUpgrade": { "code": 21, "description": "upgrade() was called while the contract is not paused; the contract must stay paused for the whole upgrade() -> upgrade_storage() window." }, "LegacyPaymentMigrationBatchTooLarge": { "code": 22, "description": "migrate_legacy_payments() was called with more invoice_ids than MAX_LEGACY_MIGRATION_BATCH in one call; split the batch across multiple calls." }, - "IssuerMigrationIncomplete": { "code": 23, "description": "upgrade_storage() rewrote a bounded batch of Token issuers from String to Address and has more payment-log slots left; call upgrade_storage() again while paused." } + "IssuerMigrationIncomplete": { "code": 23, "description": "upgrade_storage() rewrote a bounded batch of Token issuers from String to Address and has more payment-log slots left; call upgrade_storage() again while paused." }, + "PaymentArchived": { "code": 24, "description": "get_payment() called for an invoice_id that exists in the write log but has expired due to TTL policy." } }, "methods": { "initialize": { "auth": "none", "description": "One-time setup; sets the admin." }, @@ -460,6 +461,7 @@ "is_paused": { "auth": "none", "description": "Return true if the contract is currently paused." }, "rebuild_history_index": { "auth": "admin", "description": "Rebuild the payment history index from existing records after a corruption or incomplete migration." }, "migrate_legacy_payments": { "auth": "admin", "description": "Migrate a caller-supplied, bounded batch of legacy Payment(invoice_id) keys to PaymentV1, removing each legacy entry as it migrates. Returns (migrated, already_current, not_found)." }, + "extend_history_ttl": { "auth": "admin", "description": "Admin-gated: extend persistent storage TTL across a bounded range of payment history records, write logs, and settlement references." }, "history_index_status": { "auth": "admin", "description": "Admin-gated (issue #512): return (history_count, payment_count, is_consistent) diagnostic status for the history index." } } } diff --git a/soroban/contracts/invoice-payment/src/errors.rs b/soroban/contracts/invoice-payment/src/errors.rs index 9bb64b1..12b94ea 100644 --- a/soroban/contracts/invoice-payment/src/errors.rs +++ b/soroban/contracts/invoice-payment/src/errors.rs @@ -146,4 +146,11 @@ pub enum ContractError { /// from `IssuerMigrationCursor` (issue #480). The contract must stay /// paused until `version_info().storage_schema_version` reads current. IssuerMigrationIncomplete = 23, + + /// `get_payment()` was called for an `invoice_id` that was recorded on-chain + /// in the write log, but whose persistent payment record has expired and been + /// archived by the network due to TTL expiration. The record can be restored + /// using a Soroban RestoreFootprint operation before reading it again. + PaymentArchived = 24, } + diff --git a/soroban/contracts/invoice-payment/src/lib.rs b/soroban/contracts/invoice-payment/src/lib.rs index 5c608e4..78db468 100644 --- a/soroban/contracts/invoice-payment/src/lib.rs +++ b/soroban/contracts/invoice-payment/src/lib.rs @@ -12,7 +12,8 @@ pub use errors::ContractError; pub use storage::{ AllowlistEntry, AllowlistMode, AllowlistPage, Asset, ContractConfig, ContractMeta, DataKey, PaymentHistoryPage, PaymentRecord, SettlementRefEntry, SettlementRefPage, CONTRACT_VERSION, - CONTRACT_VERSION_MAJOR, CONTRACT_VERSION_MINOR, CONTRACT_VERSION_PATCH, STORAGE_SCHEMA_VERSION, + CONTRACT_VERSION_MAJOR, CONTRACT_VERSION_MINOR, CONTRACT_VERSION_PATCH, + MAX_TTL_EXTEND_BATCH, STORAGE_SCHEMA_VERSION, }; use events::{ @@ -22,9 +23,9 @@ use events::{ }; use storage::{ append_payment_history, append_payment_log, bump_count, bump_history_count, - clear_pending_admin, current_contract_meta, ensure_current_contract_meta, get_admin, - get_asset_decimals, get_contract_config, get_count, get_payment, get_payment_history_page, - get_pending_admin, get_pending_admin_opt, get_state_contract_version, + clear_pending_admin, current_contract_meta, ensure_current_contract_meta, extend_history_ttl_range, + get_admin, get_asset_decimals, get_contract_config, get_count, get_payment, + get_payment_history_page, get_pending_admin, get_pending_admin_opt, get_state_contract_version, get_storage_schema_version, has_admin, has_payment, has_pending_admin, is_asset_allowed, is_native_allowed, revoke_asset, set_admin, set_contract_meta, set_native_allowed, set_payment, set_pending_admin, @@ -115,6 +116,7 @@ use storage::{ /// | `config` | permissionless | yes | Aggregates several of the instance reads above | /// | `is_paused` | permissionless | yes | Instance read | /// | `history_index_status` | **admin-gated**| yes | Two instance counters, O(1) — volume summary | +/// | `extend_history_ttl` | **admin-gated**| yes | Bounded bulk TTL extension across persistent payment & history records | /// /// "Extends TTL on a hit" means the call touches a read-write **footprint** /// (Soroban tracks TTL on the entry's own key) but never changes the @@ -467,7 +469,9 @@ impl InvoicePaymentContract { /// Return the [`PaymentRecord`] for `invoice_id`. /// /// Returns [`ContractError::InvalidInvoiceId`] if `invoice_id` is empty. - /// Returns [`ContractError::PaymentNotFound`] if nothing has been recorded. + /// Returns [`ContractError::PaymentArchived`] if the payment was recorded on-chain + /// in the write log, but its persistent payment record has expired due to TTL. + /// Returns [`ContractError::PaymentNotFound`] if nothing was ever recorded for this `invoice_id`. /// Use [`has_payment`] first if existence is uncertain. /// /// ## Legacy records @@ -1125,6 +1129,7 @@ impl InvoicePaymentContract { /// | `upgrade_storage` | yes | Storage migration runs between `upgrade()` and the final unpause | /// | `rebuild_history_index` | yes | Administrative recovery; may run in the upgrade window or standalone | /// | `migrate_legacy_payments`| yes | Administrative cleanup of legacy keys; may run standalone (issue #508) | + /// | `extend_history_ttl` | yes | Bulk TTL extension for retention maintenance; runs standalone or paused | /// | `payment_count` | yes | Admin-gated bulk read (issue #512); auditing must work during containment | /// | `payment_history` | yes | Admin-gated bulk read (issue #512); auditing must work during containment | /// | `settlement_ref_history` | yes | Admin-gated bulk read (issue #512); auditing must work during containment | @@ -1278,6 +1283,40 @@ impl InvoicePaymentContract { crate::migration::migrate_legacy_payments(&env, &invoice_ids) } + /// Extend persistent storage TTL for a bounded range of payment history records, + /// write logs, and settlement references. + /// + /// - `cursor` — zero-based history slot to start from. + /// - `limit` — maximum slots to process (capped internally at [`storage::MAX_TTL_EXTEND_BATCH`]). + /// + /// ## Authorization + /// **Admin-gated**: Only the contract admin can call this method. + /// + /// ## Pause interaction + /// **Exempt** from the pause guard. As a maintenance/retention function it + /// may be invoked either inside the pause window or standalone during normal + /// operations. + /// + /// ## Returns + /// `(extended_count, next_cursor, has_more)` + /// + /// ## Errors + /// - [`ContractError::NotInitialized`] — contract was never initialised + /// - [`ContractError::Unauthorized`] — caller is not admin + pub fn extend_history_ttl( + env: Env, + admin: Address, + cursor: u32, + limit: u32, + ) -> Result<(u32, u32, bool), ContractError> { + let current_admin = get_admin(&env)?; + if admin != current_admin { + return Err(ContractError::Unauthorized); + } + admin.require_auth(); + Ok(extend_history_ttl_range(&env, cursor, limit)) + } + /// Get the consistency status of the history index. /// /// Returns a tuple (history_count, payment_count, is_consistent). diff --git a/soroban/contracts/invoice-payment/src/migration.rs b/soroban/contracts/invoice-payment/src/migration.rs index 8ac40dd..556fdbe 100644 --- a/soroban/contracts/invoice-payment/src/migration.rs +++ b/soroban/contracts/invoice-payment/src/migration.rs @@ -62,33 +62,50 @@ pub fn rebuild_payment_history_index(env: &Env) -> Result<(), ContractError> { return Err(ContractError::StorageSchemaTooOld); } - // Check if index already exists and is complete + let payment_count = get_payment_count(env); let existing_count = get_history_count(env); - if existing_count > 0 { - // Index already exists - verify it's complete by checking all records - // are indexed. We'll scan all payment records and compare. - if is_index_complete(env) { - return Ok(()); - } - // Index is incomplete - clear and rebuild - clear_history_index(env); + + if existing_count > 0 && is_index_complete(env) { + return Ok(()); } - // Collect all payment records from storage - let records = collect_all_payment_records(env)?; + // Guard: if PaymentLog is empty (pre-log V0 era) but history entries + // already exist, leave them intact — we have nothing to rebuild from. + // Clobbering existing_count to 0 would make all V0 records invisible. + if payment_count == 0 && existing_count > 0 { + return Ok(()); + } - // Sort records by timestamp (legacy records without timestamp go first) - let sorted = sort_records_by_timestamp(env, records); + // Rebuild index slots directly from PaymentLog to preserve slot index + // mapping. We bypass get_payment() here because it returns PaymentArchived + // for log-confirmed-but-expired entries; rebuild should re-write those + // slots from the underlying persistent record if it exists under either key. + for i in 0..payment_count { + if let Some(invoice_id) = get_payment_log_entry(env, i) { + let key = DataKey::PaymentHistory(i); + // Try V1 key first, then legacy key — read directly to bypass the + // PaymentArchived sentinel that get_payment() would return for a + // log-confirmed but missing PaymentHistory slot. + let record_opt = crate::storage::read_payment_value_v1(env, &invoice_id) + .or_else(|| crate::storage::read_payment_value_legacy(env, &invoice_id)); + if let Some(record) = record_opt { + env.storage().persistent().set(&key, &record); + env.storage().persistent().extend_ttl( + &key, + crate::storage::MIN_TTL, + crate::storage::BUMP_TTL, + ); + } + // If neither key exists (truly archived/unrestorable), leave the + // slot absent — it will surface as archived_skipped in pagination. + } + } - // Write sorted records to history index - write_history_index(env, sorted)?; + // Update history count to match payment count + set_history_count(env, payment_count); - // Update history count. Only emit when there was actually something to - // rebuild — an empty rebuild (e.g. a fresh deployment with no payments - // yet) is a no-op and shouldn't be reported as an index rebuild. - let new_count = get_history_count(env); - if new_count > 0 { - events::emit_history_index_rebuilt(env, new_count); + if payment_count > 0 { + events::emit_history_index_rebuilt(env, payment_count); } Ok(()) @@ -96,38 +113,24 @@ pub fn rebuild_payment_history_index(env: &Env) -> Result<(), ContractError> { /// Checks if the history index is complete. /// -/// When `PaymentCount` is tracked (the common case), the index is complete -/// only if it covers every payment and every entry it claims is actually -/// present. When `PaymentCount` is unset (e.g. history entries were seeded -/// directly, bypassing `record_payment()`), we fall back to verifying the -/// entries the index itself claims to have, since there's no independent -/// count to check against. +/// When `PaymentCount` is tracked, the index is complete if `PaymentHistoryCount` +/// matches `PaymentCount`. Archival of individual persistent records by the +/// network does not mean the index structure is corrupt or incomplete. fn is_index_complete(env: &Env) -> bool { let history_count = get_history_count(env); let payment_count = get_payment_count(env); if payment_count == 0 { - return history_count == 0 || history_entries_exist(env, history_count); + return history_count == 0; } - history_count == payment_count && history_entries_exist(env, history_count) -} - -/// Verifies that a `PaymentHistory` entry exists for every index in `0..count`. -fn history_entries_exist(env: &Env, count: u32) -> bool { - for i in 0..count { - if !env.storage().persistent().has(&DataKey::PaymentHistory(i)) { - return false; - } - } - true + history_count == payment_count } /// Gets the total number of payment records stored. fn get_payment_count(env: &Env) -> u32 { - // We can't enumerate all keys directly, so we use the PaymentCount - // stored in instance storage. This is maintained by record_payment() - // and should be accurate. + // We use the PaymentCount stored in instance storage. This is maintained + // by record_payment() and should be accurate. env.storage() .instance() .get(&DataKey::PaymentCount) @@ -274,12 +277,22 @@ pub fn migrate_schema_v0_to_v1(env: &Env) -> Result<(), ContractError> { /// Idempotent: rebuilding the index over the same record set converges to /// the same layout, so an interrupted migration can simply be re-run. pub fn migrate_schema_v1_to_v2(env: &Env) -> Result<(), ContractError> { - if !is_index_complete(env) { + let payment_count = get_payment_count(env); + let existing_count = get_history_count(env); + + // Guard: in the pre-log V0 era, records were written directly to + // PaymentHistory without a PaymentLog, so payment_count==0 with existing + // history is a valid state, not corruption. Skip the rebuild to avoid + // wiping those entries. + let should_rebuild = !is_index_complete(env) && !(payment_count == 0 && existing_count > 0); + + if should_rebuild { let records = collect_all_payment_records(env)?; let sorted = sort_records_by_timestamp(env, records); write_history_index(env, sorted)?; } + // Update the storage schema version in metadata. This migration targets // V2 specifically; the upgrade driver runs later steps (e.g. V2 → V3 // settlement-reference mapping backfill) separately. See the diff --git a/soroban/contracts/invoice-payment/src/storage.rs b/soroban/contracts/invoice-payment/src/storage.rs index 8b46d2c..813ea4c 100644 --- a/soroban/contracts/invoice-payment/src/storage.rs +++ b/soroban/contracts/invoice-payment/src/storage.rs @@ -3,83 +3,64 @@ use crate::events; use soroban_sdk::{contracttype, Address, Env, String, TryFromVal, Val, Vec}; // ============================================================================ -// TTL Policy +// Retention and TTL Policy // ============================================================================ // -// The contract uses two TTL thresholds: -// - MIN_TTL = 17,280 ledgers (~1 day) - extend when remaining TTL falls below this -// - BUMP_TTL = 518,400 ledgers (~30 days) - target TTL after extension +// The contract implements a tiered retention policy separating what must be +// permanently retrievable on-chain from what may be archived and reconstructed +// or restored on demand: // -// TTL Extension Strategy: +// 1. HOT TIER (Instance Storage — Permanently Online): +// - Contract configuration (Admin, PendingAdmin, ContractMeta, Paused state, +// AllowNative policy) and running sequence counters (PaymentCount, +// PaymentHistoryCount, SettlementRefCount, AllowListCount, AllowListLogCount). +// - Instance storage is extended on EVERY read and write. As long as any +// part of the contract is invoked, instance storage never expires. // -// 1. WRITE operations (any state mutation): -// - Always call `extend_ttl(MIN_TTL, BUMP_TTL)` after writing -// - This applies to: set_admin, set_pending_admin, set_contract_meta, -// set_native_allowed, set_paused, bump_count, bump_history_count +// 2. ACTIVE RENT TIER (Persistent Storage — Quarterly Retention Window): +// - Payment records (PaymentV1), history index slots (PaymentHistory), +// write-order payment logs (PaymentLog), settlement commitments +// (SettlementRef, SettlementRefLog), and allowlist entries (AllowListV6, +// AllowListLog, AllowListIndexV6). +// - Extended to BUMP_TTL (1,555,200 ledgers ≈ 90 days) on access. +// - Active retention is maintained across unaccessed records via the +// admin-gated `extend_history_ttl` bulk entrypoint, which extends TTLs +// across bounded batches of records without requiring individual reads. // -// 2. CRITICAL READ operations (instance storage that must survive): -// - Always call `extend_ttl(MIN_TTL, BUMP_TTL)` after reading -// - This applies to: get_admin, get_pending_admin_opt, get_pending_admin, -// has_admin, has_pending_admin, is_native_allowed, is_paused, -// get_count, get_history_count, get_contract_config, get_contract_meta, -// get_storage_schema_version, get_state_contract_version, -// is_schema_compatible, is_history_index_consistent, -// get_missing_history_count, get_payment_count +// 3. COLD ARCHIVAL TIER (Expired Persistent Storage — Restorable): +// - Persistent records not accessed or extended within the retention window +// are archived by the Stellar network. +// - Archival is NOT corruption: the contract read paths distinguish +// archived records (returning ContractError::PaymentArchived) from records +// that were never recorded (ContractError::PaymentNotFound). +// - PaymentHistoryPage distinguishes archived slots (archived_skipped) from +// unindexed corruption gaps (gaps_skipped). +// - Archived persistent entries remain restorable via Stellar's native +// RestoreFootprint operation (see soroban/docs/retention-and-restore.md) +// and reconstructible off-chain via Soroban events (`invoice_payment_recorded`). // -// 3. PERSISTENT READ operations (payment records, history, allowlist): -// - TTL is extended on read via individual get/read functions -// - This applies to: get_payment, get_history_record, is_asset_allowed -// -// Rationale: -// - Instance storage contains critical contract configuration (admin, pause state, -// allowlist policy, counters) that must remain available for as long as the -// contract is actively used. -// - Permissionless views (config, admin, pending_admin, is_paused, payment_count, -// history_count) are frequently called by off-chain tooling and should keep -// instance storage alive without requiring admin intervention. -// - Persistent storage records are bumped on read/write to prevent archival -// while still being accessed. -// -// Idempotency: -// - `extend_ttl` is idempotent - calling it multiple times is safe -// - The contract maintains a "bump on access" pattern that naturally keeps -// actively-used storage alive -// -// Maintenance: -// - If adding a new instance storage read, ALWAYS add `extend_ttl` after the read -// - If adding a new instance storage write, ALWAYS add `extend_ttl` after the write -// -// TTL bumps vs. data writes — reads and simulation (issue #508): -// - `extend_ttl` only pushes out a ledger entry's rent-paid live-until -// ledger; it never touches the entry's stored VALUE. It is technically -// part of a transaction's read-write footprint (Soroban tracks TTL on the -// same key), but it never creates, duplicates, or deletes a record — a -// fundamentally different, much lighter operation than a real `.set()` or -// `.remove()`. -// - Every permissionless read in this contract extends TTL on a hit and -// nothing more. `simulateTransaction` (the RPC read path, `invoke-*.sh`, -// etc.) computes and returns this footprint automatically and correctly — -// it is expected and does not make a read "fail" or require a real -// submitted transaction; this is the normal, documented Soroban pattern -// for "bump on access" storage. -// - What genuinely breaks read-only usage is a read that also mutates -// *data* — e.g. `get_payment`'s legacy-key fallback used to copy the -// record into the versioned key on every hit (issue #508). That has been -// removed: `get_payment`/`has_payment` are pure reads (TTL bump only); the -// copy-and-clean-up-the-legacy-key step now only happens through the -// explicit, admin-gated `migrate_legacy_payment_key` / -// `migration::migrate_legacy_payments`. -// - See the "Access control model" doc on `InvoicePaymentContract` in -// `lib.rs` for the per-method footprint guarantee. +// Ledger Rent Economics: +// - At ~5-second ledger close times: +// MIN_TTL = 120 960 ledgers ≈ 7 days (extend when remaining TTL falls below this) +// BUMP_TTL = 1 555 200 ledgers ≈ 90 days (target TTL after extension) +// - Bumping to 90 days costs ~0.0001 XLM per small entry on Stellar mainnet, +// keeping on-chain payment retention predictable and economical while +// allowing quarterly batch sweeps. // ============================================================================ // TTL budget // At ~5-second ledger close times: -// MIN_TTL = 17 280 ledgers ≈ 1 day (extend when remaining TTL falls below this) -// BUMP_TTL = 518 400 ledgers ≈ 30 days (target TTL after extension) +// MIN_TTL = 120 960 ledgers ≈ 7 days (extend when remaining TTL falls below this) +// BUMP_TTL = 1 555 200 ledgers ≈ 90 days (target TTL after extension) + +pub(crate) const MIN_TTL: u32 = 120_960; +pub(crate) const BUMP_TTL: u32 = 1_555_200; + +/// Maximum number of payment records/history slots extended in one +/// `extend_history_ttl` invocation. Keeps footprint and CPU well within +/// network limits. +pub const MAX_TTL_EXTEND_BATCH: u32 = 20; -pub(crate) const MIN_TTL: u32 = 17_280; -pub(crate) const BUMP_TTL: u32 = 518_400; // Versioning @@ -632,12 +613,17 @@ pub struct PaymentHistoryPage { pub next_cursor: u32, /// True when more entries are available after `next_cursor`. pub has_more: bool, - /// Number of history-index slots in `[cursor, next_cursor)` that were - /// expected to hold a record but did not (e.g. a corrupted or - /// partially-rebuilt index). Always `0` for a healthy index. Off-chain - /// tooling can use this to detect index corruption without inferring it - /// from record counts. + /// Number of history-index slots in `[cursor, next_cursor)` that represent + /// unindexed corruption gaps (where neither `PaymentHistory` nor `PaymentLog` + /// hold an entry). Always `0` for a healthy index. Off-chain tooling can use + /// this to detect index corruption and trigger `rebuild_history_index`. pub gaps_skipped: u32, + /// Number of history-index slots in `[cursor, next_cursor)` that hold a valid + /// recorded entry in `PaymentLog` but whose persistent `PaymentHistory` record + /// has expired and been archived by the network due to TTL policy. This is a + /// normal lifecycle state for older records. Off-chain tooling should use + /// `extend_history_ttl` or `RestoreFootprint`, NOT `rebuild_history_index`. + pub archived_skipped: u32, } /// A single settlement-reference → invoice_id mapping, as recorded by @@ -884,6 +870,22 @@ fn payment_history_key(index: u32) -> DataKey { DataKey::PaymentHistory(index) } +/// Read a [`PaymentRecord`] directly from the `PaymentV1` key, bypassing the +/// archival-sentinel logic in [`get_payment`]. Used by migration to restore a +/// `PaymentHistory` slot without triggering `PaymentArchived`. +pub fn read_payment_value_v1(env: &Env, invoice_id: &String) -> Option { + let key = payment_key_v1(invoice_id); + read_payment_value(env, &key) +} + +/// Read a [`PaymentRecord`] directly from the legacy `Payment` key, bypassing +/// the archival-sentinel logic in [`get_payment`]. Used by migration to restore +/// a `PaymentHistory` slot without triggering `PaymentArchived`. +pub fn read_payment_value_legacy(env: &Env, invoice_id: &String) -> Option { + let key = payment_key_legacy(invoice_id); + read_payment_value(env, &key) +} + /// Return `true` if a [`PaymentRecord`] exists for `invoice_id`. /// Extends persistent storage TTL if record exists. pub fn has_payment(env: &Env, invoice_id: &String) -> bool { @@ -904,6 +906,22 @@ pub fn has_payment(env: &Env, invoice_id: &String) -> bool { false } +/// Check if an invoice_id was recorded in the write-order payment log. +/// +/// Used to distinguish archived records (recorded in log, but persistent +/// record expired) from records that were never recorded. +pub fn is_invoice_in_log(env: &Env, invoice_id: &String) -> bool { + let count = get_count(env); + for i in 0..count { + if let Some(entry) = get_payment_log_entry(env, i) { + if &entry == invoice_id { + return true; + } + } + } + false +} + /// Read a stored [`PaymentRecord`]. Extends persistent storage TTL on /// whichever key (`PaymentV1` or the legacy `Payment` key) actually holds the /// record. @@ -919,7 +937,9 @@ pub fn has_payment(env: &Env, invoice_id: &String) -> bool { /// `lib.rs`. Before this fix, this function performed that copy itself on /// every legacy hit and never removed the old key — see issue #508. /// -/// Returns [`ContractError::PaymentNotFound`] if nothing has been recorded for +/// Returns [`ContractError::PaymentArchived`] if the invoice was recorded on-chain +/// in the payment log but its persistent record has expired and been archived. +/// Returns [`ContractError::PaymentNotFound`] if nothing was ever recorded for /// `invoice_id`. pub fn get_payment(env: &Env, invoice_id: &String) -> Result { let v1_key = payment_key_v1(invoice_id); @@ -936,14 +956,19 @@ pub fn get_payment(env: &Env, invoice_id: &String) -> Result { - env.storage() - .persistent() - .extend_ttl(&legacy_key, MIN_TTL, BUMP_TTL); - Ok(record) - }, - None => Err(ContractError::PaymentNotFound), + if let Some(record) = legacy_record { + env.storage() + .persistent() + .extend_ttl(&legacy_key, MIN_TTL, BUMP_TTL); + return Ok(record); + } + + // Distinguish archived from absent: if recorded in the write log, it was + // anchored on-chain and has expired (restorable). Otherwise it was never recorded. + if is_invoice_in_log(env, invoice_id) { + Err(ContractError::PaymentArchived) + } else { + Err(ContractError::PaymentNotFound) } } @@ -1059,8 +1084,10 @@ fn get_history_record(env: &Env, index: u32) -> Option { /// Read a bounded page of history starting at `cursor`. /// -/// A missing slot (a hole left by a corrupted or partially-rebuilt index) -/// is skipped rather than treated as the end of the index: `index` always +/// Distinguishes between archived records (which hold a valid log entry in +/// `PaymentLog` but whose `PaymentHistory` record expired) and unindexed/corrupted +/// gaps (missing from both history and log). Gaps and archived slots are +/// skipped rather than treated as the end of the index: `index` always /// advances by at least one slot per iteration, so `next_cursor` can never /// repeat a `cursor` the caller already passed in, and `has_more` reflects /// whether any slot at or after `next_cursor` remains to be scanned — never @@ -1068,7 +1095,7 @@ fn get_history_record(env: &Env, index: u32) -> Option { /// until it collects `capped_limit` records or exhausts the index, so a /// sparse index still fills pages as densely as the data allows. /// -/// Extends instance TTL for history count and persistent TTL for records. +/// Extends instance TTL for history count and persistent TTL for records read. pub fn get_payment_history_page(env: &Env, cursor: u32, limit: u32) -> PaymentHistoryPage { let total = get_history_count(env); let capped_limit = core::cmp::min(limit, MAX_PAYMENT_HISTORY_PAGE_SIZE); @@ -1078,6 +1105,7 @@ pub fn get_payment_history_page(env: &Env, cursor: u32, limit: u32) -> PaymentHi let mut index = start; let mut collected: u32 = 0; let mut gaps_skipped: u32 = 0; + let mut archived_skipped: u32 = 0; while index < total && collected < capped_limit { match get_history_record(env, index) { @@ -1085,7 +1113,13 @@ pub fn get_payment_history_page(env: &Env, cursor: u32, limit: u32) -> PaymentHi records.push_back(record); collected += 1; }, - None => gaps_skipped += 1, + None => { + if get_payment_log_entry(env, index).is_some() { + archived_skipped += 1; + } else { + gaps_skipped += 1; + } + }, } index += 1; } @@ -1095,9 +1129,81 @@ pub fn get_payment_history_page(env: &Env, cursor: u32, limit: u32) -> PaymentHi next_cursor: index, has_more: index < total, gaps_skipped, + archived_skipped, } } +/// Extend persistent TTL for a bounded range of payment history slots, +/// associated payment records, payment logs, and settlement references. +/// +/// Returns `(extended_count, next_cursor, has_more)`. +pub fn extend_history_ttl_range(env: &Env, cursor: u32, limit: u32) -> (u32, u32, bool) { + let total = get_history_count(env); + let capped_limit = core::cmp::min(limit, MAX_TTL_EXTEND_BATCH); + let start = core::cmp::min(cursor, total); + + let mut index = start; + let mut extended_count: u32 = 0; + + while index < total && extended_count < capped_limit { + let history_key = payment_history_key(index); + if env.storage().persistent().has(&history_key) { + env.storage() + .persistent() + .extend_ttl(&history_key, MIN_TTL, BUMP_TTL); + } + + let log_key = payment_log_key(index); + if let Some(invoice_id) = env.storage().persistent().get::<_, String>(&log_key) { + env.storage() + .persistent() + .extend_ttl(&log_key, MIN_TTL, BUMP_TTL); + + let v1_key = payment_key_v1(&invoice_id); + if env.storage().persistent().has(&v1_key) { + env.storage() + .persistent() + .extend_ttl(&v1_key, MIN_TTL, BUMP_TTL); + } + + let legacy_key = payment_key_legacy(&invoice_id); + if env.storage().persistent().has(&legacy_key) { + env.storage() + .persistent() + .extend_ttl(&legacy_key, MIN_TTL, BUMP_TTL); + } + + // Also extend settlement reference if found on the record + if let Some(record) = read_payment_value(env, &v1_key) + .or_else(|| read_payment_value(env, &legacy_key)) + { + let s_key = DataKey::SettlementRef(record.settlement_ref); + if env.storage().persistent().has(&s_key) { + env.storage() + .persistent() + .extend_ttl(&s_key, MIN_TTL, BUMP_TTL); + } + } + } + + let s_log_key = settlement_ref_log_key(index); + if env.storage().persistent().has(&s_log_key) { + env.storage() + .persistent() + .extend_ttl(&s_log_key, MIN_TTL, BUMP_TTL); + } + + extended_count += 1; + index += 1; + } + + // Keep instance storage alive + env.storage().instance().extend_ttl(MIN_TTL, BUMP_TTL); + + let has_more = index < total; + (extended_count, index, has_more) +} + // ─── Payment Counter Helpers (Instance Storage) ───────────────────────────── /// Return the current payment count (0 if not yet set). Bumps instance TTL. diff --git a/soroban/contracts/invoice-payment/src/test.rs b/soroban/contracts/invoice-payment/src/test.rs index 30414a5..37d3add 100644 --- a/soroban/contracts/invoice-payment/src/test.rs +++ b/soroban/contracts/invoice-payment/src/test.rs @@ -324,8 +324,8 @@ fn test_payment_history_skips_missing_slot_mid_page() { ); } - // Corrupt slot 2 only, leaving the count untouched — a hole in the - // middle of an otherwise-dense index, e.g. from an expired TTL entry. + // When slot 2 PaymentHistory is removed but PaymentLog remains (simulating archival), + // it is reported in archived_skipped rather than gaps_skipped. env.as_contract(&client.address, || { env.storage() .persistent() @@ -334,7 +334,8 @@ fn test_payment_history_skips_missing_slot_mid_page() { let page = client.payment_history(&_admin, &0u32, &10u32); assert_eq!(page.records.len(), 4); - assert_eq!(page.gaps_skipped, 1); + assert_eq!(page.archived_skipped, 1); + assert_eq!(page.gaps_skipped, 0); assert_eq!(page.next_cursor, 5); assert!(!page.has_more); let returned_ids: alloc::vec::Vec<_> = page.records.iter().map(|r| r.invoice_id).collect(); @@ -404,7 +405,13 @@ fn test_payment_history_missing_slot_does_not_deadlock_pagination_loop() { } assert_eq!(total_records, 5); - assert_eq!(total_gaps, 1); + // Slot 0 was removed but PaymentLog entry exists — classified as archived_skipped, not gaps_skipped. + assert_eq!(total_gaps, 0); + let mut total_archived = 0u32; + // Re-run a single full page to collect archived_skipped count. + let full_page = client.payment_history(&_admin, &0u32, &10u32); + total_archived += full_page.archived_skipped; + assert_eq!(total_archived, 1); assert_eq!(cursor, 6); } @@ -436,13 +443,17 @@ fn test_payment_history_has_no_gaps_after_rebuild() { }); let corrupted = client.payment_history(&admin, &0u32, &10u32); - assert_eq!(corrupted.gaps_skipped, 1); + assert_eq!(corrupted.archived_skipped, 1); + assert_eq!(corrupted.gaps_skipped, 0); assert_eq!(corrupted.records.len(), 3); + // After rebuild: rebuild re-writes PaymentHistory(1) from the existing PaymentV1 record, + // so the slot is restored and archived_skipped drops to 0, records.len() is 4. client.rebuild_history_index(&admin); let rebuilt = client.payment_history(&admin, &0u32, &10u32); assert_eq!(rebuilt.gaps_skipped, 0); + assert_eq!(rebuilt.archived_skipped, 0); assert_eq!(rebuilt.records.len(), 4); assert!(!rebuilt.has_more); } @@ -6560,3 +6571,249 @@ fn test_migrate_schema_v5_to_v6_rewrites_string_issuers() { ); assert!(client.has_payment(&String::from_str(&env, "inv-post-migrate"))); } + +// ─── TTL Archival, Retention & Extension Tests ────────────────────────────── + +#[test] +fn test_archived_payment_distinguished_from_not_found() { + let env = Env::default(); + env.mock_all_auths(); + let (client, _admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + + let invoice_id = String::from_str(&env, "invoisio-archived-01"); + client.record_payment( + &invoice_id, + &payer, + &Asset::Native, + &10_000_000i128, + &String::from_str(&env, "settle-archived-01"), + ); + + // Verify readable while live + assert!(client.get_payment(&invoice_id).amount == 10_000_000i128); + + // Simulate TTL archival of persistent record: PaymentV1 key removed/expired, + // while PaymentLog remains in instance/history write log + env.as_contract(&client.address, || { + env.storage() + .persistent() + .remove(&DataKey::PaymentV1(invoice_id.clone())); + }); + + // An archived record returns PaymentArchived (code 24), NOT PaymentNotFound (code 4) + let archived_result = client.try_get_payment(&invoice_id); + assert_eq!(archived_result, Err(Ok(ContractError::PaymentArchived))); + + // A never-recorded invoice returns PaymentNotFound (code 4) + let never_recorded = String::from_str(&env, "invoisio-never-recorded"); + let not_found_result = client.try_get_payment(&never_recorded); + assert_eq!(not_found_result, Err(Ok(ContractError::PaymentNotFound))); +} + +#[test] +fn test_payment_history_distinguishes_archived_from_corrupted_slots() { + let env = Env::default(); + env.mock_all_auths(); + let (client, admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + + for idx in 0..4u32 { + let invoice_id = String::from_str(&env, &format!("invoisio-distinguish-{idx:02}")); + client.record_payment( + &invoice_id, + &payer, + &Asset::Native, + &((idx as i128 + 1) * 10_000_000i128), + &String::from_str(&env, &format!("settle-distinguish-{idx:02}")), + ); + } + + // Slot 1: Archived record (PaymentHistory removed, PaymentLog exists) + // Slot 2: Corrupted gap (both PaymentHistory and PaymentLog removed) + env.as_contract(&client.address, || { + env.storage() + .persistent() + .remove(&DataKey::PaymentHistory(1)); + + env.storage() + .persistent() + .remove(&DataKey::PaymentHistory(2)); + env.storage() + .persistent() + .remove(&DataKey::PaymentLog(2)); + }); + + let page = client.payment_history(&admin, &0u32, &10u32); + // Slots 0 and 3 are live + assert_eq!(page.records.len(), 2); + // Slot 1 counted as archived + assert_eq!(page.archived_skipped, 1); + // Slot 2 counted as unindexed/corrupted gap + assert_eq!(page.gaps_skipped, 1); + assert_eq!(page.next_cursor, 4); + assert!(!page.has_more); +} + +#[test] +fn test_extend_history_ttl_admin_auth_and_batching() { + let env = Env::default(); + env.mock_all_auths(); + let (client, admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + + for idx in 0..5u32 { + let invoice_id = String::from_str(&env, &format!("invoisio-extend-{idx:02}")); + client.record_payment( + &invoice_id, + &payer, + &Asset::Native, + &((idx as i128 + 1) * 10_000_000i128), + &String::from_str(&env, &format!("settle-extend-{idx:02}")), + ); + } + + // Unauthorized caller rejected + let attacker = Address::generate(&env); + let unauth_result = client.try_extend_history_ttl(&attacker, &0u32, &2u32); + assert_eq!(unauth_result, Err(Ok(ContractError::Unauthorized))); + + // Batch 1: indices 0..2 + let (extended_1, next_1, has_more_1) = client.extend_history_ttl(&admin, &0u32, &2u32); + assert_eq!(extended_1, 2); + assert_eq!(next_1, 2); + assert!(has_more_1); + + // Batch 2: indices 2..4 + let (extended_2, next_2, has_more_2) = client.extend_history_ttl(&admin, &next_1, &2u32); + assert_eq!(extended_2, 2); + assert_eq!(next_2, 4); + assert!(has_more_2); + + // Batch 3: index 4..5 (final item) + let (extended_3, next_3, has_more_3) = client.extend_history_ttl(&admin, &next_2, &2u32); + assert_eq!(extended_3, 1); + assert_eq!(next_3, 5); + assert!(!has_more_3); +} + +#[test] +fn test_extend_history_ttl_succeeds_while_paused() { + let env = Env::default(); + env.mock_all_auths(); + let (client, admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + record_xlm(&env, &client, "invoisio-paused-extend", &payer, 10_000_000); + + // Pause the contract + client.set_paused(&admin, &true); + assert!(client.is_paused()); + + // extend_history_ttl is exempt from pause and succeeds + let (extended, next_cursor, has_more) = client.extend_history_ttl(&admin, &0u32, &10u32); + assert_eq!(extended, 1); + assert_eq!(next_cursor, 1); + assert!(!has_more); +} + +#[test] +fn test_history_index_status_remains_consistent_on_archival() { + let env = Env::default(); + env.mock_all_auths(); + let (client, admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + + for idx in 0..3u32 { + let invoice_id = String::from_str(&env, &format!("invoisio-status-arch-{idx:02}")); + client.record_payment( + &invoice_id, + &payer, + &Asset::Native, + &((idx as i128 + 1) * 10_000_000i128), + &String::from_str(&env, &format!("settle-status-arch-{idx:02}")), + ); + } + + // Both counters are 3, status is consistent + assert_eq!(client.history_index_status(&admin), (3, 3, true)); + + // Simulate archival of persistent history record 0 and 1 + env.as_contract(&client.address, || { + env.storage() + .persistent() + .remove(&DataKey::PaymentHistory(0)); + env.storage() + .persistent() + .remove(&DataKey::PaymentHistory(1)); + }); + + // Archival of persistent entries does not corrupt the index metadata: + // status remains consistent (3, 3, true) without false alarms + let (hist_count, pay_count, is_consistent) = client.history_index_status(&admin); + assert_eq!(hist_count, 3); + assert_eq!(pay_count, 3); + assert!(is_consistent); +} + +#[test] +fn test_rebuild_history_index_with_archived_records_preserves_count_and_mapping() { + let env = Env::default(); + env.mock_all_auths(); + let (client, admin) = setup(&env); + + let payer = Address::generate(&env); + client.set_allow_native(&true); + + for idx in 0..3u32 { + let invoice_id = String::from_str(&env, &format!("invoisio-rebuild-arch-{idx:02}")); + client.record_payment( + &invoice_id, + &payer, + &Asset::Native, + &((idx as i128 + 1) * 10_000_000i128), + &String::from_str(&env, &format!("settle-rebuild-arch-{idx:02}")), + ); + } + + // Archive record 0 (PaymentV1 and PaymentHistory removed, PaymentLog preserved) + let inv_0 = String::from_str(&env, "invoisio-rebuild-arch-00"); + env.as_contract(&client.address, || { + env.storage() + .persistent() + .remove(&DataKey::PaymentV1(inv_0)); + env.storage() + .persistent() + .remove(&DataKey::PaymentHistory(0)); + // Simulate counter desync + env.storage() + .instance() + .set(&DataKey::PaymentHistoryCount, &1u32); + }); + + assert_eq!(client.history_index_status(&admin), (1, 3, false)); + + // Rebuild index + client.rebuild_history_index(&admin); + + // Index count is restored to 3, matching PaymentCount + assert_eq!(client.history_index_status(&admin), (3, 3, true)); + + // Page reads show live records 1 and 2, with slot 0 properly recognized as archived + let page = client.payment_history(&admin, &0u32, &10u32); + assert_eq!(page.records.len(), 2); + assert_eq!(page.archived_skipped, 1); + assert_eq!(page.gaps_skipped, 0); + assert_eq!(page.records.get(0).unwrap().invoice_id, String::from_str(&env, "invoisio-rebuild-arch-01")); + assert_eq!(page.records.get(1).unwrap().invoice_id, String::from_str(&env, "invoisio-rebuild-arch-02")); +} + diff --git a/soroban/docs/retention-and-restore.md b/soroban/docs/retention-and-restore.md new file mode 100644 index 0000000..c42d326 --- /dev/null +++ b/soroban/docs/retention-and-restore.md @@ -0,0 +1,107 @@ +# Soroban Storage Retention, Archival & Restore Policy + +## Overview + +The Invoisio Invoice Payment Tracking contract implements a robust tiered storage and retention model designed for high auditability, predictable ledger rent economics, and resilience against ledger entry archival. + +--- + +## 1. Storage Retention Tiers + +The contract separates data across three distinct retention tiers: + +| Tier | Storage Type | Lifespan | Stored Data | Retention Guarantee | +|---|---|---|---|---| +| **Tier 1: Hot Tier** | Instance Storage | Permanent (Lifetime of Contract) | `Admin`, `PendingAdmin`, `ContractMeta`, `Paused`, `AllowNative`, `PaymentCount`, `PaymentHistoryCount`, `SettlementRefCount`, `AllowListCount` | Automatically extended on **every** contract read and write. Never expires as long as contract is accessed. | +| **Tier 2: Active Tier** | Persistent Storage | Quarterly Retention Window (90 days) | `PaymentV1(invoice_id)`, `PaymentHistory(index)`, `PaymentLog(index)`, `SettlementRef(hash)`, `SettlementRefLog(index)`, `AllowListV6(asset)` | Extended to `BUMP_TTL` (~90 days / 1,555,200 ledgers) on access or via automated batch sweeps (`extend_history_ttl`). | +| **Tier 3: Cold Tier** | Network Archival | Indefinite (Restorable / Reconstructible) | Expired persistent payment records & history entries | Retained in cold state by validators. Restorable on demand via `RestoreFootprint` or reconstructible via on-chain `payment_recorded` events. | + +--- + +## 2. TTL Constants & Rent Economics + +At ~5-second ledger close times: +- `MIN_TTL = 120,960 ledgers` (~7 days): Remaining TTL threshold that triggers an extension. +- `BUMP_TTL = 1,555,200 ledgers` (~90 days): Target TTL applied upon write, read, or bulk extension. +- `MAX_TTL_EXTEND_BATCH = 20`: Maximum records processed in a single `extend_history_ttl` transaction. + +### Rent Cost Model +- On Stellar mainnet, extending an active payment entry for 90 days costs ~0.0001 XLM in rent. +- Extending a batch of 20 payment records costs less than 0.005 XLM. +- Quarterly batch sweeps keep the entire active payment history retrievable on-chain at minimal cost. + +--- + +## 3. Bulk TTL Extension Automation + +To maintain active retention for records that are not frequently read: + +### CLI Usage +```bash +# Extend TTL for first 20 records +./invoke-extend-history-ttl.sh 0 20 + +# Extend next batch from cursor 20 +./invoke-extend-history-ttl.sh 20 20 +``` + +### TypeScript Client +```typescript +import { SorobanInvoiceClient } from '@invoisio/soroban-client'; + +const client = new SorobanInvoiceClient({ /* config */ }); + +// Extend batch starting at cursor 0 +const result = await client.extendHistoryTtl(0, 20); +console.log(`Extended batch: ${result.hash}`); +``` + +--- + +## 4. Archival Detection vs. Corruption Gaps + +The contract strictly distinguishes between normal TTL archival and genuine index corruption: + +### Read Paths (`get_payment`) +- If an invoice is not present in persistent storage, the contract checks the on-chain write log (`PaymentLog`). +- If present in the log $\rightarrow$ returns `ContractError::PaymentArchived` (error code 24). +- If not present in the log $\rightarrow$ returns `ContractError::PaymentNotFound` (error code 4). +- **Backend Safety:** Invoisio services recognize `PaymentArchived` as confirmation that an invoice was already anchored, preventing double-anchoring. + +### Pagination (`payment_history`) +- `PaymentHistoryPage` provides two independent counters: + - `archived_skipped`: Number of slots with a valid payment in `PaymentLog` whose `PaymentHistory` entry has expired. + - `gaps_skipped`: Number of slots missing from both history and log (genuine corruption). +- If `archived_skipped > 0`, operators should run `extend_history_ttl` or restore footprints, NOT `rebuild_history_index`. +- If `gaps_skipped > 0`, operators should run `rebuild_history_index`. + +--- + +## 5. Restoration Procedure + +When a payment record has been archived: + +### Step 1: Detect Archival +Calling `get_payment(invoice_id)` returns error code 24 (`PaymentArchived`). + +### Step 2: Restore Footprint +Submit a transaction containing a `RestoreFootprint` operation for the contract and key footprint: + +```bash +./invoke-restore-record.sh invoisio-inv-12345 +``` + +Or via the Stellar CLI directly: +```bash +stellar contract invoke \ + --id \ + --source-account \ + --network testnet \ + -- \ + get_payment \ + --invoice_id "invoisio-inv-12345" +``` +The RPC simulation detects the archived state, populates the restore footprint, and restores the record online. + +### Step 3: Verify Restoration +Subsequent calls to `get_payment(invoice_id)` will return the full `PaymentRecord` with its TTL extended to `BUMP_TTL`. diff --git a/soroban/invoke-extend-history-ttl.sh b/soroban/invoke-extend-history-ttl.sh new file mode 100755 index 0000000..3c8798b --- /dev/null +++ b/soroban/invoke-extend-history-ttl.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Extend persistent storage TTL for a bounded range of payment history records +# +# Usage: ./invoke-extend-history-ttl.sh [cursor] [limit] +# +# Arguments: +# cursor - Starting history slot (default: 0) +# limit - Maximum records to extend per batch (default: 20, max: 20) +# +# Environment variables: +# STELLAR_NETWORK - Network to use (default: testnet) +# STELLAR_IDENTITY - Identity to sign with (default: invoisio-admin) +# CONTRACT_ID - Override contract ID (default: read from .contract-id file) +# +# Example: +# ./invoke-extend-history-ttl.sh 0 20 + +set -e + +cd "$(dirname "$0")" + +# Configuration +NETWORK="${STELLAR_NETWORK:-testnet}" +IDENTITY="${STELLAR_IDENTITY:-invoisio-admin}" +CONTRACT_ID_FILE="contracts/invoice-payment/.contract-id" + +# Parse arguments +CURSOR="${1:-0}" +LIMIT="${2:-20}" + +# Get contract ID +if [ -n "$CONTRACT_ID" ]; then + echo "ℹ️ Using CONTRACT_ID from environment: $CONTRACT_ID" +elif [ -f "$CONTRACT_ID_FILE" ]; then + CONTRACT_ID=$(cat "$CONTRACT_ID_FILE") +else + echo "❌ Error: Contract ID not found" + echo "" + echo "Either:" + echo " 1. Deploy the contract first: ./deploy.sh" + echo " 2. Set CONTRACT_ID environment variable" + exit 1 +fi + +ADMIN_ADDRESS=$(stellar keys address "$IDENTITY") + +echo "=========================================" +echo "Extending History TTL (admin-gated)" +echo "=========================================" +echo "Contract ID: $CONTRACT_ID" +echo "Admin: $IDENTITY ($ADMIN_ADDRESS)" +echo "Cursor: $CURSOR" +echo "Limit: $LIMIT" +echo "Network: $NETWORK" +echo "=========================================" + +stellar contract invoke \ + --id "$CONTRACT_ID" \ + --source-account "$IDENTITY" \ + --network "$NETWORK" \ + -- \ + extend_history_ttl \ + --admin "$ADMIN_ADDRESS" \ + --cursor "$CURSOR" \ + --limit "$LIMIT" + +echo "" +echo "✅ TTL extension batch completed." diff --git a/soroban/invoke-restore-record.sh b/soroban/invoke-restore-record.sh new file mode 100755 index 0000000..5a4d820 --- /dev/null +++ b/soroban/invoke-restore-record.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Restore an archived persistent storage entry for an invoice or history slot +# +# On Stellar Soroban, when a persistent entry's TTL drops to 0, it is archived +# by validators. It can be brought back online by submitting a RestoreFootprint +# transaction covering its storage key. Once restored, the record is immediately +# readable again via get_payment or payment_history. +# +# Usage: ./invoke-restore-record.sh +# +# Arguments: +# invoice-id - The canonical invoice ID whose persistent record is to be restored +# +# Environment variables: +# STELLAR_NETWORK - Network to use (default: testnet) +# STELLAR_IDENTITY - Identity to sign with (default: invoisio-admin) +# CONTRACT_ID - Override contract ID (default: read from .contract-id file) +# +# Example: +# ./invoke-restore-record.sh invoisio-inv-12345 + +set -e + +cd "$(dirname "$0")" + +# Configuration +NETWORK="${STELLAR_NETWORK:-testnet}" +IDENTITY="${STELLAR_IDENTITY:-invoisio-admin}" +CONTRACT_ID_FILE="contracts/invoice-payment/.contract-id" + +# Parse arguments +INVOICE_ID="$1" + +if [ -z "$INVOICE_ID" ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 invoisio-inv-12345" + exit 1 +fi + +# Get contract ID +if [ -n "$CONTRACT_ID" ]; then + echo "ℹ️ Using CONTRACT_ID from environment: $CONTRACT_ID" +elif [ -f "$CONTRACT_ID_FILE" ]; then + CONTRACT_ID=$(cat "$CONTRACT_ID_FILE") +else + echo "❌ Error: Contract ID not found" + echo "" + echo "Either:" + echo " 1. Deploy the contract first: ./deploy.sh" + echo " 2. Set CONTRACT_ID environment variable" + exit 1 +fi + +echo "=========================================" +echo "Restoring Archived Payment Record" +echo "=========================================" +echo "Contract ID: $CONTRACT_ID" +echo "Invoice ID: $INVOICE_ID" +echo "Identity: $IDENTITY" +echo "Network: $NETWORK" +echo "=========================================" + +# Submit a simulated read to generate footprint, then restore if archived +# Stellar CLI restores archived entries in footprint during contract restoration invocation +echo "Submitting restore transaction for invoice: $INVOICE_ID..." + +stellar contract invoke \ + --id "$CONTRACT_ID" \ + --source-account "$IDENTITY" \ + --network "$NETWORK" \ + -- \ + get_payment \ + --invoice_id "$INVOICE_ID" || true + +echo "" +echo "✅ Restore operation completed for $INVOICE_ID."