diff --git a/dependencies.sh b/dependencies.sh index 80f1c3ba69..ebcafe4221 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -121,6 +121,7 @@ SYSTEM_PACKAGES="build-essential \ libcurl4-openssl-dev \ libhiredis-dev \ libjemalloc-dev \ + libxxhash-dev \ pkg-config \ patchelf" diff --git a/doc/en/diagrams/oplog-data-flow.puml b/doc/en/diagrams/oplog-data-flow.puml new file mode 100644 index 0000000000..32682eda9e --- /dev/null +++ b/doc/en/diagrams/oplog-data-flow.puml @@ -0,0 +1,58 @@ +@startuml oplog-data-flow +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +actor Client +participant "Primary Master" as Primary +participant "OpLogManager" as OplogMgr +participant "EtcdOpLogStore" as EtcdStore +database etcd +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier +participant "Standby Master" as Standby + +== PUT_END Operation Flow == + +Client -> Primary: PutEnd(key, ...) +activate Primary +Primary -> OplogMgr: Append(PUT_END, key) +activate OplogMgr +OplogMgr -> OplogMgr: Generate sequence_id\nGenerate key_sequence_id +OplogMgr -> EtcdStore: WriteOpLog(entry) +activate EtcdStore +EtcdStore -> etcd: PUT /oplog/{seq} +activate etcd +etcd --> EtcdStore: Success +deactivate etcd +EtcdStore --> OplogMgr: Success +deactivate EtcdStore +OplogMgr --> Primary: sequence_id +deactivate OplogMgr +Primary --> Client: Success +deactivate Primary + +== Standby Synchronization Flow == + +etcd -> Watcher: Watch Event (New OpLog) +activate Watcher +Watcher -> Applier: ApplyOpLogEntry(entry) +activate Applier +Applier -> Applier: CheckSequenceOrder() +alt Order Correct + Applier -> Standby: UpdateMetadata(key, ...) + activate Standby + Standby --> Applier: Success + deactivate Standby +else Order Violation + Applier -> Applier: RollbackAndReplay() + Applier -> etcd: ReadOpLogForKey() + etcd --> Applier: OpLog Entries + Applier -> Applier: Replay OpLog +end +Applier --> Watcher: Success +deactivate Applier +deactivate Watcher + +@enduml + diff --git a/doc/en/diagrams/oplog-failover-sequence.puml b/doc/en/diagrams/oplog-failover-sequence.puml new file mode 100644 index 0000000000..f6da4cc1f8 --- /dev/null +++ b/doc/en/diagrams/oplog-failover-sequence.puml @@ -0,0 +1,66 @@ +@startuml oplog-failover-sequence +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +participant "Primary Master" as Primary +database etcd +participant "Standby Master" as Standby +participant "MasterServiceSupervisor" as Supervisor +participant "HotStandbyService" as HotStandby +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier + +== Normal Operation Phase == + +Primary -> etcd: KeepAlive Lease +etcd -> Supervisor: Watch Leader (Exists) +activate Supervisor +Supervisor -> HotStandby: StartStandby() +activate HotStandby +HotStandby -> Watcher: Start() +activate Watcher +Watcher -> etcd: Watch OpLog +etcd -> Watcher: OpLog Events +Watcher -> Applier: ApplyOpLogEntry() +activate Applier +Applier -> Standby: UpdateMetadata() +deactivate Applier +deactivate Watcher +deactivate HotStandby +deactivate Supervisor + +== Primary Failure == + +Primary -x etcd: Lease Expired (Failure) +etcd -> Supervisor: Leader Deleted Event +activate Supervisor +Supervisor -> HotStandby: Stop() +activate HotStandby +HotStandby -> Watcher: Stop() +deactivate Watcher +deactivate HotStandby + +== Standby Promotion to Primary == + +Supervisor -> Standby: Promote() +activate Standby +Standby -> Standby: Initialize Lease\nClean Expired metadata +Standby -> etcd: ElectLeader() +activate etcd +etcd -> Standby: Leader Elected +deactivate etcd +Standby -> Supervisor: Primary Mode +deactivate Standby +deactivate Supervisor + +note over Standby + 1. Stop Standby service + 2. Iterate all metadata + 3. Grant default lease to objects with lease=0 + 4. Perform complete metadata cleanup + 5. Start leader election +end note + +@enduml + diff --git a/doc/en/diagrams/oplog-hot-standby-architecture.puml b/doc/en/diagrams/oplog-hot-standby-architecture.puml new file mode 100644 index 0000000000..6aa7e94f28 --- /dev/null +++ b/doc/en/diagrams/oplog-hot-standby-architecture.puml @@ -0,0 +1,65 @@ +@startuml oplog-hot-standby-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 12 + +package "Primary Master" #E8F4F8 { + component [MasterService] as MasterService + component [OpLogManager] as OpLogManager + component [EtcdOpLogStore] as EtcdOpLogStore + + MasterService --> OpLogManager : Record Operations + OpLogManager --> EtcdOpLogStore : Write OpLog +} + +package "etcd Cluster" #FFF4E6 { + database [etcd] as etcd +} + +package "Standby Master" #F0F8E8 { + component [MasterServiceSupervisor] as Supervisor + component [HotStandbyService] as HotStandby + component [OpLogWatcher] as Watcher + component [OpLogApplier] as Applier + component [MetadataStore] as MetadataStore + + Supervisor --> HotStandby : Start/Stop + HotStandby --> Watcher : Watch OpLog + Watcher --> Applier : Apply OpLog + Applier --> MetadataStore : Update metadata +} + +EtcdOpLogStore --> etcd : Write OpLog +etcd --> Watcher : Watch Events + +note right of OpLogManager + **Responsibilities**: + - Generate sequence_id + - Generate key_sequence_id + - Maintain memory buffer +end note + +note right of Applier + **Responsibilities**: + - Check order + - Handle out-of-order + - Clean expired entries +end note + +note right of MasterService + **Operations**: + - PutEnd() + - Remove() + - Eviction() +end note + +note right of etcd + **Key Design**: + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +@enduml + diff --git a/doc/en/rfc-oplog-hot-standby-complete.md b/doc/en/rfc-oplog-hot-standby-complete.md new file mode 100644 index 0000000000..77e6684ff1 --- /dev/null +++ b/doc/en/rfc-oplog-hot-standby-complete.md @@ -0,0 +1,318 @@ +# OpLog Hot-Standby Synchronization based on etcd - Complete RFC + +## 1. Background + +### 1.1 Current System Architecture + +Mooncake Store is a high-performance distributed KV cache storage engine designed specifically for LLM inference scenarios. The system adopts a Master-Client architecture: + +- **Master Service**: Manages object metadata, space allocation, node management, etc. +- **Client**: Acts as a storage server providing memory segments while also serving as a client to handle application requests + +### 1.2 High Availability Requirements + +The current system supports two deployment modes: + +1. **Default Mode**: Single Master node, simple deployment but with single point of failure risk +2. **High Availability Mode (unstable)**: Multiple Master nodes coordinated through etcd for leader election + +**Issues**: + +- While HA mode implements leader election, Standby Masters do not perform any operations during the waiting period +- No data synchronization mechanism is implemented; metadata may be incomplete when Standby is promoted to Primary +- Lack of reliable primary-standby data synchronization solution + +### 1.3 Business Scenarios + +In LLM inference scenarios, Master Service requires: +- **High Availability**: Fast failover when Master fails, minimizing service interruption time +- **Data Consistency**: Standby must maintain data consistency with Primary +- **Fast Recovery**: Quick service recovery after failure without lengthy data reconstruction + +### 1.4 Problems with Current Solution + +1. **No Data Synchronization**: Standby Master does not perform any data synchronization operations during the election waiting period +2. **Metadata Loss Risk**: After Primary failure, metadata may be incomplete when Standby is promoted +3. **Long Recovery Time**: Need to re-collect metadata from Client nodes, resulting in long recovery time +4. **Data Inconsistency**: Cannot guarantee data consistency between Standby and Primary + +## 2. Goals + +### 2.1 Primary Goals + +1. **Implement Reliable Primary-Standby Data Synchronization** + - Synchronize all metadata change operations from Primary Master to Standby Master + - Guarantee data consistency between Standby and Primary + +2. **Fast Failure Recovery** + - Standby can quickly promote to Primary after Primary failure + - Complete metadata when promoted, no lengthy reconstruction required + +3. **Minimize OpLog Size** + - Only record critical state change operations (PUT, DELETE) + - Do not record high-frequency but non-critical operations like lease renewals + +4. **Integration with Existing System** + - Integrate with existing snapshot mechanism + - Integrate with existing leader election mechanism + - Do not affect normal operation of existing features + +### 2.2 Non-Functional Goals + +1. **Performance**: OpLog synchronization should not significantly impact Primary performance +2. **Reliability**: Leverage etcd's strong consistency to guarantee data reliability +3. **Scalability**: Support multiple Standby Masters +4. **Maintainability**: Simple implementation, easy to understand and maintain + +## 3. Proposal + +### 3.1 Core Design Approach + +**Use etcd as an intermediate reliability component to implement OpLog primary-standby synchronization**: + +1. **OpLog Mechanism**: Primary Master records all state change operations to OpLog +2. **etcd Storage**: OpLog is written to etcd, leveraging etcd's strong consistency and persistence capabilities +3. **Watch Mechanism**: Standby Master receives OpLog in real-time through etcd Watch mechanism +4. **Ordering Guarantee**: Guarantee operation order through global sequence_id and key-level key_sequence_id + +### 3.2 Architecture Design + +#### 3.2.1 Overall Architecture + +The overall architecture diagram shows the interaction relationships between Primary Master, etcd Cluster, and Standby Master: + +![OpLog Hot-Standby Architecture](./diagrams/oplog-hot-standby-architecture.puml) + +**Architecture Description**: +- **Primary Master**: Handles client requests, records OpLog and writes to etcd +- **etcd Cluster**: Acts as intermediate storage, providing strong consistency and Watch mechanism +- **Standby Master**: Receives OpLog in real-time by watching etcd and applies to local metadata store + +#### 3.2.2 Data Flow Diagram + +The data flow diagram shows the complete flow from Client request to Standby synchronization: + +![OpLog Data Flow](./diagrams/oplog-data-flow.puml) + +**Flow Description**: +1. Client sends `PutEnd` request to Primary Master +2. Primary Master records operation through `OpLogManager`, generating sequence_id +3. `EtcdOpLogStore` writes OpLog to etcd +4. etcd notifies Standby Master through Watch mechanism +5. `OpLogWatcher` receives events and passes to `OpLogApplier` +6. `OpLogApplier` checks order and applies to Standby's metadata store + +#### 3.2.3 Failover Sequence + +The failover sequence diagram shows the complete process from Primary failure to Standby promotion to Primary: + +![OpLog Failover Sequence](./diagrams/oplog-failover-sequence.puml) + +**Flow Description**: +1. **Normal Operation**: Primary maintains Lease, Standby continuously synchronizes OpLog through Watch +2. **Primary Failure**: Primary's Lease expires, etcd notifies Standby +3. **Standby Promotion**: Stop Standby service, initialize Lease, clean expired metadata, start leader election + +### 3.3 Core Component Design + +#### 3.3.1 OpLogManager (Primary Side) + +**Responsibilities**: +- Record all state change operations (PUT_END, PUT_REVOKE, REMOVE) +- Generate global sequence_id and key-level key_sequence_id +- Maintain memory buffer (for fast queries) + +**Key Methods**: +```cpp +class OpLogManager { + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = ""); + std::vector GetEntriesSince(uint64_t since_seq_id, + size_t limit = 1000) const; + uint64_t GetLastSequenceId() const; +}; +``` + +#### 3.3.2 EtcdOpLogStore (Primary Side) + +**Responsibilities**: +- Write OpLog to etcd +- Update latest sequence_id +- Record snapshot corresponding sequence_id +- Clean up old OpLog + +**etcd Key Design**: +- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` +- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` +- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.3.3 OpLogWatcher (Standby Side) + +**Responsibilities**: +- Watch etcd OpLog changes +- Read historical OpLog (for initial synchronization) +- Process Watch events and pass to OpLogApplier + +**Key Methods**: +```cpp +class OpLogWatcher { + void Start(); + void Stop(); + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); +}; +``` + +#### 3.3.4 OpLogApplier (Standby Side) + +**Responsibilities**: +- Apply OpLog Entry to local metadata store +- Check global and key-level order +- Handle sequence number discontinuities and out-of-order cases +- Periodically clean up key_sequence_map_ (memory optimization) + +**Key Methods**: +```cpp +class OpLogApplier { + bool ApplyOpLogEntry(const OpLogEntry& entry); + bool CheckSequenceOrder(const OpLogEntry& entry); + void CleanupStaleKeySequences(); +}; +``` + +#### 3.3.5 HotStandbyService (Standby Side) + +**Responsibilities**: +- Manage Standby mode lifecycle +- Coordinate OpLogWatcher and OpLogApplier +- Handle Standby promotion to Primary logic + +**Key Methods**: +```cpp +class HotStandbyService { + void StartStandby(); + void Stop(); + void Promote(); +}; +``` + +### 3.4 OpLog Entry Data Structure + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // Globally monotonically increasing sequence + uint64_t timestamp_ms{0}; // Timestamp (milliseconds) + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // Object key + std::string payload; // Optional payload (carries replica info for PUT_END) + uint32_t checksum{0}; // Checksum + uint32_t prefix_hash{0}; // Key prefix hash + uint64_t key_sequence_id{0}; // Per-key operation sequence (for ordering guarantee) +}; +``` + +**JSON Serialization Format**: +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +### 3.5 Ordering Guarantee Mechanism + +#### 3.5.1 Global Sequence Number (sequence_id) + +- **Purpose**: Guarantee global order of all OpLog events +- **Generation**: Generated globally incrementally by Primary's `OpLogManager` +- **Check**: Standby checks if sequence_id is continuous + +#### 3.5.2 Key-Level Sequence Number (key_sequence_id) + +- **Purpose**: Guarantee operation order for the same key +- **Generation**: Incremented separately for each key on Primary side +- **Check**: Standby checks if key_sequence_id is increasing + +#### 3.5.3 Out-of-Order Handling + +When key_sequence_id out-of-order is detected: +1. **Rollback**: Delete all state of the key from metadata_store +2. **Replay**: Re-read all OpLog from etcd starting from the key's first sequence_id +3. **Rewrite**: Re-apply all OpLog in correct order to rebuild metadata + +For detailed design, please refer to: `doc/en/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 3.6 Snapshot Integration + +#### 3.6.1 Record Sequence ID During Snapshot + +- When snapshot is generated, record current OpLog sequence_id +- Write snapshot info to etcd: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.6.2 OpLog Cleanup + +- After snapshot generation, OpLog before snapshot can be cleaned up +- Cleanup strategy: Query minimum existing sequence_id from etcd, use DeleteRange to delete + +For detailed design, please refer to: `doc/en/rfc-oplog-cleanup-start-sequence-id.md` + +### 3.7 Standby Service Integration + +#### 3.7.1 Problem + +In existing code, Standby only blocks and waits during leader election, without running Standby service to synchronize OpLog. + +#### 3.7.2 Solution + +In `MasterServiceSupervisor::Start()`: +1. Check if there is currently a leader +2. If there is a leader and it's not self → Start Standby service (watch OpLog and apply) +3. After successful election → Stop Standby service and promote to Primary + +For detailed design, please refer to: `doc/en/rfc-standby-service-integration.md` + +### 3.8 Lease Initialization When Standby Promotes to Primary + +#### 3.8.1 Problem + +Objects on Standby all have lease = 0 (because OpLog only contains PUT_END, not renewal information), and all objects will expire immediately after promotion to Primary. + +#### 3.8.2 Solution + +In `HotStandbyService::Promote()`: +1. Stop Standby service +2. Iterate through all metadata +3. For objects with lease_timeout = 0, grant default lease time (`default_kv_lease_ttl`) + +For detailed design, please refer to: `doc/en/rfc-standby-promotion-lease-initialization.md` + +### 3.9 Memory Optimization: key_sequence_map_ Cleanup + +#### 3.9.1 Problem + +`key_sequence_map_` on Standby side is used to track `key_sequence_id` for each key. After metadata is deleted, these entries are still retained, which may cause memory leaks during long-term operation. + +#### 3.9.2 Solution + +Implement periodic cleanup mechanism: +- **Cleanup Condition**: Last operation is `REMOVE` and more than 1 hour has passed +- **Cleanup Frequency**: Scan once per hour +- **Retention Strategy**: Keys with `PUT_END` and `PUT_REVOKE` operations are not cleaned + +For detailed design, please refer to: `doc/en/rfc-oplog-key-sequence-map-cleanup.md` + +## 4. Key Design Points Summary + +1. **etcd as Intermediate Storage**: Leverage etcd's strong consistency and Watch mechanism +2. **Record Only Critical Operations**: PUT_END, PUT_REVOKE, REMOVE, do not record LEASE_RENEW +3. **Dual Sequence Number Guarantee**: Global sequence_id + key-level key_sequence_id +4. **Snapshot Integration**: Integrate with existing snapshot mechanism, support OpLog cleanup +5. **Standby Service Runs in Parallel**: Continuously synchronize data during election waiting period +6. **Memory Optimization**: Periodically clean up expired entries in key_sequence_map_ + diff --git a/doc/zh/diagrams/etcd-hot-standby-architecture.puml b/doc/zh/diagrams/etcd-hot-standby-architecture.puml new file mode 100644 index 0000000000..1b27d929ed --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-architecture.puml @@ -0,0 +1,139 @@ +@startuml etcd-hot-standby-architecture +!theme plain +skinparam componentStyle rectangle +skinparam linetype ortho + +title etcd热备架构图 + +package "Primary Master" { + component [MasterService] as MasterService { + + AppendOpLogAndNotify() + + SerializeMetadataForOpLog() + + RestoreFromStandbySnapshot() + } + + component [OpLogManager] as OpLogManager { + + Append() + + SetEtcdOpLogStore() + + SetInitialSequenceId() + } + + component [EtcdOpLogStore] as EtcdOpLogStore { + + WriteOpLog() + + UpdateLatestSequenceId() + + CleanupOpLogBefore() + } +} + +package "Standby Master" { + component [HotStandbyService] as HotStandbyService { + + Start() + + Stop() + + Promote() + + GetSyncStatus() + + ExportMetadataSnapshot() + } + + component [OpLogWatcher] as OpLogWatcher { + + StartFromSequenceId() + + WatchOpLog() + + ReadOpLogSinceWithRevision() + } + + component [OpLogApplier] as OpLogApplier { + + ApplyOpLogEntry() + + ApplyOpLogEntries() + + ProcessPendingEntries() + + RequestMissingOpLog() + } + + component [StandbyMetadataStore] as StandbyMetadataStore { + + PutMetadata() + + Remove() + + Snapshot() + } +} + +package "协调组件" { + component [MasterServiceSupervisor] as Supervisor { + + Start() + + StartStandbyService() + } + + component [MasterViewHelper] as ViewHelper { + + ElectLeader() + + KeepLeader() + } + + component [EtcdHelper] as EtcdHelper { + + ConnectToEtcdStoreClient() + + GetRangeAsJson() + + WatchWithPrefixFromRevisionV2() + + GrantLease() + + CreateWithLease() + } +} + +cloud "etcd" { + database "OpLog Storage" as OpLogStorage { + + /oplog/{cluster_id}/{sequence_id} + + /oplog/{cluster_id}/latest + } + + database "Leader Election" as LeaderElection { + + /mooncake-store/{cluster_id}/master_view + } +} + +' Primary Master 内部关系 +MasterService --> OpLogManager : 生成OpLog +OpLogManager --> EtcdOpLogStore : 写入etcd +EtcdOpLogStore --> OpLogStorage : 存储OpLog + +' Standby Master 内部关系 +HotStandbyService --> OpLogWatcher : 启动watch +HotStandbyService --> OpLogApplier : 应用OpLog +HotStandbyService --> StandbyMetadataStore : 存储metadata +OpLogWatcher --> OpLogApplier : 转发OpLog事件 +OpLogApplier --> StandbyMetadataStore : 更新metadata + +' Standby 与 etcd 关系 +OpLogWatcher --> OpLogStorage : Watch + Read +OpLogApplier --> OpLogStorage : 请求缺失OpLog + +' 协调组件关系 +Supervisor --> ViewHelper : Leader选举 +Supervisor --> MasterService : 启动Primary +Supervisor --> HotStandbyService : 启动Standby +ViewHelper --> LeaderElection : 选举Leader +ViewHelper --> EtcdHelper : etcd操作 +EtcdOpLogStore --> EtcdHelper : etcd操作 +OpLogWatcher --> EtcdHelper : etcd操作 + +' 故障切换流程 +HotStandbyService ..> Supervisor : Promote()后返回metadata +Supervisor ..> MasterService : RestoreFromStandbySnapshot() + +note right of OpLogStorage + OpLog存储格式: + Key: /oplog/{cluster_id}/{sequence_id} + Value: JSON序列化的OpLogEntry + 包含: op_type, object_key, payload等 +end note + +note right of LeaderElection + Leader选举: + - 使用etcd lease机制 + - TTL: 5秒 + - 通过CreateWithLease竞争 +end note + +note bottom of OpLogApplier + 顺序保证: + - 使用全局sequence_id保证顺序 + - 乱序的OpLog会进入pending队列 + - 缺失的OpLog会从etcd请求 +end note + +@enduml + diff --git a/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md b/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md new file mode 100644 index 0000000000..b970339898 --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-diagrams-README.md @@ -0,0 +1,127 @@ +# etcd热备架构图表说明 + +本文档包含基于当前代码实现的etcd热备架构的PlantUML图表。 + +## 图表文件 + +### 1. `etcd-hot-standby-architecture.puml` - 整体架构图 + +展示了etcd热备系统的整体架构，包括： + +- **Primary Master组件**: + - `MasterService`: 核心服务，处理客户端请求 + - `OpLogManager`: 生成和管理OpLog + - `EtcdOpLogStore`: 将OpLog写入etcd + +- **Standby Master组件**: + - `HotStandbyService`: Standby服务主控制器 + - `OpLogWatcher`: 从etcd监听OpLog变化 + - `OpLogApplier`: 应用OpLog到本地metadata store + - `StandbyMetadataStore`: Standby的metadata存储 + +- **协调组件**: + - `MasterServiceSupervisor`: 管理Primary/Standby切换 + - `MasterViewHelper`: 处理Leader选举 + - `EtcdHelper`: etcd操作的C++ wrapper + +- **etcd存储**: + - OpLog存储: `/oplog/{cluster_id}/{sequence_id}` + - Leader选举: `/mooncake-store/{cluster_id}/master_view` + +### 2. `etcd-hot-standby-sequence.puml` - 时序图 + +展示了关键流程的时序关系，包括： + +1. **Primary启动流程**: + - Leader选举 + - MasterService初始化 + - OpLogManager设置EtcdOpLogStore + +2. **Standby启动流程**: + - 检测已有Leader + - 热启动 vs 冷启动 + - 快照加载(可选) + - 历史OpLog读取 + - Watch启动 + +3. **写入操作流程**: + - 客户端写入请求 + - Primary生成OpLog + - 写入etcd + - Standby接收并应用 + +4. **故障切换流程**: + - Leader lease过期检测 + - Standby最终同步 + - 重新选举 + - 新Primary初始化 + +### 3. `etcd-hot-standby-flow.puml` - 流程图 + +展示了数据流和控制流，包括： + +1. **OpLog写入流程**: Primary如何生成和写入OpLog +2. **Standby同步流程**: Standby如何启动和同步数据 +3. **OpLog应用流程**: Standby如何应用OpLog(包括乱序处理) +4. **故障切换流程**: Standby如何提升为Primary +5. **OpLog清理流程**: 如何清理etcd中的旧OpLog +6. **批量更新流程**: latest_sequence_id的批量更新机制 + +## 关键设计点 + +### 1. 顺序保证 +- 使用全局`sequence_id`保证OpLog顺序 +- Standby通过`expected_sequence_id`检测乱序 +- 乱序的OpLog进入`pending_entries_`队列 +- 缺失的OpLog从etcd主动请求 + +### 2. 一致性保证 +- 使用etcd revision实现"read then watch"的一致性 +- `ReadOpLogSinceWithRevision`返回revision +- Watch从`revision + 1`开始，确保不丢失事件 + +### 3. 性能优化 +- `latest_sequence_id`批量更新(每100条或每1秒) +- OpLog读取使用分页(每批1000条) +- 使用固定宽度sequence_id确保etcd key的字典序 + +### 4. 故障恢复 +- Standby提升前进行最终同步 +- 新Primary从Standby的metadata快照恢复 +- OpLog sequence_id连续，避免回退 + +## 使用方法 + +### 查看图表 + +1. **在线查看**: 使用PlantUML在线服务器 + - 访问: http://www.plantuml.com/plantuml/uml/ + - 复制`.puml`文件内容粘贴查看 + +2. **VS Code插件**: 安装PlantUML插件 + - 插件: `PlantUML` + - 打开`.puml`文件，按`Alt+D`预览 + +3. **命令行工具**: 使用PlantUML命令行工具 + ```bash + java -jar plantuml.jar etcd-hot-standby-architecture.puml + ``` + +### 导出图片 + +```bash +# 导出为PNG +java -jar plantuml.jar -tpng *.puml + +# 导出为SVG +java -jar plantuml.jar -tsvg *.puml + +# 导出为PDF +java -jar plantuml.jar -tpdf *.puml +``` + +## 相关文档 + +- [RFC: etcd热备完整方案](../rfc-oplog-hot-standby-complete.md) +- [实现计划](../rfc-oplog-implementation-plan.md) + diff --git a/doc/zh/diagrams/etcd-hot-standby-flow.puml b/doc/zh/diagrams/etcd-hot-standby-flow.puml new file mode 100644 index 0000000000..33de9cb159 --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-flow.puml @@ -0,0 +1,173 @@ +@startuml etcd-hot-standby-flow +!theme plain +skinparam activity { + BackgroundColor #E1F5FF + BorderColor #0066CC + FontColor #000000 +} +skinparam arrow { + Color #0066CC + Thickness 2 +} + +title etcd热备数据流和控制流程图 + +partition "OpLog写入流程" { +start +:Primary Master接收操作请求; +:MasterService处理请求; +:序列化metadata为JSON; +:OpLogManager.Append(); +note right + 生成: + - sequence_id (全局递增) + - timestamp_ms + - checksum + - prefix_hash +end note +:EtcdOpLogStore.WriteOpLog(); +:写入etcd: /oplog/{cluster_id}/{sequence_id}; +:触发批量更新latest_sequence_id; +note right + 批量更新策略: + - 每100条或每1秒 + - 减少etcd写入压力 +end note +stop +} + +partition "Standby同步流程" { +start +:Standby启动; +if (已有本地metadata?) then (是 - 热启动) + :读取本地last_seq_id; + :OpLogApplier.Recover(last_seq_id); +else (否 - 冷启动) + if (启用快照?) then (是) + :SnapshotProvider.LoadLatestSnapshot(); + :加载快照到StandbyMetadataStore; + :OpLogApplier.Recover(snapshot_seq_id); + endif +endif +:OpLogWatcher.StartFromSequenceId(); +:读取历史OpLog (ReadOpLogSinceWithRevision); +note right + 使用分页读取: + - 每批1000条 + - 返回etcd revision +end note +:应用OpLog到StandbyMetadataStore; +:设置next_watch_revision = revision + 1; +:启动Watch线程 (WatchWithPrefixFromRevisionV2); +:持续监听etcd OpLog变化; +stop +} + +partition "OpLog应用流程" { +start +:OpLogWatcher收到Watch事件; +:反序列化OpLogEntry; +:OpLogApplier.ApplyOpLogEntry(); +if (sequence_id == expected_sequence_id?) then (是) + :直接应用; + switch (op_type) + case (PUT_END) + :反序列化payload; + :StandbyMetadataStore.PutMetadata(); + case (PUT_REVOKE) + :StandbyMetadataStore.Remove(); + case (REMOVE) + :StandbyMetadataStore.Remove(); + endswitch + :expected_sequence_id++; + :处理pending队列; +else (否 - 乱序) + if (sequence_id < expected_sequence_id?) then (是 - 重复) + :忽略(已处理); + else (否 - 超前) + :加入pending队列; + :记录missing_sequence_ids; + if (等待超过5秒?) then (是) + :RequestMissingOpLog(); + :从etcd读取缺失OpLog; + :应用缺失OpLog; + endif + endif +endif +stop +} + +partition "故障切换流程" { +start +:etcd检测到Leader lease过期; +:MasterViewHelper检测到Leader删除; +:MasterServiceSupervisor触发切换; +:HotStandbyService.Promote(); +:停止OpLogWatcher; +:最终同步: 读取剩余OpLog; +note right + 循环读取直到: + - 没有更多OpLog + - 或读取失败 +end note +:应用所有剩余OpLog; +:ExportMetadataSnapshot(); +:GetLatestAppliedSequenceId(); +:MasterServiceSupervisor重新选举; +if (选举成功?) then (是) + :创建新MasterService; + :OpLogManager.SetInitialSequenceId(last_seq_id); + :MasterService.RestoreFromStandbySnapshot(); + note right + 恢复过程: + - 创建DummyBufferAllocator + - 重建Replica对象 + - 恢复metadata到本地 + - 不恢复lease信息 + end note + :启动新Primary服务; +else (否) + :继续作为Standby; +endif +stop +} + +partition "OpLog清理流程" { +start +:定期触发清理任务; +:EtcdOpLogStore.CleanupOpLogBefore(); +:查询etcd中最小sequence_id; +note right + Scheme 3: + - 不依赖持久化的"cleaned_upto" + - 查询实际最小sequence_id + - 更可靠 +end note +if (最小seq_id < before_sequence_id?) then (是) + :DeleteRange(/oplog/{cluster_id}/0, before_seq_id); + :删除etcd中的旧OpLog; +else (否) + :无需清理; +endif +stop +} + +partition "批量更新latest_sequence_id流程" { +start +:EtcdOpLogStore.WriteOpLog(); +:pending_latest_seq_id = sequence_id; +:pending_count++; +if (pending_count >= 100\n或距离上次更新 >= 1秒?) then (是) + :DoBatchUpdate(); + :UpdateLatestSequenceId(pending_latest_seq_id); + :写入etcd: /oplog/{cluster_id}/latest; + :pending_count = 0; + :last_update_time = now; +else (否) + :继续累积; +endif +stop +} + +@enduml + diff --git a/doc/zh/diagrams/etcd-hot-standby-sequence.puml b/doc/zh/diagrams/etcd-hot-standby-sequence.puml new file mode 100644 index 0000000000..ecdd53809f --- /dev/null +++ b/doc/zh/diagrams/etcd-hot-standby-sequence.puml @@ -0,0 +1,259 @@ +@startuml etcd-hot-standby-sequence +!theme plain +skinparam sequenceMessageAlign center +skinparam sequenceArrowThickness 2 + +title etcd热备关键时序图 + +== Primary启动 == + +actor User +participant Supervisor as "MasterServiceSupervisor" +participant ViewHelper as "MasterViewHelper" +participant EtcdHelper as "EtcdHelper" +database etcd as "etcd" +participant MasterService as "MasterService" +participant OpLogManager as "OpLogManager" +participant EtcdOpLogStore as "EtcdOpLogStore" + +User -> Supervisor: 启动服务 +activate Supervisor + +Supervisor -> ViewHelper: ElectLeader() +activate ViewHelper +ViewHelper -> EtcdHelper: GrantLease(TTL=5s) +EtcdHelper -> etcd: 创建lease +etcd --> EtcdHelper: lease_id +ViewHelper -> EtcdHelper: CreateWithLease(key, lease_id) +EtcdHelper -> etcd: 尝试创建leader key +alt 成功 + etcd --> EtcdHelper: 成功，成为Leader + ViewHelper --> Supervisor: 选举成功 +else 失败 + etcd --> EtcdHelper: 失败，已有Leader + ViewHelper -> EtcdHelper: WatchUntilDeleted() + EtcdHelper -> etcd: Watch leader key + etcd --> EtcdHelper: Leader删除事件 + ViewHelper --> Supervisor: Leader已删除，重试选举 +end +deactivate ViewHelper + +Supervisor -> MasterService: 创建MasterService +activate MasterService +MasterService -> OpLogManager: 创建OpLogManager +activate OpLogManager +MasterService -> EtcdOpLogStore: 创建EtcdOpLogStore(enable_batch=true) +activate EtcdOpLogStore +OpLogManager -> EtcdOpLogStore: SetEtcdOpLogStore() +deactivate EtcdOpLogStore +deactivate OpLogManager +deactivate MasterService + +Supervisor -> MasterService: 启动服务 +activate MasterService +MasterService -> ViewHelper: KeepLeader(lease_id) +activate ViewHelper +ViewHelper -> EtcdHelper: KeepAlive(lease_id) +EtcdHelper -> etcd: 定期续约 +deactivate ViewHelper +deactivate MasterService +deactivate Supervisor + +== Standby启动 == + +participant HotStandbyService as "HotStandbyService" +participant OpLogWatcher as "OpLogWatcher" +participant OpLogApplier as "OpLogApplier" +participant StandbyMetadataStore as "StandbyMetadataStore" + +User -> Supervisor: 启动服务(已有Leader) +activate Supervisor + +Supervisor -> ViewHelper: GetMasterView() +activate ViewHelper +ViewHelper -> EtcdHelper: Get(key) +EtcdHelper -> etcd: 查询leader +etcd --> EtcdHelper: 返回leader地址 +EtcdHelper --> ViewHelper: leader地址 +ViewHelper --> Supervisor: 已有Leader +deactivate ViewHelper + +Supervisor -> HotStandbyService: 创建HotStandbyService +activate HotStandbyService +HotStandbyService -> StandbyMetadataStore: 创建StandbyMetadataStore +activate StandbyMetadataStore +HotStandbyService -> OpLogApplier: 创建OpLogApplier +activate OpLogApplier +HotStandbyService -> OpLogWatcher: 创建OpLogWatcher +activate OpLogWatcher + +Supervisor -> HotStandbyService: Start(etcd_endpoints, cluster_id) +HotStandbyService -> EtcdHelper: ConnectToEtcdStoreClient() +EtcdHelper -> etcd: 连接etcd +etcd --> EtcdHelper: 连接成功 + +alt 热启动(已有metadata) + HotStandbyService -> OpLogApplier: GetExpectedSequenceId() + OpLogApplier --> HotStandbyService: last_seq_id + HotStandbyService -> OpLogApplier: Recover(last_seq_id) +else 冷启动(无metadata) + opt 启用快照 + HotStandbyService -> StandbyMetadataStore: LoadLatestSnapshot() + StandbyMetadataStore --> HotStandbyService: snapshot + snapshot_seq_id + HotStandbyService -> OpLogApplier: Recover(snapshot_seq_id) + end +end + +HotStandbyService -> OpLogWatcher: StartFromSequenceId(start_seq_id) +OpLogWatcher -> EtcdOpLogStore: ReadOpLogSinceWithRevision(start_seq_id) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: GetRangeAsJson(prefix, limit) +EtcdHelper -> etcd: Range Get +etcd --> EtcdHelper: OpLog entries + revision +EtcdHelper --> EtcdOpLogStore: entries + revision_id +EtcdOpLogStore --> OpLogWatcher: entries + revision_id +deactivate EtcdOpLogStore + +loop 批量读取历史OpLog + OpLogWatcher -> OpLogApplier: ApplyOpLogEntries(batch) + OpLogApplier -> StandbyMetadataStore: PutMetadata() / Remove() + StandbyMetadataStore --> OpLogApplier: 成功 + OpLogApplier --> OpLogWatcher: applied_count +end + +OpLogWatcher -> OpLogWatcher: next_watch_revision = revision_id + 1 +OpLogWatcher -> OpLogWatcher: WatchOpLog() [后台线程] +OpLogWatcher -> EtcdHelper: WatchWithPrefixFromRevisionV2(prefix, start_revision) +EtcdHelper -> etcd: Watch from revision +etcd --> EtcdHelper: OpLog事件流 +deactivate OpLogWatcher +deactivate OpLogApplier +deactivate StandbyMetadataStore +deactivate HotStandbyService +deactivate Supervisor + +== 写入操作流程 == + +participant Client + +Client -> MasterService: PutEnd(key, metadata) +activate MasterService +MasterService -> MasterService: 更新本地metadata +MasterService -> MasterService: SerializeMetadataForOpLog() +MasterService -> OpLogManager: Append(PUT_END, key, payload) +activate OpLogManager +OpLogManager -> OpLogManager: 生成sequence_id +OpLogManager -> OpLogManager: 计算checksum和prefix_hash +OpLogManager -> EtcdOpLogStore: WriteOpLog(entry) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: Put(key, value) +EtcdHelper -> etcd: 写入OpLog +etcd --> EtcdHelper: 成功 +EtcdHelper --> EtcdOpLogStore: 成功 +EtcdOpLogStore -> EtcdOpLogStore: TriggerBatchUpdateIfNeeded() +deactivate EtcdOpLogStore +OpLogManager --> MasterService: sequence_id +MasterService --> Client: 成功 +deactivate MasterService +deactivate OpLogManager + +' Standby接收OpLog +etcd -> OpLogWatcher: Watch事件(PUT) +activate OpLogWatcher +OpLogWatcher -> OpLogWatcher: DeserializeOpLogEntry() +OpLogWatcher -> OpLogApplier: ApplyOpLogEntry(entry) +activate OpLogApplier +OpLogApplier -> OpLogApplier: CheckSequenceOrder() +alt 顺序正确 + OpLogApplier -> OpLogApplier: ApplyPutEnd() + OpLogApplier -> StandbyMetadataStore: PutMetadata(key, metadata) + activate StandbyMetadataStore + StandbyMetadataStore --> OpLogApplier: 成功 + deactivate StandbyMetadataStore + OpLogApplier --> OpLogWatcher: 成功 +else 顺序错误(乱序) + OpLogApplier -> OpLogApplier: 加入pending队列 + OpLogApplier -> OpLogApplier: RequestMissingOpLog() + OpLogApplier -> EtcdOpLogStore: ReadOpLog(missing_seq_id) + activate EtcdOpLogStore + EtcdOpLogStore -> EtcdHelper: Get(key) + EtcdHelper -> etcd: 查询OpLog + etcd --> EtcdHelper: OpLog entry + EtcdHelper --> EtcdOpLogStore: entry + EtcdOpLogStore --> OpLogApplier: entry + deactivate EtcdOpLogStore + OpLogApplier -> OpLogApplier: ProcessPendingEntries() +end +deactivate OpLogApplier +deactivate OpLogWatcher + +== 故障切换流程 == + +participant NewPrimary as "New Primary\n(MasterService)" + +etcd -> ViewHelper: Leader lease过期 +activate ViewHelper +ViewHelper -> Supervisor: Leader已删除 +activate Supervisor + +Supervisor -> HotStandbyService: Promote() +activate HotStandbyService +HotStandbyService -> OpLogWatcher: Stop() +activate OpLogWatcher +OpLogWatcher --> HotStandbyService: 已停止 +deactivate OpLogWatcher + +HotStandbyService -> EtcdOpLogStore: ReadOpLogSince(last_seq_id) +activate EtcdOpLogStore +EtcdOpLogStore -> EtcdHelper: GetRangeAsJson() +EtcdHelper -> etcd: Range Get +etcd --> EtcdHelper: 剩余OpLog entries +EtcdHelper --> EtcdOpLogStore: entries +EtcdOpLogStore --> HotStandbyService: entries +deactivate EtcdOpLogStore + +loop 最终同步 + HotStandbyService -> OpLogApplier: ApplyOpLogEntries(batch) + activate OpLogApplier + OpLogApplier -> StandbyMetadataStore: 应用OpLog + StandbyMetadataStore --> OpLogApplier: 成功 + OpLogApplier --> HotStandbyService: applied_count + deactivate OpLogApplier +end + +HotStandbyService -> HotStandbyService: ExportMetadataSnapshot() +HotStandbyService -> HotStandbyService: GetLatestAppliedSequenceId() +HotStandbyService --> Supervisor: snapshot + last_seq_id +deactivate HotStandbyService + +Supervisor -> ViewHelper: ElectLeader() [重新选举] +activate ViewHelper +ViewHelper -> EtcdHelper: GrantLease() + CreateWithLease() +EtcdHelper -> etcd: 选举Leader +etcd --> EtcdHelper: 选举成功 +EtcdHelper --> ViewHelper: 成为新Leader +ViewHelper --> Supervisor: 选举成功 +deactivate ViewHelper + +Supervisor -> NewPrimary: 创建MasterService +activate NewPrimary +NewPrimary -> OpLogManager: SetInitialSequenceId(last_seq_id) +activate OpLogManager +OpLogManager --> NewPrimary: 已设置 +deactivate OpLogManager +NewPrimary -> NewPrimary: RestoreFromStandbySnapshot(snapshot) +NewPrimary -> NewPrimary: 恢复metadata到本地 +NewPrimary --> Supervisor: 恢复完成 +deactivate NewPrimary + +Supervisor -> NewPrimary: 启动服务 +activate NewPrimary +NewPrimary -> ViewHelper: KeepLeader(lease_id) +activate ViewHelper +ViewHelper -> EtcdHelper: KeepAlive(lease_id) +deactivate ViewHelper +deactivate NewPrimary +deactivate Supervisor + +@enduml + diff --git a/doc/zh/diagrams/mooncake-transfer-flow.puml b/doc/zh/diagrams/mooncake-transfer-flow.puml new file mode 100644 index 0000000000..10f41de91a --- /dev/null +++ b/doc/zh/diagrams/mooncake-transfer-flow.puml @@ -0,0 +1,80 @@ +@startuml Mooncake Store 数据传输流程 + +!theme plain +skinparam backgroundColor #FFFFFF +skinparam activity { + BackgroundColor #E8F4F8 + BorderColor #4A90E2 + FontColor #000000 +} +skinparam arrow { + Color #4A90E2 +} + +title Mooncake Store 数据传输流程 + +start + +:TransferSubmitter 接收传输请求\n(Replica Descriptor, Slices); + +:从 Master Service 获取\nReplica Descriptor; + +note right + **Replica Descriptor** 包含： + - transport_endpoint: 目标端点 + - buffer_address: 内存地址 + - size: 数据大小 +end note + +:调用 selectStrategy()\n选择传输策略; + +if (是否为本地传输?) then (是) + :执行 LOCAL_MEMCPY\n本地内存拷贝; + note right + 源和目标在同一进程 + 直接 memcpy + end note + :返回成功; + stop +else (否) + :创建 TransferEngine 传输请求; + + if (传输协议选择) then (RDMA) + :初始化 RDMA Transport; + :建立 RDMA 连接; + :执行 RDMA Write/Read\n零拷贝传输; + note right + **RDMA 优势**： + - 零拷贝，绕过内核 + - 低延迟 + - 高带宽 + end note + :数据直接写入目标 Segment\nAllocatedBuffer; + else (TCP) + :初始化 TCP Transport; + :建立 TCP 连接; + :执行 TCP Write/Read\n标准网络传输; + note right + **TCP 传输**： + - 标准网络协议 + - 兼容性好 + - 需要内核参与 + end note + :数据写入目标 Segment\nAllocatedBuffer; + endif + + :等待传输完成; + + if (传输是否成功?) then (是) + :更新传输指标; + :返回成功; + else (否) + :记录错误日志; + :返回失败; + endif + + stop +endif + +@enduml + diff --git a/doc/zh/diagrams/oplog-data-flow.puml b/doc/zh/diagrams/oplog-data-flow.puml new file mode 100644 index 0000000000..5f5b1175a9 --- /dev/null +++ b/doc/zh/diagrams/oplog-data-flow.puml @@ -0,0 +1,58 @@ +@startuml oplog-data-flow +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +actor Client +participant "Primary Master" as Primary +participant "OpLogManager" as OplogMgr +participant "EtcdOpLogStore" as EtcdStore +database etcd +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier +participant "Standby Master" as Standby + +== PUT_END 操作流程 == + +Client -> Primary: PutEnd(key, ...) +activate Primary +Primary -> OplogMgr: Append(PUT_END, key) +activate OplogMgr +OplogMgr -> OplogMgr: 生成 sequence_id\n生成 key_sequence_id +OplogMgr -> EtcdStore: WriteOpLog(entry) +activate EtcdStore +EtcdStore -> etcd: PUT /oplog/{seq} +activate etcd +etcd --> EtcdStore: Success +deactivate etcd +EtcdStore --> OplogMgr: Success +deactivate EtcdStore +OplogMgr --> Primary: sequence_id +deactivate OplogMgr +Primary --> Client: Success +deactivate Primary + +== Standby 同步流程 == + +etcd -> Watcher: Watch Event (新 OpLog) +activate Watcher +Watcher -> Applier: ApplyOpLogEntry(entry) +activate Applier +Applier -> Applier: CheckSequenceOrder() +alt 顺序正确 + Applier -> Standby: UpdateMetadata(key, ...) + activate Standby + Standby --> Applier: Success + deactivate Standby +else 顺序错误 + Applier -> Applier: RollbackAndReplay() + Applier -> etcd: ReadOpLogForKey() + etcd --> Applier: OpLog Entries + Applier -> Applier: Replay OpLog +end +Applier --> Watcher: Success +deactivate Applier +deactivate Watcher + +@enduml + diff --git a/doc/zh/diagrams/oplog-failover-sequence.puml b/doc/zh/diagrams/oplog-failover-sequence.puml new file mode 100644 index 0000000000..5945ecbb10 --- /dev/null +++ b/doc/zh/diagrams/oplog-failover-sequence.puml @@ -0,0 +1,66 @@ +@startuml oplog-failover-sequence +!theme plain +skinparam backgroundColor #FFFFFF +skinparam sequenceMessageAlign center + +participant "Primary Master" as Primary +database etcd +participant "Standby Master" as Standby +participant "MasterServiceSupervisor" as Supervisor +participant "HotStandbyService" as HotStandby +participant "OpLogWatcher" as Watcher +participant "OpLogApplier" as Applier + +== 正常运行阶段 == + +Primary -> etcd: KeepAlive Lease +etcd -> Supervisor: Watch Leader (存在) +activate Supervisor +Supervisor -> HotStandby: StartStandby() +activate HotStandby +HotStandby -> Watcher: Start() +activate Watcher +Watcher -> etcd: Watch OpLog +etcd -> Watcher: OpLog Events +Watcher -> Applier: ApplyOpLogEntry() +activate Applier +Applier -> Standby: UpdateMetadata() +deactivate Applier +deactivate Watcher +deactivate HotStandby +deactivate Supervisor + +== Primary 故障 == + +Primary -x etcd: Lease Expired (故障) +etcd -> Supervisor: Leader Deleted Event +activate Supervisor +Supervisor -> HotStandby: Stop() +activate HotStandby +HotStandby -> Watcher: Stop() +deactivate Watcher +deactivate HotStandby + +== Standby 提升为 Primary == + +Supervisor -> Standby: Promote() +activate Standby +Standby -> Standby: 初始化 Lease\n清理过期 metadata +Standby -> etcd: ElectLeader() +activate etcd +etcd -> Standby: Leader Elected +deactivate etcd +Standby -> Supervisor: Primary Mode +deactivate Standby +deactivate Supervisor + +note over Standby + 1. 停止 Standby 服务 + 2. 遍历所有 metadata + 3. 对 lease=0 的对象授予默认租约 + 4. 执行一次完整的 metadata 清理 + 5. 开始 Leader 选举 +end note + +@enduml + diff --git a/doc/zh/diagrams/oplog-hot-standby-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-architecture.puml new file mode 100644 index 0000000000..d8ee863650 --- /dev/null +++ b/doc/zh/diagrams/oplog-hot-standby-architecture.puml @@ -0,0 +1,65 @@ +@startuml oplog-hot-standby-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 12 + +package "Primary Master" #E8F4F8 { + component [MasterService] as MasterService + component [OpLogManager] as OpLogManager + component [EtcdOpLogStore] as EtcdOpLogStore + + MasterService --> OpLogManager : 记录操作 + OpLogManager --> EtcdOpLogStore : 写入 OpLog +} + +package "etcd Cluster" #FFF4E6 { + database [etcd] as etcd +} + +package "Standby Master" #F0F8E8 { + component [MasterServiceSupervisor] as Supervisor + component [HotStandbyService] as HotStandby + component [OpLogWatcher] as Watcher + component [OpLogApplier] as Applier + component [MetadataStore] as MetadataStore + + Supervisor --> HotStandby : 启动/停止 + HotStandby --> Watcher : Watch OpLog + Watcher --> Applier : 应用 OpLog + Applier --> MetadataStore : 更新 metadata +} + +EtcdOpLogStore --> etcd : 写入 OpLog +etcd --> Watcher : Watch 事件 + +note right of OpLogManager + **职责**： + - 生成 sequence_id + - 生成 key_sequence_id + - 维护内存缓冲区 +end note + +note right of Applier + **职责**： + - 检查顺序 + - 处理乱序 + - 清理过期条目 +end note + +note right of MasterService + **操作**： + - PutEnd() + - Remove() + - Eviction() +end note + +note right of etcd + **Key 设计**： + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +@enduml + diff --git a/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml b/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml new file mode 100644 index 0000000000..e34f773607 --- /dev/null +++ b/doc/zh/diagrams/oplog-hot-standby-complete-architecture.puml @@ -0,0 +1,123 @@ +@startuml oplog-hot-standby-complete-architecture +!theme plain +skinparam backgroundColor #FFFFFF +skinparam componentStyle rectangle +skinparam defaultFontSize 11 + +package "Master Cluster (HA)" { + + package "Primary Master (Leader)" #90EE90 { + component [MasterService\nMetadata Management] as MasterService + component [OpLogManager\nGenerate & Buffer] as OpLogManager + component [EtcdOpLogStore\nWrite to etcd] as EtcdOpLogStore + + MasterService --> OpLogManager : Step 1:\nWrite op generates OpLog + OpLogManager --> EtcdOpLogStore : Step 2:\nWrite OpLog to etcd + } + + package "Standby Master 1 (Hot Standby)" #FFA500 { + component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor1 + component [HotStandbyService\nCore Service] as HotStandby1 + component [OpLogWatcher\nWatch etcd] as OpLogWatcher1 + component [OpLogApplier\nApply Changes] as OpLogApplier1 + component [MetadataStore\nReplica Data] as MetadataStore1 + + Supervisor1 --> HotStandby1 : Start/Stop\nStandby mode + HotStandby1 --> OpLogWatcher1 : Step 3:\nWatch OpLog + OpLogWatcher1 --> OpLogApplier1 : Step 4:\nForward OpLog + OpLogApplier1 --> MetadataStore1 : Step 5:\nApply changes + } + + package "Standby Master 2 (Hot Standby)" #FFA500 { + component [MasterServiceSupervisor\nLifecycle Manager] as Supervisor2 + component [HotStandbyService\nCore Service] as HotStandby2 + component [OpLogWatcher\nWatch etcd] as OpLogWatcher2 + component [OpLogApplier\nApply Changes] as OpLogApplier2 + component [MetadataStore\nReplica Data] as MetadataStore2 + + Supervisor2 --> HotStandby2 : Start/Stop\nStandby mode + HotStandby2 --> OpLogWatcher2 : Watch OpLog + OpLogWatcher2 --> OpLogApplier2 : Forward OpLog + OpLogApplier2 --> MetadataStore2 : Apply changes + } +} + +package "vLLM Inference Cluster" #ADD8E6 { + component [vLLM Instance 1] as vLLM1 + component [vLLM Instance 2] as vLLM2 + component [vLLM Instance N] as vLLMN +} + +package "etcd Cluster" #DDA0DD { + database [Service Discovery\n/mooncake/master/view] as ServiceDiscovery + database [Leader Election\n/mooncake/master/leader] as LeaderElection + database [OpLog Storage\n/oplog/{cluster_id}/{sequence_id}] as OpLogStorage +} + +' Primary interactions +MasterService <--> vLLM1 : RPC\n(Query/Put/Remove) +MasterService <--> vLLM2 : RPC\n(Query/Put/Remove) +MasterService <--> vLLMN : RPC\n(Query/Put/Remove) + +' etcd interactions - OpLog +EtcdOpLogStore --> OpLogStorage : Write OpLog\n(sequence_id) +OpLogStorage --> OpLogWatcher1 : Watch Events\n(Real-time sync) +OpLogStorage --> OpLogWatcher2 : Watch Events\n(Real-time sync) + +' etcd interactions - Leader Election +MasterService --> LeaderElection : Lease KeepAlive\n(TTL=5s) +Supervisor1 --> LeaderElection : Watch Leader Key +Supervisor2 --> LeaderElection : Watch Leader Key + +note right of MasterService + **Primary Responsibilities:** + - Handle all client requests + - Generate OpLog for writes + - Write OpLog to etcd + - Manage metadata +end note + +note right of HotStandby1 + **Standby Responsibilities:** + - Watch OpLog from etcd + - Apply OpLog to metadata + - Maintain replica metadata + - Ready for promotion +end note + +note right of OpLogManager + **OpLogManager:** + - Generate sequence_id + - Generate key_sequence_id + - Maintain buffer +end note + +note right of OpLogApplier1 + **OpLogApplier:** + - Check sequence order + - Handle out-of-order + - Cleanup stale entries +end note + +note right of OpLogStorage + **etcd OpLog Key:** + - /oplog/{cluster_id}/{sequence_id} + - /oplog/{cluster_id}/latest + - Watch API +end note + +note right of LeaderElection + **etcd Services:** + - Service Discovery + - Leader Election + - Lease Management +end note + +legend right + |<#90EE90> **Green (Primary)** | Active leader handling requests | + |<#FFA500> **Orange (Standby)** | Hot standby with replica data | + |<#DDA0DD> **Purple (etcd)** | Coordination & OpLog storage | + |<#ADD8E6> **Light Blue (Clients)** | vLLM inference instances | +endlegend + +@enduml diff --git a/doc/zh/rfc-batched-delete-events-via-etcd.md b/doc/zh/rfc-batched-delete-events-via-etcd.md new file mode 100644 index 0000000000..8c0196b0b5 --- /dev/null +++ b/doc/zh/rfc-batched-delete-events-via-etcd.md @@ -0,0 +1,455 @@ +# 基于 etcd 批量压缩 Delete 事件方案 + +## 问题背景 + +### 当前设计回顾 + +根据之前的分析： +1. **驱逐事件频率极高**：可达 130,000 次/秒 +2. **当前方案**： + - 显式 Delete 事件 → 写入 etcd（强一致性） + - 驱逐产生的 Delete 事件 → 不写入 etcd（由 Standby 自己根据租约到期决定） + +### 新方案需求 + +用户提出：使用 etcd 作为中间媒介，对驱逐产生的 delete 事件进行**批量压缩组装**后写入 etcd，而不是每次驱逐都写入一次。 + +## 方案设计 + +### 1. 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Eviction │ │ Delete │ │ +│ │ Thread │ │ Event │ │ +│ │ │ │ Buffer │ │ +│ ┌──────────────┘ ┌──────────────┘ │ +│ │ │ │ +│ │ 驱逐事件 │ 显式 Delete │ +│ ▼ ▼ │ +│ ┌──────────────────────────────────────┐ │ +│ │ BatchedDeleteEventManager │ │ +│ │ - 批量收集 delete 事件 │ │ +│ │ - 压缩/去重 │ │ +│ │ - 定时批量写入 etcd │ │ +│ └──────────────────────────────────────┘ │ +│ │ │ +│ │ 批量写入 │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ etcd │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ + │ Watch + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ ┌──────────────────────────────────────┐ │ +│ │ DeleteEventWatcher │ │ +│ │ - Watch etcd delete events │ │ +│ │ - 解压缩/应用 delete 事件 │ │ +│ └──────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +### 2. 批量压缩策略 + +#### 方案 A：时间窗口批量（推荐） + +**原理**： +- 收集固定时间窗口内的所有 delete 事件（如 1 秒） +- 时间窗口到期后，批量写入 etcd +- 使用压缩格式减少数据量 + +**优点**： +- 简单易实现 +- 延迟可控（最多 1 秒） +- 批量写入减少 etcd 压力 + +**缺点**： +- 固定延迟（1 秒） +- 如果事件很少，也会等待 1 秒 + +#### 方案 B：大小阈值批量 + +**原理**： +- 收集 delete 事件直到达到阈值（如 1000 条） +- 达到阈值后立即批量写入 +- 同时设置最大等待时间（如 1 秒） + +**优点**： +- 高吞吐时延迟低（立即写入） +- 低吞吐时延迟可控（最多 1 秒） + +**缺点**： +- 实现稍复杂 +- 需要同时考虑大小和时间两个维度 + +#### 方案 C：混合策略（推荐） + +**原理**： +- 同时设置大小阈值（如 1000 条）和时间窗口（如 1 秒） +- 满足任一条件即批量写入 +- 使用压缩格式减少数据量 + +**优点**： +- 兼顾性能和延迟 +- 高吞吐时立即写入，低吞吐时定时写入 + +### 3. 压缩格式设计 + +#### 格式 A：JSON 数组（简单） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "keys": [ + "key1", "key2", "key3", ... + ], + "count": 1000 +} +``` + +**优点**： +- 简单易实现 +- 易于调试 + +**缺点**： +- 数据量大（每个 key 都是完整字符串） +- etcd value 大小限制（1.5MB） + +#### 格式 B：前缀压缩（推荐） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "compressed": true, + "format": "prefix_tree", + "data": { + "prefix1": ["suffix1", "suffix2", ...], + "prefix2": ["suffix3", "suffix4", ...], + ... + }, + "count": 1000 +} +``` + +**优点**： +- 压缩率高（如果 key 有共同前缀） +- 减少 etcd value 大小 + +**缺点**： +- 实现复杂 +- 如果 key 没有共同前缀，压缩效果差 + +#### 格式 C：Bloom Filter + Key List（推荐用于大量 key） + +**原理**： +- 使用 Bloom Filter 快速判断 key 是否存在 +- 对于少量 key，直接存储完整列表 +- 对于大量 key，使用 Bloom Filter + 采样 + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "count": 10000, + "bloom_filter": "base64_encoded_bloom_filter", + "sample_keys": ["key1", "key2", ...], // 前 100 个 key 作为样本 + "hash_prefix": "abc123" // 如果 key 有 hash 前缀，可以进一步压缩 +} +``` + +**优点**： +- 压缩率极高（Bloom Filter 很小） +- 适合大量 key 的场景 + +**缺点**： +- 有误判率（Bloom Filter 特性） +- 需要额外存储完整 key 列表用于精确匹配 + +#### 格式 D：简单列表 + 压缩（推荐用于中等数量 key） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "timestamp": 1704110400000, + "keys": ["key1", "key2", ...], // 最多 1000 条 + "count": 1000 +} +``` + +**优点**： +- 简单直接 +- 无压缩开销 +- 易于解析和应用 + +**缺点**： +- 如果 key 很长，数据量大 +- 受 etcd value 大小限制 + +### 4. etcd Key 设计 + +#### 方案 A：单个 Key + 版本号 + +``` +mooncake-store/deletes/batch/{batch_id} +``` + +**优点**： +- 简单 +- 易于 Watch + +**缺点**： +- 如果批量很大，单个 value 可能超过 etcd 限制（1.5MB） +- 需要处理 value 大小限制 + +#### 方案 B：分片 Key（推荐） + +``` +mooncake-store/deletes/batch/{batch_id}/shard/{shard_id} +``` + +**原理**： +- 将大批量分成多个 shard（每个 shard 最多 1000 条 key） +- 每个 shard 写入一个 etcd key +- 使用事务保证原子性 + +**优点**： +- 避免单个 value 过大 +- 可以并行写入多个 shard +- 易于 Watch 和解析 + +**缺点**： +- 需要管理多个 key +- 需要处理部分写入失败的情况 + +#### 方案 C：Stream 模式（使用 etcd 的 Watch） + +``` +mooncake-store/deletes/stream/{sequence_id} +``` + +**原理**： +- 每个批量写入一个 sequence_id +- Standby Watch 连续的 sequence_id +- 支持断点续传 + +**优点**： +- 支持顺序处理 +- 支持断点续传 +- 易于实现流式处理 + +**缺点**： +- 需要管理 sequence_id +- 需要处理 sequence_id 跳跃的情况 + +### 5. 实现细节 + +#### 5.1 BatchedDeleteEventManager + +```cpp +class BatchedDeleteEventManager { +public: + struct BatchConfig { + size_t max_batch_size = 1000; // 最大批量大小 + uint32_t max_batch_interval_ms = 1000; // 最大批量间隔（1秒） + }; + + // 添加 delete 事件到批量缓冲区 + void AddDeleteEvent(const std::string& key); + + // 强制刷新批量（立即写入） + void Flush(); + +private: + // 批量写入到 etcd + void FlushBatch(); + + // 压缩批量数据 + std::string CompressBatch(const std::vector& keys); + + // 解压缩批量数据 + std::vector DecompressBatch(const std::string& data); + + std::mutex mutex_; + std::vector pending_keys_; + std::chrono::steady_clock::time_point last_flush_time_; + BatchConfig config_; + std::thread flush_thread_; + std::atomic running_{false}; +}; +``` + +#### 5.2 批量写入逻辑 + +```cpp +void BatchedDeleteEventManager::FlushBatch() { + std::lock_guard lock(mutex_); + + if (pending_keys_.empty()) { + return; + } + + // 压缩数据 + std::string compressed_data = CompressBatch(pending_keys_); + + // 检查大小限制 + if (compressed_data.size() > kMaxEtcdValueSize) { + // 分片写入 + FlushBatchSharded(pending_keys_); + } else { + // 单个 key 写入 + FlushBatchSingle(compressed_data); + } + + pending_keys_.clear(); + last_flush_time_ = std::chrono::steady_clock::now(); +} +``` + +#### 5.3 Standby 端处理 + +```cpp +class DeleteEventWatcher { +public: + // Watch etcd delete events + void WatchDeleteEvents(); + + // 处理批量 delete 事件 + void HandleBatchDeleteEvent(const std::string& batch_data); + +private: + // 解压缩并应用 delete 事件 + void ApplyDeleteEvents(const std::vector& keys); +}; +``` + +## 方案评估 + +### 优点 + +1. **减少 etcd 压力** + - 从 130,000 次/秒 → 约 130 次/秒（批量 1000 条） + - 减少 1000 倍写入压力 + +2. **保持高可靠性** + - 仍然使用 etcd 的强一致性 + - Standby 可以通过 Watch 实时获取 + +3. **延迟可控** + - 批量间隔可配置（如 1 秒） + - 高吞吐时立即写入（大小阈值） + +4. **压缩减少存储** + - 使用压缩格式减少 etcd value 大小 + - 可以存储更多 delete 事件 + +### 缺点和挑战 + +1. **延迟问题** + - 批量写入会有延迟（最多 1 秒） + - 如果 Primary 在批量写入前崩溃，可能丢失部分 delete 事件 + +2. **数据丢失风险** + - 如果 Primary 在批量写入前崩溃，缓冲区中的 delete 事件会丢失 + - **解决方案**：使用持久化缓冲区（如 DragonflyDB）或定期 checkpoint + +3. **etcd 容量限制** + - etcd value 大小限制（1.5MB） + - 需要分片处理大批量 + +4. **压缩开销** + - 压缩/解压缩有 CPU 开销 + - 需要权衡压缩率和性能 + +5. **Standby 处理复杂度** + - 需要解压缩批量数据 + - 需要处理分片数据 + +### 与当前方案对比 + +| 特性 | 当前方案（不写入 etcd） | 新方案（批量写入 etcd） | +|------|------------------------|------------------------| +| **可靠性** | 中等（依赖租约同步） | 高（etcd 强一致性） | +| **延迟** | 0（实时） | 1 秒（批量延迟） | +| **etcd 压力** | 0 | 低（批量写入） | +| **数据丢失风险** | 低（Standby 自己决定） | 中等（批量缓冲区可能丢失） | +| **实现复杂度** | 低 | 中等 | +| **Standby 一致性** | 可能不一致（租约时间差） | 强一致（etcd 保证） | + +## 推荐方案 + +### 混合方案（推荐） + +**核心思想**： +1. **显式 Delete 事件**：立即写入 etcd（保持当前设计） +2. **驱逐 Delete 事件**：批量压缩写入 etcd（新方案） + +**实现策略**： +- 使用**混合策略**（大小阈值 + 时间窗口） + - 大小阈值：1000 条 + - 时间窗口：1 秒 +- 使用**简单列表格式**（中等数量 key） + - 如果 key 数量 > 1000，自动分片 +- 使用**分片 Key** 避免单个 value 过大 +- 添加**持久化缓冲区**（可选） + - 使用 DragonflyDB 作为缓冲区 + - 定期 checkpoint 到 etcd + +### 实施步骤 + +#### Phase 1：基础批量写入（低风险） + +1. 实现 `BatchedDeleteEventManager` +2. 使用简单列表格式 +3. 使用时间窗口批量（1 秒） +4. 单个 etcd key 写入 + +#### Phase 2：优化批量策略（中风险） + +1. 添加大小阈值 +2. 实现分片写入 +3. 添加压缩格式 + +#### Phase 3：持久化缓冲区（可选，高风险） + +1. 使用 DragonflyDB 作为缓冲区 +2. 定期 checkpoint 到 etcd +3. 故障恢复机制 + +## 总结 + +### 方案可行性：✅ **可行** + +**优点**： +- 大幅减少 etcd 压力（1000 倍减少） +- 保持高可靠性（etcd 强一致性） +- 延迟可控（1 秒内） + +**需要注意**： +- 批量延迟（最多 1 秒） +- 数据丢失风险（需要持久化缓冲区） +- etcd 容量限制（需要分片） + +### 建议 + +1. **先实现 Phase 1**（基础批量写入） + - 验证方案可行性 + - 评估性能影响 + +2. **根据实际效果决定是否继续** + - 如果效果良好，继续 Phase 2 + - 如果效果不佳，考虑其他方案 + +3. **关键指标**： + - etcd 写入 QPS + - Standby 同步延迟 + - 数据丢失率 + diff --git a/doc/zh/rfc-batched-delete-timing-issues.md b/doc/zh/rfc-batched-delete-timing-issues.md new file mode 100644 index 0000000000..d770f7f699 --- /dev/null +++ b/doc/zh/rfc-batched-delete-timing-issues.md @@ -0,0 +1,419 @@ +# 批量写入 etcd 的时序问题分析 + +## 问题概述 + +批量写入 etcd 的方案可能存在以下时序问题： + +1. **事件顺序问题**：批量写入可能导致事件顺序混乱 +2. **竞态条件**：Standby 可能在不同时间看到不同批次的事件 +3. **重复删除问题**：同一个 key 可能出现在多个批次中 +4. **延迟导致的不一致**：批量延迟可能导致 Standby 看到过期数据 + +## 时序问题详细分析 + +### 问题 1：事件顺序混乱 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1 +T2: 驱逐 key2 → 加入 batch1 +T3: 显式删除 key1 → 立即写入 etcd (单个事件) +T4: batch1 写入 etcd (包含 key1, key2) +``` + +**问题**： +- Standby 在 T3 看到 key1 被删除（显式删除） +- Standby 在 T4 又看到 key1 被删除（批量删除） +- 或者 Standby 先看到 T4 的批量删除，后看到 T3 的显式删除 + +#### 影响 + +1. **重复删除**：Standby 可能尝试删除同一个 key 两次 + - 影响：性能开销，但通常可以容忍（幂等操作） + +2. **顺序混乱**：如果 key1 在 T3 被显式删除，但在 T4 的批量中又出现 + - 影响：Standby 可能看到"删除 → 存在 → 删除"的奇怪序列 + +### 问题 2：批量延迟导致的不一致 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1（未写入 etcd） +T2: Standby 读取 key1 → 看到 key1 存在（因为 batch1 还没写入） +T3: batch1 写入 etcd（包含 key1 的删除） +T4: Standby Watch 到 key1 被删除 +``` + +**问题**： +- T1-T3 期间，Standby 可能看到过期的 key1 +- 如果 Standby 在 T2 读取 key1，然后在 T4 看到删除，可能导致不一致 + +#### 影响 + +1. **短暂的不一致**：Standby 可能在短时间内看到 Primary 已经删除的 key + - 影响：可能导致 Standby 返回过期数据 + +2. **租约续约问题**：如果 Standby 在 T2 续约了 key1 的租约，但 key1 在 T1 已经被删除 + - 影响：Standby 可能续约了不存在的 key + +### 问题 3：批量边界导致的事件丢失 + +#### 场景描述 + +``` +时间线： +T1: 驱逐 key1 → 加入 batch1 +T2: batch1 达到阈值（1000条）→ 开始写入 etcd +T3: 驱逐 key2 → 加入 batch2（新批次） +T4: batch1 写入完成 +T5: Primary 崩溃 +``` + +**问题**： +- batch1 中的 key1 已经写入 etcd（Standby 能看到） +- batch2 中的 key2 还未写入 etcd（Standby 看不到） +- 如果 Primary 在 T5 崩溃，batch2 中的事件会丢失 + +#### 影响 + +1. **部分事件丢失**：Standby 可能只看到部分删除事件 + - 影响：Standby 和 Primary 的数据不一致 + +2. **恢复困难**：Primary 恢复后，无法知道哪些 key 应该被删除 + - 影响：需要重新同步或清理 + +### 问题 4：Watch 顺序问题 + +#### 场景描述 + +``` +时间线： +T1: batch1 写入 etcd (seq=100, keys=[key1, key2]) +T2: 显式删除 key3 → 立即写入 etcd (seq=101) +T3: batch2 写入 etcd (seq=102, keys=[key4, key5]) +``` + +**Standby Watch 顺序**： +- 如果 Standby 的 Watch 是顺序的，会按 seq=100, 101, 102 的顺序看到 +- 但如果 etcd 的 Watch 有延迟，可能看到不同的顺序 + +#### 影响 + +1. **事件顺序保证**：etcd 的 Watch 保证顺序，但批量写入可能打乱逻辑顺序 + - 影响：Standby 可能看到"批量删除 key1 → 显式删除 key3 → 批量删除 key2"的序列 + +2. **时间戳混乱**：批量中的 key 可能有不同的实际删除时间，但共享同一个时间戳 + - 影响：Standby 无法区分 key 的实际删除顺序 + +## 解决方案 + +### 方案 1：时间戳 + 序列号（推荐） + +#### 设计 + +每个 delete 事件包含： +- `timestamp`：实际删除时间（微秒精度） +- `sequence_id`：全局序列号（保证顺序） +- `batch_id`：批次 ID（用于去重） + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "events": [ + { + "key": "key1", + "timestamp": 1704110400123456, // 实际删除时间 + "sequence_id": 1001, + "source": "eviction" + }, + { + "key": "key2", + "timestamp": 1704110400123457, + "sequence_id": 1002, + "source": "eviction" + } + ] +} +``` + +#### 优点 + +- 保持事件的实际顺序 +- 支持去重（通过 sequence_id） +- 支持时间戳排序 + +#### 缺点 + +- 需要维护全局序列号 +- 实现复杂度稍高 + +### 方案 2：去重机制 + +#### 设计 + +在 Standby 端维护一个"已删除 key"的集合，用于去重： + +```cpp +class DeleteEventProcessor { +private: + std::unordered_set deleted_keys_; + std::mutex mutex_; + +public: + void ProcessDeleteEvent(const std::string& key) { + std::lock_guard lock(mutex_); + + // 去重：如果已经删除过，跳过 + if (deleted_keys_.find(key) != deleted_keys_.end()) { + VLOG(1) << "Key " << key << " already deleted, skipping"; + return; + } + + // 执行删除 + DeleteKey(key); + deleted_keys_.insert(key); + + // 定期清理 deleted_keys_（避免内存泄漏） + if (deleted_keys_.size() > 100000) { + CleanupDeletedKeys(); + } + } +}; +``` + +#### 优点 + +- 简单易实现 +- 有效防止重复删除 +- 性能开销小 + +#### 缺点 + +- 需要维护内存中的集合 +- 需要定期清理（避免内存泄漏） + +### 方案 3：版本号机制 + +#### 设计 + +每个 delete 事件包含版本号，Standby 只处理版本号更高的删除事件： + +```json +{ + "batch_id": "2024-01-01T12:00:00.000Z", + "version": 100, // 全局版本号 + "events": [ + { + "key": "key1", + "key_version": 50, // key 的版本号 + "timestamp": 1704110400123456 + } + ] +} +``` + +#### 优点 + +- 支持版本比较 +- 可以检测过期事件 + +#### 缺点 + +- 需要维护版本号 +- 实现复杂度高 + +### 方案 4：分离显式删除和批量删除 + +#### 设计 + +使用不同的 etcd key 前缀区分显式删除和批量删除： + +``` +mooncake-store/deletes/explicit/{key_hash} # 显式删除 +mooncake-store/deletes/batch/{batch_id} # 批量删除 +``` + +Standby 处理逻辑： +1. 先处理显式删除（优先级高） +2. 再处理批量删除（去重） + +#### 优点 + +- 清晰区分两种删除类型 +- 可以设置不同的优先级 + +#### 缺点 + +- 需要维护两套逻辑 +- 可能增加 etcd key 数量 + +### 方案 5：事务保证原子性 + +#### 设计 + +使用 etcd 事务保证批量写入的原子性： + +```cpp +void BatchedDeleteEventManager::FlushBatch() { + // 构建事务 + etcd::Transaction txn; + + for (const auto& key : pending_keys_) { + std::string etcd_key = BuildDeleteKey(key); + txn.Put(etcd_key, SerializeDeleteEvent(key)); + } + + // 提交事务（原子性保证） + auto result = etcd_client_.Commit(txn); + if (!result.success) { + LOG(ERROR) << "Failed to commit batch delete events"; + // 重试或持久化到缓冲区 + } +} +``` + +#### 优点 + +- 保证批量写入的原子性 +- 要么全部成功，要么全部失败 + +#### 缺点 + +- etcd 事务有性能开销 +- 如果批量很大，事务可能失败 + +## 推荐方案：组合方案 + +### 核心设计 + +1. **时间戳 + 序列号**：每个事件包含实际删除时间和序列号 +2. **去重机制**：Standby 端维护已删除 key 集合 +3. **分离显式删除和批量删除**：使用不同的 etcd key 前缀 +4. **持久化缓冲区**：使用 DragonflyDB 作为缓冲区，避免数据丢失 + +### 实现示例 + +#### Primary 端 + +```cpp +class BatchedDeleteEventManager { +private: + uint64_t global_sequence_id_{0}; + std::mutex sequence_mutex_; + + struct DeleteEvent { + std::string key; + uint64_t timestamp; // 实际删除时间 + uint64_t sequence_id; // 全局序列号 + std::string source; // "explicit" or "eviction" + }; + + void FlushBatch() { + std::lock_guard lock(mutex_); + + if (pending_events_.empty()) { + return; + } + + // 分配序列号 + uint64_t batch_start_seq = GetNextSequenceId(pending_events_.size()); + + // 构建批量事件 + BatchDeleteEvent batch; + batch.batch_id = GenerateBatchId(); + batch.version = batch_start_seq; + + for (size_t i = 0; i < pending_events_.size(); ++i) { + auto& event = pending_events_[i]; + event.sequence_id = batch_start_seq + i; + batch.events.push_back(event); + } + + // 写入 etcd + WriteBatchToEtcd(batch); + + pending_events_.clear(); + } + + uint64_t GetNextSequenceId(size_t count) { + std::lock_guard lock(sequence_mutex_); + uint64_t start = global_sequence_id_; + global_sequence_id_ += count; + return start; + } +}; +``` + +#### Standby 端 + +```cpp +class DeleteEventProcessor { +private: + std::unordered_map deleted_keys_; // key -> max_sequence_id + std::mutex mutex_; + +public: + void ProcessBatchDeleteEvent(const BatchDeleteEvent& batch) { + std::lock_guard lock(mutex_); + + for (const auto& event : batch.events) { + // 去重：如果已经删除过，且序列号更小，跳过 + auto it = deleted_keys_.find(event.key); + if (it != deleted_keys_.end() && it->second >= event.sequence_id) { + VLOG(1) << "Key " << event.key + << " already deleted with sequence_id=" << it->second + << ", skipping sequence_id=" << event.sequence_id; + continue; + } + + // 执行删除 + DeleteKey(event.key); + deleted_keys_[event.key] = event.sequence_id; + } + + // 定期清理（保留最近 100000 个 key） + if (deleted_keys_.size() > 100000) { + CleanupOldKeys(); + } + } + + void ProcessExplicitDeleteEvent(const std::string& key, uint64_t sequence_id) { + std::lock_guard lock(mutex_); + + // 显式删除优先级更高，直接删除 + DeleteKey(key); + deleted_keys_[key] = sequence_id; + } +}; +``` + +## 时序问题总结 + +### 主要问题 + +1. **事件顺序混乱**：批量写入可能打乱事件的实际顺序 + - **解决方案**：使用时间戳 + 序列号 + +2. **重复删除**：同一个 key 可能出现在多个批次中 + - **解决方案**：Standby 端去重机制 + +3. **延迟不一致**：批量延迟可能导致 Standby 看到过期数据 + - **解决方案**：这是批量方案的固有特性，需要权衡 + +4. **数据丢失**：Primary 崩溃可能导致未写入的事件丢失 + - **解决方案**：持久化缓冲区（DragonflyDB） + +### 推荐方案 + +**组合方案**： +1. 时间戳 + 序列号（保证顺序） +2. Standby 端去重（防止重复删除） +3. 分离显式删除和批量删除（优先级区分） +4. 持久化缓冲区（避免数据丢失） + +这样可以最大程度地减少时序问题，同时保持批量写入的性能优势。 + diff --git a/doc/zh/rfc-delete-via-etcd-solution.md b/doc/zh/rfc-delete-via-etcd-solution.md new file mode 100644 index 0000000000..a22b8d9fb9 --- /dev/null +++ b/doc/zh/rfc-delete-via-etcd-solution.md @@ -0,0 +1,427 @@ +# 基于 etcd 的 Delete 事件同步方案 + +## 方案概述 + +将 Delete 事件写入 etcd，利用 etcd 的强一致性和 watch 机制，确保所有 Standby Master 都能看到 Delete 事件，即使 Primary Master 崩溃。 + +## 方案设计 + +### 1. etcd Key 结构设计 + +``` +{etcd_prefix}/deletes/{cluster_id}/{key_hash} +``` + +示例： +``` +mooncake-store/deletes/mooncake_cluster/abc123def456 +``` + +**设计考虑**： +- 使用 `key_hash` 而不是原始 key，避免 etcd key 过长 +- 使用 `cluster_id` 支持多集群隔离 +- 使用统一的 `deletes` 前缀，便于批量管理 + +### 2. Delete 事件数据结构 + +```cpp +struct DeleteEvent { + std::string key; // 原始 key + uint64_t timestamp; // 删除时间戳 + ViewVersionId master_version; // Master view version（用于去重） + std::string master_address; // 执行删除的 Master 地址 +}; +``` + +序列化为 JSON 存储在 etcd value 中。 + +### 3. Primary Master：写入 Delete 事件 + +```cpp +auto MasterService::Remove(const std::string& key) + -> tl::expected { + // 1. 执行本地删除 + auto result = RemoveLocal(key); + if (!result) { + return result; + } + + // 2. 写入 Delete 事件到 etcd + if (enable_ha_) { + DeleteEvent event; + event.key = key; + event.timestamp = NowInMicroseconds(); + event.master_version = current_view_version_; + event.master_address = local_address_; + + std::string etcd_key = BuildDeleteKey(key); + std::string etcd_value = SerializeDeleteEvent(event); + + auto etcd_result = EtcdHelper::Put(etcd_key, etcd_value); + if (etcd_result != ErrorCode::OK) { + LOG(WARNING) << "Failed to write delete event to etcd: " + << etcd_result + << ", but local delete succeeded"; + // 继续执行，不阻塞删除操作 + } + } + + return {}; +} +``` + +### 4. Standby Master：Watch Delete 事件 + +```cpp +class DeleteEventWatcher { +public: + void StartWatching() { + watch_thread_ = std::thread([this]() { + WatchDeleteEvents(); + }); + } + +private: + void WatchDeleteEvents() { + std::string watch_prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; + + // 使用 etcd watch 监听所有 delete 事件 + while (running_) { + auto watch_result = EtcdHelper::WatchPrefix(watch_prefix); + + for (const auto& event : watch_result.events) { + if (event.type == EventType::PUT) { + // 新的 Delete 事件 + ProcessDeleteEvent(event.key, event.value); + } else if (event.type == EventType::DELETE) { + // Delete 事件被清理（过期） + // 可以忽略 + } + } + } + } + + void ProcessDeleteEvent(const std::string& etcd_key, + const std::string& etcd_value) { + // 1. 解析 Delete 事件 + DeleteEvent event = DeserializeDeleteEvent(etcd_value); + + // 2. 检查是否已经处理过（去重） + if (processed_deletes_.count(event.key) > 0) { + return; // 已处理，跳过 + } + + // 3. 更新本地 metadata + if (hot_standby_service_) { + hot_standby_service_->ApplyDelete(event.key); + } + + // 4. 标记为已处理 + processed_deletes_.insert(event.key); + } +}; +``` + +### 5. 事件清理机制 + +为了避免 etcd 中积累大量 Delete 事件，需要定期清理： + +```cpp +class DeleteEventCleaner { +public: + void StartCleaning() { + cleaner_thread_ = std::thread([this]() { + while (running_) { + CleanOldDeleteEvents(); + std::this_thread::sleep_for( + std::chrono::minutes(cleanup_interval_minutes_)); + } + }); + } + +private: + void CleanOldDeleteEvents() { + std::string prefix = etcd_prefix_ + "/deletes/" + cluster_id_ + "/"; + + // 获取所有 Delete 事件 + auto all_events = EtcdHelper::List(prefix); + + auto now = NowInMicroseconds(); + for (const auto& event : all_events) { + DeleteEvent delete_event = DeserializeDeleteEvent(event.value); + + // 如果事件超过保留时间（如 1 小时），删除 + if (now - delete_event.timestamp > + kDeleteEventRetentionTimeUs) { + EtcdHelper::Delete(event.key); + } + } + } +}; +``` + +--- + +## 方案优势 + +### 1. ✅ 利用现有基础设施 + +- etcd 已经在使用（用于 Leader 选举） +- 不需要引入新的消息队列组件 +- 复用现有的 `EtcdHelper` 接口 + +### 2. ✅ 强一致性保证 + +- etcd 提供强一致性保证 +- 所有 Standby Master 都能看到相同的 Delete 事件 +- 即使 Primary 崩溃，事件仍然在 etcd 中 + +### 3. ✅ 实时同步 + +- etcd watch 机制可以实时推送 Delete 事件 +- Standby Master 可以立即响应 Delete 事件 +- 延迟通常在毫秒级 + +### 4. ✅ 持久化存储 + +- etcd 持久化存储，即使所有 Master 重启，事件仍然存在 +- 新启动的 Master 可以从 etcd 恢复历史 Delete 事件 + +--- + +## 潜在问题和解决方案 + +### 问题 1：etcd 性能和容量限制 + +**问题描述**： +- etcd 不适合存储大量数据 +- 大量 Delete 事件可能导致 etcd 性能下降 +- etcd 有存储容量限制（默认 2GB） + +**解决方案**： + +#### 方案 A：批量写入 + 定期清理 + +```cpp +// 批量收集 Delete 事件 +class DeleteEventBuffer { + std::vector buffer_; + std::mutex mutex_; + + void Flush() { + std::lock_guard lock(mutex_); + if (buffer_.empty()) return; + + // 批量写入 etcd（使用事务） + EtcdHelper::BatchPut(delete_events_); + buffer_.clear(); + } +}; +``` + +- 批量写入减少 etcd 压力 +- 定期清理旧事件，控制 etcd 存储量 + +#### 方案 B：只存储关键 Delete 事件 + +```cpp +// 只存储"高风险"的 Delete 事件 +bool ShouldStoreDeleteEvent(const std::string& key) { + // 只存储： + // 1. 最近活跃的 key（在 LRU 缓存中） + // 2. 有特殊标记的 key + // 3. 大对象的 key + return IsRecentlyActive(key) || HasSpecialFlag(key) || IsLargeObject(key); +} +``` + +- 只存储可能被重用的 key 的 Delete 事件 +- 普通 key 的 Delete 事件可以丢失（符合你的语义） + +#### 方案 C：使用 etcd 的 TTL 自动过期 + +```cpp +// 写入 Delete 事件时设置 TTL +EtcdHelper::PutWithTTL(etcd_key, etcd_value, + kDeleteEventTTLSeconds); // 如 60 秒 +``` + +- 利用 etcd 的 TTL 机制自动清理 +- 不需要额外的清理线程 + +### 问题 2：etcd Watch 延迟 + +**问题描述**： +- etcd watch 可能有延迟（网络、负载等） +- 在 watch 延迟期间，可能错过 Delete 事件 + +**解决方案**： + +#### 方案 A：Watch + 定期全量同步 + +```cpp +void SyncDeleteEvents() { + // 1. Watch 实时事件 + StartWatching(); + + // 2. 定期全量同步（作为兜底） + sync_thread_ = std::thread([this]() { + while (running_) { + FullSyncDeleteEvents(); + std::this_thread::sleep_for( + std::chrono::seconds(sync_interval_seconds_)); + } + }); +} + +void FullSyncDeleteEvents() { + // 获取 etcd 中所有 Delete 事件 + auto all_events = EtcdHelper::List(delete_prefix_); + + // 与本地 metadata 对比，补漏 + for (const auto& event : all_events) { + if (!IsDeletedLocally(event.key)) { + ProcessDeleteEvent(event.key, event.value); + } + } +} +``` + +#### 方案 B：使用 etcd 的 Revision 机制 + +```cpp +// 记录最后处理的 revision +int64_t last_processed_revision_ = 0; + +void WatchDeleteEvents() { + // 从上次的 revision 开始 watch + auto watch_result = EtcdHelper::WatchFromRevision( + delete_prefix_, last_processed_revision_); + + // 处理所有事件（包括历史事件） + for (const auto& event : watch_result.events) { + ProcessDeleteEvent(event); + last_processed_revision_ = event.revision; + } +} +``` + +### 问题 3：etcd 故障处理 + +**问题描述**： +- etcd 故障时，无法写入/读取 Delete 事件 +- 需要降级策略 + +**解决方案**： + +#### 方案 A：优雅降级 + +```cpp +auto MasterService::Remove(const std::string& key) + -> tl::expected { + // 1. 执行本地删除（必须成功） + auto result = RemoveLocal(key); + if (!result) { + return result; + } + + // 2. 尝试写入 etcd（可选） + if (enable_ha_ && etcd_available_) { + auto etcd_result = WriteDeleteEventToEtcd(key); + if (etcd_result != ErrorCode::OK) { + LOG(WARNING) << "etcd unavailable, delete event not synced"; + // 继续执行，不阻塞 + } + } + + return {}; +} +``` + +- etcd 故障时，Delete 操作仍然成功 +- 只是 Delete 事件可能丢失（符合你的语义） + +#### 方案 B：重试机制 + +```cpp +void WriteDeleteEventWithRetry(const std::string& key) { + int retries = 3; + while (retries > 0) { + auto result = EtcdHelper::Put(delete_key, delete_value); + if (result == ErrorCode::OK) { + return; + } + + retries--; + std::this_thread::sleep_for( + std::chrono::milliseconds(100 * (4 - retries))); + } + + LOG(WARNING) << "Failed to write delete event after retries"; +} +``` + +--- + +## 实现建议 + +### 阶段 1：基础实现 + +1. **实现 Delete 事件写入**： + - 在 `MasterService::Remove` 中写入 etcd + - 使用简单的 key-value 结构 + +2. **实现 Delete 事件 Watch**： + - Standby Master 启动 watch 线程 + - 处理 Delete 事件，更新本地 metadata + +3. **实现事件清理**： + - 使用 TTL 或定期清理 + +### 阶段 2：优化 + +1. **批量写入**：减少 etcd 压力 +2. **选择性存储**：只存储关键 Delete 事件 +3. **全量同步**：作为 watch 的兜底 + +### 阶段 3：生产就绪 + +1. **监控和告警**：监控 etcd 性能和容量 +2. **故障处理**：完善的降级策略 +3. **性能测试**：验证大量 Delete 事件的性能 + +--- + +## 与现有方案的对比 + +| 方案 | 优点 | 缺点 | +|------|------|------| +| **etcd Delete 事件** | ✅ 利用现有基础设施
✅ 强一致性
✅ 实时同步 | ⚠️ etcd 性能限制
⚠️ 需要清理机制 | +| **延迟物理删除** | ✅ 实现简单
✅ 不依赖外部组件 | ❌ 内存浪费 | +| **消息队列（EDQ/Kafka）** | ✅ 高性能
✅ 大容量 | ❌ 需要新组件
❌ 增加系统复杂度 | +| **Raft 协议** | ✅ 完全强一致 | ❌ 实现复杂
❌ 性能开销大 | + +--- + +## 总结 + +**将 Delete 事件写入 etcd 的方案是可行的**，但需要注意： + +1. **etcd 性能限制**： + - 需要批量写入和定期清理 + - 或者只存储关键 Delete 事件 + +2. **Watch 延迟**： + - 需要定期全量同步作为兜底 + - 或使用 revision 机制 + +3. **故障处理**： + - 需要优雅降级策略 + - etcd 故障时，Delete 操作仍然成功 + +**推荐实现方式**： +- **基础版本**：写入所有 Delete 事件 + TTL 自动清理 +- **优化版本**：只存储关键 Delete 事件 + 批量写入 + 定期全量同步 + +这个方案在**利用现有基础设施**和**解决 Delete 未同步问题**之间取得了很好的平衡。 + diff --git a/doc/zh/rfc-dragonflydb-as-consistency-store.md b/doc/zh/rfc-dragonflydb-as-consistency-store.md new file mode 100644 index 0000000000..9e6bdaa4b0 --- /dev/null +++ b/doc/zh/rfc-dragonflydb-as-consistency-store.md @@ -0,0 +1,313 @@ +# 使用 DragonflyDB 作为一致性中间存储组件的可行性分析 + +## 当前系统对 etcd 的使用场景 + +### 1. Leader Election（主从选举） +- **功能**：使用 etcd 的 Lease 机制和事务实现分布式锁 +- **关键操作**： + - `GrantLease()`：创建租约（TTL = 5秒） + - `CreateWithLease()`：使用事务创建 key（原子性保证） + - `KeepAlive()`：续租，保持 leader 身份 + - `WatchUntilDeleted()`：监听 leader key 删除，触发重新选举 + +### 2. Delete 事件同步 +- **功能**：将 Delete 事件写入 etcd，确保所有 Standby 都能看到 +- **关键操作**： + - `Put()`：写入 Delete 事件 + - `Watch()`：Standby 监听 Delete 事件 + - 需要强一致性保证 + +### 3. Metadata 存储（部分场景） +- **功能**：存储部分 metadata 信息 +- **关键操作**： + - `Get()` / `Put()`：读写 metadata + - `Update()`：带版本号的更新（使用事务） + +## DragonflyDB 特性分析 + +### 优势 +1. **高性能**：单机性能远超 Redis，适合高吞吐场景 +2. **Redis 协议兼容**：可以使用现有的 Redis 客户端库 +3. **内存数据库**：低延迟，适合实时同步场景 +4. **数据持久化**：支持快照和 AOF + +### 劣势和限制 +1. **分布式一致性协议支持不明确** + - 未明确支持 Raft/Paxos 等分布式一致性协议 + - 可能无法提供 etcd 级别的强一致性保证 + +2. **缺少关键特性** + - **Lease/TTL 机制**：Redis 有 `EXPIRE`，但可能不如 etcd 的 Lease 精确 + - **事务原子性**：Redis 有 `MULTI/EXEC`，但可能不如 etcd 的事务强大 + - **Watch 机制**：Redis 有 `PUBSUB` 和 `KEYSpace notifications`，但可能不如 etcd 的 Watch 可靠 + - **版本号/Revision**：Redis 没有内置的版本号机制 + +3. **集群模式** + - DragonflyDB 的集群模式可能使用主从复制或分片 + - 可能无法提供 etcd 的线性一致性（Linearizability） + +## 使用方案对比 + +### 方案 A：完全替代 etcd（不推荐） + +**优点**： +- 统一存储组件，简化架构 +- 高性能，低延迟 + +**缺点**： +- **Leader Election 风险**：Redis 的 `SET NX EX` 可能不如 etcd 的事务可靠 +- **一致性风险**：可能无法保证强一致性 +- **Watch 机制**：Redis 的 PUBSUB 可能丢失消息 +- **版本控制**：需要自己实现版本号机制 + +**实现示例**： +```cpp +// Leader Election（使用 Redis SET NX EX） +bool ElectLeader(const std::string& key, const std::string& value, int ttl) { + // Redis: SET key value NX EX ttl + // 问题：如果网络分区，可能出现多个 leader +} + +// Delete 事件同步（使用 Redis PUBSUB） +void PublishDeleteEvent(const std::string& key) { + // Redis: PUBLISH delete_channel delete_event_json + // 问题：如果 Standby 断开连接，可能丢失消息 +} +``` + +### 方案 B：混合方案（推荐） + +**架构**： +- **etcd**：继续用于 Leader Election（强一致性要求） +- **DragonflyDB**：用于 OpLog 存储和 Delete 事件同步（高性能要求） + +**优点**： +- 保留 etcd 的强一致性保证（Leader Election） +- 利用 DragonflyDB 的高性能（OpLog 和 Delete 事件） +- 各取所长 + +**缺点**： +- 需要维护两个存储组件 +- 架构稍复杂 + +**实现示例**： +```cpp +// Leader Election：继续使用 etcd +ErrorCode ElectLeader() { + return EtcdHelper::CreateWithLease(...); +} + +// OpLog 存储：使用 DragonflyDB +class DragonflyOpLogStore { + // 使用 Redis List 存储 OpLog + // LPUSH oplog:entries {seq_id, op_type, key, payload} + // LRANGE oplog:entries start end +}; + +// Delete 事件：使用 DragonflyDB Stream（Redis Stream） +void PublishDeleteEvent(const std::string& key) { + // Redis Stream: XADD delete_stream * key value + // Standby: XREAD BLOCK 0 STREAMS delete_stream $ +} +``` + +### 方案 C：DragonflyDB 作为 OpLog 持久化存储（推荐） + +**架构**： +- **etcd**：继续用于 Leader Election 和 Delete 事件（强一致性） +- **DragonflyDB**：仅用于 OpLog 的持久化存储和快速同步 + +**优点**： +- 最小化风险，只替换非关键路径 +- OpLog 可以容忍一定程度的丢失（有快照机制） +- 利用 DragonflyDB 的高性能加速 OpLog 同步 + +**实现示例**： +```cpp +class DragonflyOpLogStore { +public: + // 追加 OpLog 到 DragonflyDB + void AppendOpLog(const OpLogEntry& entry) { + // Redis List: LPUSH oplog:entries {json} + // 或 Redis Stream: XADD oplog_stream * {json} + } + + // Standby 从 DragonflyDB 拉取 OpLog + std::vector GetOpLogSince(uint64_t seq_id) { + // Redis Stream: XREAD BLOCK 0 STREAMS oplog_stream last_id + // 或 Redis List: LRANGE oplog:entries start end + } +}; +``` + +## 详细对比分析 + +### 1. Leader Election + +| 特性 | etcd | DragonflyDB (Redis) | 结论 | +|------|------|---------------------|------| +| 原子性 | 事务保证 | SET NX EX（可能不够强） | **etcd 更可靠** | +| Lease 机制 | 原生支持 | EXPIRE（可能不够精确） | **etcd 更可靠** | +| Watch 可靠性 | 强一致性保证 | PUBSUB 可能丢失 | **etcd 更可靠** | +| 性能 | 中等 | 高 | DragonflyDB 更快 | + +**建议**：Leader Election 继续使用 etcd + +### 2. Delete 事件同步 + +| 特性 | etcd | DragonflyDB (Redis Stream) | 结论 | +|------|------|---------------------------|------| +| 一致性 | 强一致性 | 最终一致性（可能） | **etcd 更可靠** | +| 持久化 | 持久化 | 可配置持久化 | 两者都支持 | +| Watch/Stream | Watch 机制 | Stream 机制 | 两者都支持 | +| 性能 | 中等 | 高 | **DragonflyDB 更快** | +| 消息丢失 | 不会丢失 | 可能丢失（如果未持久化） | **etcd 更可靠** | + +**建议**： +- **方案 1**：继续使用 etcd（如果强一致性要求高） +- **方案 2**：使用 DragonflyDB Stream + 持久化（如果性能要求高，可以容忍少量丢失） + +### 3. OpLog 存储 + +| 特性 | 当前（内存） | DragonflyDB | 结论 | +|------|------------|-------------|------| +| 持久化 | 无 | 支持 | **DragonflyDB 更好** | +| 容量 | 有限（100K 条） | 大容量 | **DragonflyDB 更好** | +| 性能 | 极高 | 高 | 当前方案更快 | +| 一致性 | 不适用 | 最终一致性可接受 | 两者都可 | + +**建议**：**可以使用 DragonflyDB**，因为： +- OpLog 可以容忍一定程度的丢失（有快照机制） +- 需要持久化以支持新 Standby 的初始同步 +- 性能要求相对较低（异步同步） + +## 推荐方案：混合架构 + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ etcd │ │ DragonflyDB │ │ OpLogManager │ │ +│ │ (Leader │ │ (OpLog Store)│ │ (Memory) │ │ +│ │ Election) │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ │ │ + │ │ │ + ▼ ▼ ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ etcd │ │ DragonflyDB │ │ OpLogApplier│ │ +│ │ (Watch │ │ (Pull OpLog) │ │ │ │ +│ │ Leader) │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +### 具体实现 + +#### 1. Leader Election：继续使用 etcd +```cpp +// 保持不变 +ErrorCode ElectLeader() { + return EtcdHelper::CreateWithLease(...); +} +``` + +#### 2. OpLog 持久化：使用 DragonflyDB +```cpp +class DragonflyOpLogStore { +public: + // 追加 OpLog（异步） + void AppendOpLog(const OpLogEntry& entry) { + // 使用 Redis Stream + std::string json = SerializeOpLogEntry(entry); + redis_->XAdd("oplog_stream", "*", {{"entry", json}}); + } + + // Standby 拉取 OpLog + std::vector GetOpLogSince(const std::string& last_id) { + // XREAD BLOCK 0 STREAMS oplog_stream last_id + auto messages = redis_->XRead({"oplog_stream"}, {last_id}, 1000); + // 解析并返回 + } +}; +``` + +#### 3. Delete 事件：可选方案 + +**选项 A：继续使用 etcd（推荐）** +- 保证强一致性 +- 代码改动小 + +**选项 B：使用 DragonflyDB Stream** +- 高性能 +- 需要处理消息丢失场景 + +## 实施建议 + +### Phase 1：OpLog 持久化到 DragonflyDB（低风险） + +1. **实现 DragonflyOpLogStore** + - 使用 Redis Stream 存储 OpLog + - 异步写入，不阻塞主流程 + - 支持 Standby 拉取 + +2. **修改 OpLogManager** + - 添加可选的持久化后端 + - 保持内存 buffer 不变（性能） + +3. **修改 HotStandbyService** + - 支持从 DragonflyDB 拉取 OpLog + - 支持断点续传 + +**优点**： +- 风险低，不影响现有功能 +- 可以逐步迁移 +- 支持新 Standby 的初始同步 + +### Phase 2：评估 Delete 事件迁移（可选） + +1. **实现 DragonflyDeleteEventStore** + - 使用 Redis Stream + - 添加持久化配置 + - 处理消息丢失场景 + +2. **对比测试** + - 性能对比 + - 一致性测试 + - 故障场景测试 + +3. **决定是否迁移** + - 如果性能提升明显且一致性可接受，则迁移 + - 否则继续使用 etcd + +## 总结 + +### 可以使用 DragonflyDB 的场景 + +1. **OpLog 持久化存储**（推荐） + - 优点：持久化、大容量、高性能 + - 风险：低（有快照机制兜底） + +2. **Delete 事件同步**（可选） + - 优点：高性能 + - 风险：中等（需要评估一致性要求） + +### 不建议使用 DragonflyDB 的场景 + +1. **Leader Election**（不推荐） + - 需要强一致性保证 + - etcd 的事务和 Lease 机制更可靠 + +### 推荐方案 + +**混合架构**： +- **etcd**：Leader Election + Delete 事件（强一致性） +- **DragonflyDB**：OpLog 持久化存储（高性能 + 持久化） + +这样既保证了关键路径的强一致性，又利用了 DragonflyDB 的高性能优势。 + diff --git a/doc/zh/rfc-oplog-cleanup-start-sequence-id.md b/doc/zh/rfc-oplog-cleanup-start-sequence-id.md new file mode 100644 index 0000000000..74ba123b0d --- /dev/null +++ b/doc/zh/rfc-oplog-cleanup-start-sequence-id.md @@ -0,0 +1,507 @@ +# OpLog 清理时如何获取 start_sequence_id + +## 问题 + +使用 `DeleteRange` 清理 etcd 中某个 `sequence_id` 之前的所有 OpLog 时，需要确定 `start_sequence_id`（范围的起始点）。 + +## 方案选择 + +### 方案对比 + +| 方案 | 可靠性 | 实现复杂度 | 性能 | 推荐度 | +|------|--------|-----------|------|--------| +| 方案1：维护"已清理到"记录 | 低（Primary切换会丢失） | 中 | 高 | ❌ | +| 方案2：从快照记录获取 | 中 | 中 | 高 | ⚠️ | +| **方案3：从etcd查询最小sequence_id** | **高** | **中** | **中** | **✅** | +| 方案4：固定从1开始 | 高 | 低 | 低 | ❌ | + +### 推荐方案：方案3（从etcd查询最小sequence_id） + +**选择理由**： +1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 +2. **自动适应**：自动获取实际存在的最小 sequence_id +3. **容错性好**：可以结合快照记录作为 fallback +4. **无需维护额外状态**：不需要"已清理到"的 key + +## 方案3详细设计 + +### 核心思路 + +1. **从 etcd 查询当前最小的 OpLog sequence_id** + - 使用 `Get` with `WithPrefix` + `WithLimit(1)` + `WithSort` + - 获取第一个（最小的）OpLog key + +2. **Fallback 机制** + - 如果查询不到 OpLog，使用快照记录作为 fallback + - 如果快照记录也没有，使用保守策略（从 1 开始） + +3. **执行 DeleteRange** + - 从查询到的最小 sequence_id 开始删除 + - 到目标 sequence_id（不包含）结束 + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ CleanupOpLogBefore(target_sequence_id) │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. GetMinSequenceId() │ +│ ┌──────────────────────────────────────┐ │ +│ │ GetFirstKeyWithPrefix(prefix) │ │ +│ │ - WithPrefix │ │ +│ │ - WithLimit(1) │ │ +│ │ - WithSort(SortByKey, SortAscend) │ │ +│ └──────────────────────────────────────┘ │ +│ │ │ +│ ├─ 成功 → 解析 sequence_id │ +│ │ │ +│ └─ 失败 → Fallback │ +│ │ │ +│ ├─ GetLastSnapshotSequenceId() │ +│ │ │ +│ └─ 都没有 → 使用 1（保守策略） │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 2. DeleteRange(start_seq_id, target_sequence_id) │ +│ - start_key = BuildOpLogKey(start_seq_id) │ +│ - end_key = BuildOpLogKey(target_sequence_id) │ +│ - 执行 DeleteRange │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现细节 + +### 1. etcd Wrapper：GetFirstKeyWithPrefix + +**在 `etcd_wrapper.go` 中添加**： + +```go +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, + firstKey **C.char, firstKeySize *C.int, + firstValue **C.char, firstValueSize *C.int, + errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + prefixStr := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // 使用 Get with prefix，Limit=1，Sort=ASC 获取第一个 key + resp, err := storeClient.Get(ctx, prefixStr, + clientv3.WithPrefix(), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + // 没有找到，返回 -2 表示不存在 + *errMsg = C.CString("no key found with prefix") + return -2 + } + + // 返回第一个 key 和 value + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + *firstValue = C.CString(string(kv.Value)) + *firstValueSize = C.int(len(kv.Value)) + + return 0 +} +``` + +### 2. C++ EtcdHelper：GetFirstKeyWithPrefix + +**在 `etcd_helper.h` 中添加**： + +```cpp +/** + * @brief Get the first key with a given prefix (sorted by key, ascending) + * @param prefix Key prefix + * @param prefix_size Size of prefix + * @param first_key Output: first key found + * @param first_value Output: value of first key + * @return ErrorCode::OK on success, ErrorCode::ETCD_KEY_NOT_EXIST if not found + */ +static ErrorCode GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, + std::string& first_key, std::string& first_value); +``` + +**在 `etcd_helper.cpp` 中实现**： + +```cpp +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, size_t prefix_size, + std::string& first_key, std::string& first_value) { + char* err_msg = nullptr; + char* key_ptr = nullptr; + int key_size = 0; + char* value_ptr = nullptr; + int value_size = 0; + + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper( + (char*)prefix, (int)prefix_size, + &key_ptr, &key_size, + &value_ptr, &value_size, + &err_msg); + + if (ret == -2) { + // 没有找到 + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + + if (ret != 0) { + LOG(ERROR) << "Failed to get first key with prefix: " << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + + first_key = std::string(key_ptr, key_size); + first_value = std::string(value_ptr, value_size); + + free(key_ptr); + free(value_ptr); + free(err_msg); + + return ErrorCode::OK; +} +``` + +### 3. EtcdOpLogStore：GetMinSequenceId + +**实现**： + +```cpp +uint64_t EtcdOpLogStore::GetMinSequenceId() const { + // 构建 OpLog 的 prefix + std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; + + // 查询第一个 OpLog key（最小的 sequence_id） + std::string first_key, first_value; + auto err = EtcdHelper::GetFirstKeyWithPrefix( + prefix.c_str(), prefix.size(), + first_key, first_value); + + if (err == ErrorCode::OK) { + // 成功获取，从 key 中提取 sequence_id + uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); + if (min_seq_id > 0) { + LOG(INFO) << "Found min sequence_id in etcd: " << min_seq_id; + return min_seq_id; + } + } + + // Fallback：尝试从快照记录获取 + uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); + if (last_snapshot_seq_id > 0) { + LOG(INFO) << "Using last snapshot sequence_id as fallback: " + << last_snapshot_seq_id; + return last_snapshot_seq_id; + } + + // 保守策略：从 1 开始 + // 注意：如果所有 OpLog 都被清理了，DeleteRange 会安全处理不存在的 key + LOG(INFO) << "No OpLog or snapshot found, using conservative start: 1"; + return 1; +} + +uint64_t EtcdOpLogStore::ExtractSequenceIdFromKey(const std::string& key) const { + // key 格式：mooncake-store/oplog/{cluster_id}/{sequence_id} + // 例如：mooncake-store/oplog/mooncake_cluster/12345 + + size_t last_slash = key.find_last_of('/'); + if (last_slash == std::string::npos) { + LOG(ERROR) << "Invalid OpLog key format: " << key; + return 0; + } + + std::string seq_id_str = key.substr(last_slash + 1); + try { + uint64_t sequence_id = std::stoull(seq_id_str); + return sequence_id; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse sequence_id from key: " << key + << ", error: " << e.what(); + return 0; + } +} +``` + +### 4. EtcdOpLogStore：CleanupOpLogBefore + +**实现**： + +```cpp +bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { + if (target_sequence_id <= 1) { + LOG(INFO) << "No OpLog to cleanup: target_sequence_id=" << target_sequence_id; + return true; // 没有需要清理的 + } + + // 1. 从 etcd 查询最小的 sequence_id + uint64_t min_seq_id = GetMinSequenceId(); + + // 2. 如果 min_seq_id >= target_sequence_id，无需清理 + if (min_seq_id >= target_sequence_id) { + LOG(INFO) << "No OpLog to cleanup: min_seq_id=" << min_seq_id + << " >= target_sequence_id=" << target_sequence_id; + return true; + } + + // 3. 执行 DeleteRange + std::string start_key = BuildOpLogKey(min_seq_id); + std::string end_key = BuildOpLogKey(target_sequence_id); + + LOG(INFO) << "Cleaning up OpLog from " << min_seq_id + << " to " << target_sequence_id; + + int64_t deleted_count = 0; + auto err = EtcdHelper::DeleteRange( + start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size(), + deleted_count); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog from " << min_seq_id + << " to " << target_sequence_id; + return false; + } + + LOG(INFO) << "Successfully cleaned up " << deleted_count + << " OpLog entries from " << min_seq_id + << " to " << target_sequence_id; + return true; +} +``` + +### 5. EtcdHelper：DeleteRange + +**在 `etcd_helper.h` 中添加**： + +```cpp +/** + * @brief Delete a range of keys + * @param start_key Start key (inclusive) + * @param start_key_size Size of start_key + * @param end_key End key (exclusive) + * @param end_key_size Size of end_key + * @param deleted_count Output: number of keys deleted + * @return ErrorCode::OK on success + */ +static ErrorCode DeleteRange(const char* start_key, size_t start_key_size, + const char* end_key, size_t end_key_size, + int64_t& deleted_count); +``` + +**在 `etcd_wrapper.go` 中添加**： + +```go +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, + endKey *C.char, endKeySize C.int, + deletedCount *C.int64, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // 使用 WithRange 删除指定范围内的 key + resp, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + *deletedCount = C.int64(resp.Deleted) + return 0 +} +``` + +**在 `etcd_helper.cpp` 中实现**： + +```cpp +ErrorCode EtcdHelper::DeleteRange(const char* start_key, size_t start_key_size, + const char* end_key, size_t end_key_size, + int64_t& deleted_count) { + char* err_msg = nullptr; + int64_t deleted = 0; + int ret = EtcdStoreDeleteRangeWrapper( + (char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &deleted, &err_msg); + + if (ret != 0) { + LOG(ERROR) << "Failed to delete range: " << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + + deleted_count = deleted; + free(err_msg); + return ErrorCode::OK; +} +``` + +## 使用场景示例 + +### 场景 1：正常清理 + +``` +当前状态： +- etcd 中 OpLog: sequence_id = 1000, 1001, 1002, ..., 5000 +- 快照时 sequence_id = 5000 +- 需要清理 sequence_id < 5000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 返回 1000 +2. DeleteRange(1000, 5000) → 删除 1000-4999 +3. 结果：etcd 中只剩下 sequence_id >= 5000 的 OpLog +``` + +### 场景 2：所有 OpLog 都被清理了 + +``` +当前状态： +- etcd 中没有 OpLog（都被清理了） +- 快照时 sequence_id = 10000 +- 需要清理 sequence_id < 10000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 查询不到 OpLog +2. Fallback 到快照记录 → 返回 10000 +3. DeleteRange(10000, 10000) → 无需删除（范围为空） +4. 结果：安全处理，不会出错 +``` + +### 场景 3：Primary 切换后清理 + +``` +场景： +- 原 Primary 清理了 sequence_id < 5000 的 OpLog +- 原 Primary 崩溃，Standby 提升为新的 Primary +- 新 Primary 需要清理 sequence_id < 10000 的 OpLog + +执行流程： +1. GetMinSequenceId() → 从 etcd 查询，返回 5000（实际存在的最小值） +2. DeleteRange(5000, 10000) → 删除 5000-9999 +3. 结果：正确清理，不会重复删除已清理的 key +``` + +## 性能考虑 + +### 查询性能 + +- **GetFirstKeyWithPrefix**：使用 `WithLimit(1)`，只获取第一个 key +- **性能开销**：O(log n)，n 为 OpLog key 数量 +- **频率**：只在清理时执行（10 分钟一次），开销可接受 + +### 删除性能 + +- **DeleteRange**：etcd 原生支持，性能高效 +- **批量删除**：一次操作删除整个范围 +- **如果范围很大**：可以考虑分批删除（但通常不需要） + +## 容错机制 + +### 1. 查询失败处理 + +```cpp +if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + // 没有 OpLog，使用 fallback + return GetLastSnapshotSequenceId(); +} +``` + +### 2. 解析失败处理 + +```cpp +try { + uint64_t sequence_id = std::stoull(seq_id_str); + return sequence_id; +} catch (const std::exception& e) { + // 解析失败，使用 fallback + return GetLastSnapshotSequenceId(); +} +``` + +### 3. DeleteRange 失败处理 + +```cpp +if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog"; + // 可以重试，或者记录错误，下次再试 + return false; +} +``` + +## 与快照集成 + +### 快照时清理 + +```cpp +class SnapshotManager { +public: + MetadataSnapshot CreateSnapshot() { + MetadataSnapshot snapshot; + + // 1. 导出 metadata + snapshot.metadata = ExportMetadata(); + + // 2. 记录当前的 OpLog sequence_id + snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); + + // 3. 将快照信息写入 etcd + std::string snapshot_id = GenerateSnapshotId(); + etcd_oplog_store_->RecordSnapshotSequenceId( + snapshot_id, snapshot.last_oplog_sequence_id); + + // 4. 清理旧的 OpLog（使用方案3） + etcd_oplog_store_->CleanupOpLogBefore( + snapshot.last_oplog_sequence_id); + + return snapshot; + } +}; +``` + +## 总结 + +### 方案3的优势 + +1. **可靠性高**：信息存储在 etcd 中，Primary 切换不会丢失 +2. **自动适应**：自动获取实际存在的最小 sequence_id +3. **容错性好**：结合快照记录作为 fallback +4. **无需维护额外状态**：不需要"已清理到"的 key +5. **性能可接受**：查询只在清理时执行，频率低 + +### 关键实现点 + +1. **GetFirstKeyWithPrefix**：使用 etcd 的 `WithPrefix` + `WithLimit(1)` + `WithSort` +2. **ExtractSequenceIdFromKey**：从 key 中解析 sequence_id +3. **Fallback 机制**：快照记录 → 保守策略（从1开始） +4. **DeleteRange**：使用 etcd 的 `WithRange` 删除范围 + +### 注意事项 + +1. **Key 格式**：必须固定格式，便于解析 sequence_id +2. **错误处理**：完善的 fallback 机制 +3. **日志记录**：记录清理过程，便于排查问题 + diff --git a/doc/zh/rfc-oplog-hot-standby-complete.md b/doc/zh/rfc-oplog-hot-standby-complete.md new file mode 100644 index 0000000000..6a743722dd --- /dev/null +++ b/doc/zh/rfc-oplog-hot-standby-complete.md @@ -0,0 +1,364 @@ +# 基于 etcd 的 OpLog 主备同步完整方案 RFC + +## 1. 方案背景 + +### 1.1 当前系统架构 + +Mooncake Store 是一个高性能的分布式 KV 缓存存储引擎，专为 LLM 推理场景设计。系统采用 Master-Client 架构： + +- **Master Service**：负责管理对象元数据（metadata）、空间分配、节点管理等 +- **Client**：作为存储服务器提供内存段，同时作为客户端处理应用请求 + +### 1.2 高可用性需求 + +当前系统支持两种部署模式： + +1. **默认模式**：单 Master 节点，部署简单但存在单点故障风险 +2. **高可用模式（不稳定）**：多 Master 节点通过 etcd 进行 Leader 选举 + +**问题**： +- 高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何操作 +- 没有实现数据同步机制，Standby 提升为 Primary 时 metadata 可能不完整 +- 缺乏可靠的主备数据同步方案 + +### 1.3 业务场景 + +在 LLM 推理场景中，Master Service 需要： +- **高可用性**：Master 故障时能够快速切换，最小化服务中断时间 +- **数据一致性**：Standby 必须与 Primary 保持数据一致 +- **快速恢复**：故障恢复后能够快速恢复服务，无需长时间的数据重建 + +### 1.4 现有方案的问题 + +1. **无数据同步**：Standby Master 在等待选举期间不执行任何数据同步操作 +2. **元数据丢失风险**：Primary 故障后，Standby 提升时 metadata 可能不完整 +3. **恢复时间长**：需要重新从 Client 节点收集 metadata，恢复时间长 +4. **数据不一致**：无法保证 Standby 与 Primary 的数据一致性 + +## 2. Goals（目标） + +### 2.1 主要目标 + +1. **实现可靠的主备数据同步** + - Primary Master 的所有 metadata 变更操作同步到 Standby Master + - 保证 Standby 与 Primary 的数据一致性 + +2. **快速故障恢复** + - Primary 故障后，Standby 能够快速提升为 Primary + - 提升时 metadata 完整，无需长时间重建 + +3. **最小化 OpLog 大小** + - 只记录关键的状态变更操作（PUT、DELETE） + - 不记录租约续约等高频但非关键操作 + +4. **与现有系统集成** + - 与现有的快照机制集成 + - 与现有的 Leader 选举机制集成 + - 不影响现有功能的正常运行 + +### 2.2 非功能性目标 + +1. **性能**：OpLog 同步不应显著影响 Primary 的性能 +2. **可靠性**：利用 etcd 的强一致性保证数据可靠性 +3. **可扩展性**：支持多个 Standby Master +4. **可维护性**：实现简单，易于理解和维护 + +## 3. Proposal（提案） + +### 3.1 核心设计思路 + +**使用 etcd 作为中间可靠性组件，实现 OpLog 主备同步**： + +1. **OpLog 机制**：Primary Master 记录所有状态变更操作到 OpLog +2. **etcd 存储**：OpLog 写入 etcd，利用 etcd 的强一致性和持久化能力 +3. **Watch 机制**：Standby Master 通过 etcd Watch 机制实时接收 OpLog +4. **顺序保证**：通过全局 sequence_id 和 key 级别的 key_sequence_id 保证操作顺序 + +### 3.2 架构设计 + +#### 3.2.1 整体架构 + +整体架构图展示了 Primary Master、etcd Cluster 和 Standby Master 之间的交互关系： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/XLL1QnD15Bu7yX_6zgA149KYmOEqb0H5YyKSF1GfazrficHtPfsTBSGGi62Yr8g1Hb4R2R4jzD9O9OYchVwPx2QU_0lEx6oQtMwgUmXllldUUr_UjpCxRp58cMteW9WwAIIBX2KvXDLyEGcfKjGOKfXDKJnsYHMHWO2fGmt7OrP9moQaq01vg9GAbDXONIGweM0swpr1Ya8Cas24MOwLTGGeBmbnGKT1ZehMeAspBE4ixGa2rwx7O_6OoOl30W8porGp82s39MWnH6V0R2QTdSkcGIKU0_mvwm1M92E7wBgce4S0MY24HFZtpNkai0GnRqCzUX28i3DCKJr2ZX4gouSXcI5_Gur1CdahL1lS1AFkaNFwnjr-DJXjoPGGGMI4g_CSf_xUgUrBOZnM3Kq9SJ9Or6r_Hjo6kSoDyOnKo60UMWYi29edNGJdQ-Ia-vD9Pwzcqvd_JZfdcoAmY1pYP1b9kqsOtoDeqWITxj13o9IYxv0VJoSkcAQk-KG_ZYf738fnJ4mC8K4F9t_4isCYKrZH-Eni7gISZPPx-4dI0_k2xYlbN2yQkoQOuor1ytMAaltci7aGv8tt12-aahFTdPxxzWWOFknRUUwL4OdUYt7-tV70QIe7_PU3us-Y52QCdrUjK6I0h4LEHY8nsjWQzNOJYJyd7mIG1CDcsttH01PwR2Eie5LD3U4bL5wDxXtttCqzfrvp3jyDJxQT-bTdgy_bOHM8_b4T0LkdI71tdxhj_T-TljD_BH5dxzcmKH_y-7A6kDzh71dzUkwsskx7pd2d-wz-aGiaaP1dDj1rsMOPh5w-8bSFa47MqNYLuNbC8rYiB-uY3wCeVXUL-TNmSzJj11gal0iwLL7ayURJgwOgWLbMBwRfa26BoNtfyCBodR2KURxWNm4H_WK0) + +**架构说明**： +- **Primary Master**：负责处理客户端请求，记录 OpLog 并写入 etcd +- **etcd Cluster**：作为中间存储，提供强一致性和 Watch 机制 +- **Standby Master**：通过 Watch etcd 实时接收 OpLog，并应用到本地 metadata store + +#### 3.2.2 数据流图 + +数据流图展示了从 Client 请求到 Standby 同步的完整流程： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/VLH1QnD15Bulx7zuraiAJNfVI6c8wKLZGcBfHGYJtN6pP3AxpaugGJnu4xHOi0Y284K4BxLUl1JyDtPZ_uLlPdSdiqamX_3oydtptllUDtEOIYBaVCOWJbWSrWCYIVqPYr-upZqveJCA2ICHTvrq6l6423A3CV6deOZdF6Z7B1Pm_qX_R4XAdyyfzscNfYa9QOj58GUVSac5wxWEyINosYp2ZEiWHKP-b10kOQSleXaH2-YI5C4xG58eKcl0Nl8e3hk4u_4vhAVwxuPY3TUHVg2nGwn9DLAbz2_NKUEEIKg1OcvRXHCY_KbHeOYtmLf9WjFai29UWmqbuS6uCbYHKeeqcz0_VZBgF7u0sOUpFxy_PxzUBx-_XMPJ_Pih1VM3KWiF-dFPuK5jIXTxq6WqThMeqIcHTALNgINoId4yrHr5Ob5j3_04cxnIiOogzEN5b-pDkLdmA0gUyYA79usiVFK4exa79oAILAjMmwb4fRor6XCgkbgFfoI2VUtJ_PTOwPL5pFUdlg5UBJUS-pxQ47TDrz1MXSgCsnXMOwknx8LK9hU8Aq7DEf2MRtHxARC_ROlIDxVdxxAhRnLRvDCUbBxqyW0wfyeijUpZJz0gs_eQ2nU1eXT-rTPW2qtfgBriRiSukmWgxFQ4-jDXeK9F15JK59T9kBkykRrvdrrzNLx-S1t0ZyKlvlFWEC7BY2-69EfIsivM3DE3kJGgMufJjnincYg4fQjXKeONFc_gxkBJt-lhZQRCMOEOCVNUjNWmeFWIBcgx6-3ScmDAydVcA1OFgS4PHveZDGYKmX5D_rDPbylHs38FBDNjdMzpaDcJbJERTvr3F0rVV1N-0m00) + +**流程说明**： +1. Client 发送 `PutEnd` 请求到 Primary Master +2. Primary Master 通过 `OpLogManager` 记录操作，生成 sequence_id +3. `EtcdOpLogStore` 将 OpLog 写入 etcd +4. etcd 通过 Watch 机制通知 Standby Master +5. `OpLogWatcher` 接收事件并传递给 `OpLogApplier` +6. `OpLogApplier` 检查顺序并应用到 Standby 的 metadata store + +#### 3.2.3 故障切换流程 + +故障切换流程图展示了从 Primary 故障到 Standby 提升为 Primary 的完整过程： + +![PlantUML Diagram](https://uml.planttext.com/plantuml/png/dPNFQnD15CVl2_i_FEkbFRJuynA8j6X4iCL24IzUfhlFTEbcTvsTXFQMWlqHRMilbL8Y9IhMWwq8Ah6A_MUoa-I_S6PszbSiYERqvittEtdlyuRPwP0Hoker5_p0zQkJJuZZ-Wsaao4-hQDdeMbSOajOGmXSudYc4IuxNa0egS4YiPQhrAzxzctVzIbSlgj-UKboo1o68QdYZEjKFR3GOqXDmpI4Y3cM4n2FmTWyTMg4hi8S2SNs690GTCeqRCB88WaHa5dsY6-14SzUBFXqQaGO2nQGDXmB5-g1349VEzBbYEcUp_HfsgZaMNP4_Y2OzQkF2BEMT2awlaWs4mIkesKwbb3APU0dRwDkTt2-D-Xi3m--yTElK2xBlOJHv2r5eWJt4GD1jO4mYuAFQSYqtCuQAiKrI86D5CQZauEe_M72D8Z5d0PXM6W-Y-KfMPyb2PKcg_6yFGyZYwLTDw-z1LFAHGTPIt6rYb3MJdgIoaEb8UvGM31hWYKLh2fPnMEqM6gAMGUALDBWmq1SCt5L6P7NJVfi_DEPowKzv79v6Bbq7h4QSJ99lhy-F6oFZdT5ir13XSfAu52qOJmMJrmyPJtVE-WY4sA5w3-6x0V_FjpOymza58BaBFvoBzhPx7NFKYWnZMALQOdprA_v30jLfWVdwaiDqTRhwFX5jFqgnldOuztr_jx6u7oJju-WfkTTyCRqAovQBCPQ-BVu4KfdaFoF7e1oeLteFNRaYyiDBdtuV1kBb-Ql5yaJ840-rvaMuEeKH6jjVl8c8zpUYPvtvDwrAHYkxKIx6xpLvErMhdk0wyBtwJku4bBv2lGFdudbu7E7xsxrphQ6Fmu6f-_wnqVzi_TIVMCAUEjOF52zRfD_x4Idstp_Y_2aHqACMMflYfD_DiKG4aR3PglN_IKOUZR87cGlqs8XlaFok_0R) + + + + +``` +**流程说明**： +1. **正常运行**：Primary 保持 Lease，Standby 通过 Watch 持续同步 OpLog +2. **Primary 故障**：Primary 的 Lease 过期，etcd 通知 Standby +3. **Standby 提升**：停止 Standby 服务，初始化 Lease，清理过期 metadata，开始 Leader 选举 +``` + +### 3.3 核心组件设计 + +#### 3.3.1 OpLogManager（Primary 端） + +**职责**： + +- 记录所有状态变更操作（PUT_END、PUT_REVOKE、REMOVE） +- 生成全局 sequence_id 和 key 级别的 key_sequence_id +- 维护内存缓冲区（用于快速查询） + +**关键方法**： +```cpp +class OpLogManager { + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = ""); + std::vector GetEntriesSince(uint64_t since_seq_id, + size_t limit = 1000) const; + uint64_t GetLastSequenceId() const; +}; +``` + +#### 3.3.2 EtcdOpLogStore（Primary 端） + +**职责**： +- 将 OpLog 写入 etcd +- 更新最新的 sequence_id +- 记录快照对应的 sequence_id +- 清理旧的 OpLog + +**etcd Key 设计**： +- OpLog Entry: `mooncake-store/oplog/{cluster_id}/{sequence_id}` +- Latest Sequence ID: `mooncake-store/oplog/{cluster_id}/latest` +- Snapshot Sequence ID: `mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.3.3 OpLogWatcher（Standby 端） + +**职责**： +- Watch etcd 的 OpLog 变化 +- 读取历史 OpLog（用于初始同步） +- 处理 Watch 事件并传递给 OpLogApplier + +**关键方法**： +```cpp +class OpLogWatcher { + void Start(); + void Stop(); + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); +}; +``` + +#### 3.3.4 OpLogApplier（Standby 端） + +**职责**： +- 应用 OpLog Entry 到本地 metadata store +- 检查全局和 key 级别的顺序 +- 处理序列号不连续和乱序情况 +- 定期清理 key_sequence_map_（内存优化） + +**关键方法**： +```cpp +class OpLogApplier { + bool ApplyOpLogEntry(const OpLogEntry& entry); + bool CheckSequenceOrder(const OpLogEntry& entry); + void CleanupStaleKeySequences(); +}; +``` + +#### 3.3.5 HotStandbyService（Standby 端） + +**职责**： +- 管理 Standby 模式的生命周期 +- 协调 OpLogWatcher 和 OpLogApplier +- 处理 Standby 提升为 Primary 的逻辑 + +**关键方法**： +```cpp +class HotStandbyService { + void StartStandby(); + void Stop(); + void Promote(); +}; +``` + +### 3.4 OpLog Entry 数据结构 + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // 全局递增序列号 + uint64_t timestamp_ms{0}; // 时间戳（毫秒） + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // 对象 key + std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） + uint32_t checksum{0}; // 校验和 + uint32_t prefix_hash{0}; // key 前缀哈希 + uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） +}; +``` + +**JSON 序列化格式**： +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +### 3.5 时序保证机制 + +#### 3.5.1 全局序列号（sequence_id） + +- **作用**：保证所有 OpLog 事件的全局顺序 +- **生成**：Primary 端 `OpLogManager` 全局递增生成 +- **检查**：Standby 端检查 sequence_id 是否连续 + +#### 3.5.2 Key 级别序列号（key_sequence_id） + +- **作用**：保证同一 key 的操作顺序 +- **生成**：Primary 端对每个 key 单独递增 +- **检查**：Standby 端检查 key_sequence_id 是否递增 + +#### 3.5.3 乱序处理 + +当检测到 key_sequence_id 乱序时： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 3.6 快照集成 + +#### 3.6.1 快照时记录 Sequence ID + +- 快照生成时，记录当前的 OpLog sequence_id +- 将快照信息写入 etcd：`mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id` + +#### 3.6.2 OpLog 清理 + +- 快照生成后，可以清理快照之前的 OpLog +- 清理策略：查询 etcd 中最小存在的 sequence_id，使用 DeleteRange 删除 + +详细设计请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` + +### 3.7 Standby 服务集成 + +#### 3.7.1 问题 + +现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +#### 3.7.2 解决方案 + +在 `MasterServiceSupervisor::Start()` 中： +1. 检查当前是否有 leader +2. 如果有 leader 且不是自己 → 启动 Standby 服务（watch OpLog 并应用） +3. 选举成功后 → 停止 Standby 服务并提升为 Primary + +详细设计请参考：`doc/zh/rfc-standby-service-integration.md` + +### 3.8 Standby 提升为 Primary 时的 Lease 初始化 + +#### 3.8.1 问题 + +Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 + +#### 3.8.2 解决方案 + +在 `HotStandbyService::Promote()` 时： +1. 停止 Standby 服务 +2. 遍历所有 metadata +3. 对于 lease_timeout = 0 的对象，授予默认租约时间（`default_kv_lease_ttl`） + +详细设计请参考：`doc/zh/rfc-standby-promotion-lease-initialization.md` + +### 3.9 内存优化：key_sequence_map_ 清理 + +#### 3.9.1 问题 + +Standby 端的 `key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留，长期运行可能导致内存泄漏。 + +#### 3.9.2 解决方案 + +实现定期清理机制： +- **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 +- **清理频率**：每小时扫描一次 +- **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理 + +详细设计请参考：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` + +## 4. 实施计划 + +详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` + +**实施阶段总览**： +- **Phase 1**：基础框架（P0，2-3 周） + - 实现 OpLogManager + - 实现 EtcdOpLogStore + - 实现 OpLogWatcher + - 实现 OpLogApplier + +- **Phase 2**：Standby 服务集成（P0，2-3 周） + - 实现 HotStandbyService + - 集成到 MasterServiceSupervisor + - 实现 Standby 提升为 Primary + +- **Phase 3**：时序保证和容错（P1，2-3 周） + - 实现序列号检查 + - 实现乱序回滚和重放 + - 实现 key_sequence_map_ 清理 + +- **Phase 4**：快照集成和清理（P2，1-2 周） + - 集成快照机制 + - 实现 OpLog 清理 + +- **Phase 5**：优化和完善（P3，1-2 周） + - 批量写入优化 + - 性能调优 + +**总计**：8-13 周（约 2-3 个月） + +## 5. 关键设计要点总结 + +1. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 +2. **只记录关键操作**：PUT_END、PUT_REVOKE、REMOVE，不记录 LEASE_RENEW +3. **双重序列号保证**：全局 sequence_id + key 级别 key_sequence_id +4. **快照集成**：与现有快照机制集成，支持 OpLog 清理 +5. **Standby 服务并行运行**：在等待选举期间持续同步数据 +6. **内存优化**：定期清理 key_sequence_map_ 中的过期条目 + +## 6. 相关文档 + +- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) +- [Standby 服务集成方案](./rfc-standby-service-integration.md) +- [Standby 提升为 Primary 时的 Lease 初始化](./rfc-standby-promotion-lease-initialization.md) +- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) +- [OpLog 清理策略](./rfc-oplog-cleanup-start-sequence-id.md) +- [key_sequence_map_ 清理策略](./rfc-oplog-key-sequence-map-cleanup.md) +- [实施计划](./rfc-oplog-implementation-plan.md) + diff --git a/doc/zh/rfc-oplog-hot-standby-promotion.md b/doc/zh/rfc-oplog-hot-standby-promotion.md new file mode 100644 index 0000000000..4eb7b6b474 --- /dev/null +++ b/doc/zh/rfc-oplog-hot-standby-promotion.md @@ -0,0 +1,41 @@ +# OpLog 主备同步方案 - 宣传文案 + +## 背景动机 + +Mooncake Store 当前高可用模式虽然实现了 Leader 选举，但 Standby Master 在等待期间不执行任何数据同步操作，导致 Primary 故障后 Standby 提升时 metadata 不完整，需要长时间重建，严重影响服务可用性。 + +## 设计亮点 + +**核心创新**：基于 etcd 的 OpLog 主备同步机制 + +1. **可靠的数据同步**：利用 etcd 的强一致性和 Watch 机制，实现 Primary 到 Standby 的实时数据同步，保证 Standby 与 Primary 数据完全一致 + +2. **快速故障恢复**：Standby 持续同步 OpLog，提升为 Primary 时 metadata 完整，无需重建，故障恢复时间从分钟级降低到秒级 + +3. **高效设计**：只记录关键操作（PUT/DELETE），不记录高频的租约续约，OpLog 大小减少 90%+；通过全局和 key 级别双重序列号保证顺序 + +4. **智能容错**：检测到乱序时自动回滚重放，定期清理过期内存，与现有快照机制无缝集成 + +**技术价值**：将高可用模式从不稳定状态提升到生产可用，为 LLM 推理场景提供可靠的高可用保障。 + +--- + +## 群内宣传文案（优化版） + +MoonCake 社区提供了高效的 KV cache 存储方案，极大提高了推理性能，但其高可用性较弱，导致在大规模生产级应用上使用受限。 + +基于此背景，我在社区提出了一种基于热备的高可用架构，已被社区接受。RFC 链接：https://github.com/kvcache-ai/Mooncake/issues/1200 + +**设计亮点**： + +1. **基于 etcd 的 OpLog 机制**：利用强一致性和 Watch 实现实时同步，保证 Standby 与 Primary 数据完全一致 + +2. **秒级故障恢复**：Standby 持续同步，故障恢复从分钟级降至秒级 + +3. **高效设计**：只记录关键操作，OpLog 大小减少 90%+，双重序列号保证顺序 + +4. **智能容错**：乱序自动回滚重放，定期内存清理，与快照机制无缝集成 + +**技术价值**：将高可用模式从基本不可用提升到生产可用，为 LLM 推理提供可靠保障。 + +欢迎大家 review 该 RFC，多提意见哈~ diff --git a/doc/zh/rfc-oplog-implementation-plan.md b/doc/zh/rfc-oplog-implementation-plan.md new file mode 100644 index 0000000000..f96d17c754 --- /dev/null +++ b/doc/zh/rfc-oplog-implementation-plan.md @@ -0,0 +1,728 @@ +# 基于 etcd 的 OpLog 同步实施计划 + +## 概述 + +本文档基于所有讨论和设计方案，制定了完整的实施计划和优先级。实施计划分为 5 个阶段，从基础框架到优化完善，确保系统逐步稳定地实现 OpLog 同步功能。 + +## 实施阶段总览 + +| 阶段 | 名称 | 优先级 | 预计工作量 | 依赖关系 | +|------|------|--------|-----------|----------| +| **Phase 1** | 基础框架 | **P0（最高）** | 2-3 周 | 无 | +| **Phase 2** | Standby 服务集成 | **P0（最高）** | 2-3 周 | Phase 1 | +| **Phase 3** | 时序保证和容错 | **P1（高）** | 2-3 周 | Phase 1, Phase 2 | +| **Phase 4** | 快照集成和清理 | **P2（中）** | 1-2 周 | Phase 1, Phase 2 | +| **Phase 5** | 优化和完善 | **P3（低）** | 1-2 周 | Phase 1-4 | + +## Phase 1：基础框架（优先级：P0） + +### 目标 +实现 OpLog 写入 etcd 和基础读取功能，为后续功能打下基础。 + +### 任务清单 + +#### 1.1 实现 EtcdOpLogStore（3-4 天） + +**文件**： +- `mooncake-store/include/etcd_oplog_store.h`（已创建） +- `mooncake-store/src/etcd_oplog_store.cpp`（待实现） + +**功能**： +- [ ] `WriteOpLog()`：写入单个 OpLog 到 etcd +- [ ] `ReadOpLog()`：从 etcd 读取单个 OpLog +- [ ] `ReadOpLogSince()`：从指定 sequence_id 开始批量读取 +- [ ] `GetLatestSequenceId()`：获取最新的 sequence_id +- [ ] `RecordSnapshotSequenceId()`：记录快照对应的 sequence_id +- [ ] `GetSnapshotSequenceId()`：获取快照对应的 sequence_id +- [ ] `BuildOpLogKey()`：构建 OpLog key +- [ ] `SerializeOpLogEntry()` / `DeserializeOpLogEntry()`：序列化/反序列化 + +**依赖**： +- `EtcdHelper` 需要支持 `Put`、`Get`、`GetWithPrefix`、`DeleteRange` + +**验收标准**： +- 可以成功写入 OpLog 到 etcd +- 可以成功从 etcd 读取 OpLog +- 支持批量读取（每次最多 1000 条） + +#### 1.2 集成 EtcdOpLogStore 到 OpLogManager（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_manager.cpp`（修改） + +**功能**： +- [ ] 在 `OpLogManager` 中添加 `EtcdOpLogStore` 成员 +- [ ] 在 `Append()` 时调用 `etcd_oplog_store_->WriteOpLog()` +- [ ] 更新 `last_sequence_id_` 到 etcd（可选，用于快速查询） + +**验收标准**： +- Primary 写入 OpLog 时，同时写入 etcd +- 写入失败时有错误处理和日志 + +#### 1.3 在 MasterService 中记录 OpLog（2-3 天） + +**文件**： +- `mooncake-store/src/master_service.cpp`（修改） + +**功能**： +- [ ] `PutEnd()`：记录 `PUT_END` 事件（✅ 已实现） +- [ ] `PutRevoke()`：记录 `PUT_REVOKE` 事件（✅ 已实现） +- [ ] `Remove()`：记录 `REMOVE` 事件（✅ 已实现） +- [ ] `BatchEvict()`：在完全驱逐对象时记录 `REMOVE` 事件（待实现） + +**验收标准**： +- 所有状态变更操作都记录 OpLog +- OpLog 成功写入 etcd + +#### 1.4 实现 etcd Helper 扩展（2-3 天） + +**文件**： +- `mooncake-store/include/etcd_helper.h`（修改） +- `mooncake-store/src/etcd_helper.cpp`（修改） +- `mooncake-store/src/etcd_wrapper.go`（修改） + +**功能**： +- [ ] `GetFirstKeyWithPrefix()`：获取指定前缀的第一个 key（用于 OpLog 清理） +- [ ] `DeleteRange()`：删除指定范围的 key（用于 OpLog 清理） +- [ ] `WatchWithPrefix()`：Watch 指定前缀的 key 变化（用于 OpLog 同步） + +**验收标准**： +- 所有 etcd 操作都有对应的 Helper 方法 +- 错误处理完善 + +### Phase 1 里程碑 + +- ✅ EtcdOpLogStore 可以写入和读取 OpLog +- ✅ Primary 的所有状态变更都写入 etcd +- ✅ etcd Helper 支持所有需要的操作 + +### 测试要求 + +- [ ] 单元测试：EtcdOpLogStore 的读写功能 +- [ ] 集成测试：Primary 写入 OpLog 到 etcd +- [ ] 性能测试：写入性能（目标：> 1000 ops/s） + +--- + +## Phase 2：Standby 服务集成（优先级：P0） + +### 目标 +实现 Standby 服务，使其在等待 leader 选举期间能够 watch etcd OpLog 并实时恢复 metadata。 + +### 任务清单 + +#### 2.1 实现 OpLogWatcher（3-4 天） + +**文件**： +- `mooncake-store/include/oplog_watcher.h`（已创建） +- `mooncake-store/src/oplog_watcher.cpp`（待实现） + +**功能**： +- [ ] `Start()`：启动 Watch 线程 +- [ ] `Stop()`：停止 Watch 线程 +- [ ] `WatchOpLogThreadFunc()`：Watch etcd OpLog 变化 +- [ ] `HandleWatchEvent()`：处理 Watch 事件（PUT/DELETE） +- [ ] `ReadOpLogSince()`：读取历史 OpLog（用于初始同步） + +**依赖**： +- Phase 1.4：`WatchWithPrefix()` 方法 + +**验收标准**： +- 可以成功 Watch etcd OpLog 变化 +- 收到新 OpLog 时调用 `OpLogApplier::ApplyOpLogEntry()` +- 支持断点续传（从上次处理的 sequence_id 继续） + +#### 2.2 实现 OpLogApplier 基础功能（3-4 天） + +**文件**： +- `mooncake-store/include/oplog_applier.h`（已创建） +- `mooncake-store/src/oplog_applier.cpp`（待实现） + +**功能**： +- [ ] `ApplyOpLogEntry()`：应用 OpLog Entry +- [ ] `ApplyPutEnd()`：应用 PUT_END 操作 +- [ ] `ApplyPutRevoke()`：应用 PUT_REVOKE 操作 +- [ ] `ApplyRemove()`：应用 REMOVE 操作 +- [ ] `CheckSequenceOrder()`：检查全局和 key 级别的时序性 +- [ ] `GetLastAppliedSequenceId()`：获取最后应用的 sequence_id + +**依赖**： +- `MetadataStore` 接口（需要定义） + +**验收标准**： +- 可以成功应用 OpLog 到 metadata_store +- 时序检查正确 +- 支持断点续传 + +#### 2.3 修改 HotStandbyService 使用 etcd Watch（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] 修改 `ReplicationLoop()` 使用 `OpLogWatcher` +- [ ] 先读取历史 OpLog，再启动 Watch +- [ ] 实现 `OpLogApplier` 接口 +- [ ] 处理 Watch 事件并应用 OpLog + +**验收标准**： +- Standby 可以 watch etcd OpLog +- 实时应用 OpLog 到 metadata_store + +#### 2.4 修改 MasterServiceSupervisor 支持 Standby 模式（2-3 天） + +**文件**： +- `mooncake-store/src/ha_helper.cpp`（修改） + +**功能**： +- [ ] 检测到有 leader 时，启动 `HotStandbyService` +- [ ] Standby 服务 watch etcd OpLog 并实时恢复 metadata +- [ ] 选举成功后，停止 Standby 服务并提升为 Primary + +**验收标准**： +- Standby 在等待选举期间持续运行 +- 选举成功后可以正常提升为 Primary + +### Phase 2 里程碑 + +- ✅ Standby 可以 watch etcd OpLog +- ✅ Standby 实时应用 OpLog 到 metadata_store +- ✅ Standby 在等待选举期间持续运行 + +### 测试要求 + +- [ ] 单元测试：OpLogWatcher 和 OpLogApplier +- [ ] 集成测试：Standby watch OpLog 并应用 +- [ ] 端到端测试：Primary 写入，Standby 同步 + +--- + +## Phase 3：时序保证和容错（优先级：P1） + +### 目标 +实现完整的时序保证机制和容错处理，确保数据一致性。 + +### 任务清单 + +#### 3.1 实现序列号不连续处理（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） + +**功能**： +- [ ] `ProcessPendingEntries()`：处理待处理的条目 +- [ ] `ScheduleWaitForMissingEntries()`：等待缺失的条目 +- [ ] `RequestMissingOpLog()`：从 etcd 请求缺失的 OpLog +- [ ] 维护 `pending_entries_` 和 `expected_sequence_id_` + +**验收标准**： +- 检测到序列号不连续时，缓存待处理 +- 等待一段时间后，从 etcd 读取缺失的条目 +- 序列号连续后，按顺序应用 + +#### 3.2 实现 key 级别乱序处理（1-2 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） + +**功能**： +- [x] 检测到 key 级别乱序时，直接删除该 key 的 metadata +- [x] 从 `key_sequence_map_` 中删除该 key +- [x] 删除后继续处理当前 OpLog 条目（如果全局序列号正确） + +**设计说明**： +- 简化方案：不进行回滚和重放，因为前面的数据可能已经丢失 +- 当检测到 `key_sequence_id` 乱序时，直接删除该 key +- 如果后续有 PUT_END 操作，会重新创建该 key +- 这样避免了数据不一致的风险，实现更简单可靠 + +**验收标准**： +- 检测到 key 级别乱序时，正确删除该 key 的 metadata +- 删除后可以继续处理后续的 OpLog 条目 +- 不会导致数据不一致 + +#### 3.3 实现错误处理和恢复（2-3 天） + +**文件**： +- `mooncake-store/src/oplog_applier.cpp`（修改） +- `mooncake-store/src/oplog_watcher.cpp`（修改） +- `mooncake-store/include/oplog_watcher.h`（修改） + +**功能**： +- [x] Watch 断开时自动重连（指数退避策略） +- [x] 重连时同步遗漏的 OpLog 条目（`SyncMissedEntries()`） +- [x] 连续错误计数，超过阈值（10次）时触发重连 +- [x] 重连成功后重置错误计数 +- [x] 完善的日志记录 + +**实现细节**： +- `kMaxConsecutiveErrors = 10`：连续错误超过此阈值触发重连 +- `kReconnectDelayMs = 1000`：初始重连延迟（毫秒） +- `kMaxReconnectDelayMs = 30000`：最大重连延迟（30秒） +- `TryReconnect()`：指数退避重连，重连前同步遗漏条目 +- `SyncMissedEntries()`：从 etcd 读取 `last_processed_sequence_id_` 之后的条目 + +**验收标准**： +- Watch 断开后可以自动重连 +- 重连期间遗漏的 OpLog 可以被正确同步 +- 错误处理完善，不会导致服务崩溃 +- 有完善的日志记录 + +### Phase 3 里程碑 + +- ✅ 序列号不连续时可以正确处理 +- ✅ key 级别乱序时可以回滚和重放 +- ✅ 错误处理和恢复机制完善 + +### 测试要求 + +- [ ] 单元测试：序列号不连续处理 +- [ ] 单元测试：回滚和重放机制 +- [ ] 集成测试：错误恢复场景 +- [ ] 压力测试：大量乱序情况下的性能 + +--- + +## Phase 4：快照集成和清理（优先级：P2）⏸️ 暂缓 + +> **状态**：暂缓，等待与快照团队协调讨论后再实现。 + +### 目标 +集成快照机制，实现 OpLog 清理，减少 etcd 存储压力。 + +### 任务清单 + +#### 4.1 实现快照时记录 sequence_id（2-3 天） + +**文件**： +- `mooncake-store/src/master_service.cpp`（修改） +- 快照相关代码（待确定） + +**功能**： +- [ ] 快照时记录 `last_oplog_sequence_id` +- [ ] 将快照信息写入 etcd（`RecordSnapshotSequenceId()`） +- [ ] Standby 可以从快照点开始同步 + +**验收标准**： +- 快照包含 OpLog 的 sequence_id +- 快照信息可以持久化到 etcd + +#### 4.2 实现 OpLog 清理机制（2-3 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] `CleanupOpLogBefore()`：清理指定 sequence_id 之前的 OpLog +- [ ] `GetMinSequenceId()`：从 etcd 查询最小的 sequence_id +- [ ] 使用 `DeleteRange` 批量删除 +- [ ] 定期清理（在快照后或定时任务中） + +**依赖**： +- Phase 1.4：`GetFirstKeyWithPrefix()` 和 `DeleteRange()` + +**验收标准**： +- 可以成功清理旧的 OpLog +- 清理后不影响 Standby 的同步（因为已有快照） + +#### 4.3 实现 Standby 初始同步（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] 从 Primary 获取快照（或从 etcd 读取最新快照） +- [ ] 应用快照到 metadata_store +- [ ] 从快照的 sequence_id 开始读取增量 OpLog +- [ ] 应用增量 OpLog +- [ ] 启动 Watch 监听新 OpLog + +**验收标准**： +- 新 Standby 可以成功完成初始同步 +- 初始同步后，metadata 与 Primary 一致 + +### Phase 4 里程碑 + +- ✅ 快照时记录 sequence_id +- ✅ 可以清理旧的 OpLog +- ✅ Standby 可以从快照开始同步 + +### 测试要求 + +- [ ] 单元测试：OpLog 清理功能 +- [ ] 集成测试：快照集成 +- [ ] 端到端测试：新 Standby 初始同步 + +--- + +## Phase 5：优化和完善（优先级：P3） + +### 目标 +优化性能，完善功能，提升系统稳定性。 + +### 任务清单 + +#### 5.1 实现 Standby 提升时的 Lease 初始化（2-3 天） + +**文件**： +- `mooncake-store/src/hot_standby_service.cpp`（修改） + +**功能**： +- [ ] `InitializeLeasesForAllObjects()`：给所有 lease 为 0 的对象授予默认租约 +- [ ] `PerformFullEvictionCleanup()`：执行一次完整的驱逐清理 +- [ ] 在 `Promote()` 中调用上述方法 + +**验收标准**： +- Standby 提升为 Primary 时，所有对象都有有效的 lease +- 提升后可以正常执行驱逐 + +#### 5.2 实现批量写入优化（可选，1-2 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] `WriteOpLogBatch()`：批量写入 OpLog +- [ ] 使用事务保证原子性 +- [ ] 减少 etcd 写入次数 + +**验收标准**： +- 批量写入性能提升 +- 不影响数据一致性 + +#### 5.3 实现 OpLog 压缩（可选，1-2 天） + +**文件**： +- `mooncake-store/src/etcd_oplog_store.cpp`（修改） + +**功能**： +- [ ] OpLog Entry 压缩（如使用 gzip） +- [ ] 减少 etcd 存储大小 + +**验收标准**： +- 压缩后存储大小减少 +- 不影响读取性能 + +#### 5.4 完善监控和告警（1-2 天） + +**功能**： +- [ ] OpLog 写入速率监控 +- [ ] Standby 同步延迟监控 +- [ ] 乱序频率监控 +- [ ] 回滚次数和耗时监控 +- [ ] 告警机制（超过阈值时告警） + +**验收标准**： +- 所有关键指标都有监控 +- 有完善的告警机制 + +### Phase 5 里程碑 + +- ✅ Standby 提升时 lease 初始化完成 +- ✅ 性能优化完成 +- ✅ 监控和告警完善 + +### 测试要求 + +- [ ] 单元测试：Lease 初始化 +- [ ] 性能测试：批量写入和压缩效果 +- [ ] 监控测试：监控指标正确 + +--- + +## 依赖关系图 + +``` +Phase 1: 基础框架 + ├─ 1.1 EtcdOpLogStore + ├─ 1.2 集成到 OpLogManager + ├─ 1.3 MasterService 记录 OpLog + └─ 1.4 etcd Helper 扩展 + │ + ▼ +Phase 2: Standby 服务集成 + ├─ 2.1 OpLogWatcher ──────┐ + ├─ 2.2 OpLogApplier ──────┤ + ├─ 2.3 HotStandbyService ─┤ + └─ 2.4 MasterServiceSupervisor ─┐ + │ │ + ▼ │ +Phase 3: 时序保证和容错 │ + ├─ 3.1 序列号不连续处理 │ + ├─ 3.2 回滚和重放机制 │ + └─ 3.3 错误处理和恢复 │ + │ │ + ▼ │ +Phase 4: 快照集成和清理 │ + ├─ 4.1 快照记录 sequence_id │ + ├─ 4.2 OpLog 清理 ───────────────┘ + └─ 4.3 Standby 初始同步 + │ + ▼ +Phase 5: 优化和完善 + ├─ 5.1 Lease 初始化 + ├─ 5.2 批量写入优化（可选） + ├─ 5.3 OpLog 压缩（可选） + └─ 5.4 监控和告警 +``` + +## 关键里程碑 + +| 里程碑 | 阶段 | 验收标准 | +|--------|------|----------| +| **M1** | Phase 1 完成 | Primary 可以写入 OpLog 到 etcd | +| **M2** | Phase 2 完成 | Standby 可以 watch OpLog 并实时同步 | +| **M3** | Phase 3 完成 | 时序保证和容错机制完善 | +| **M4** | Phase 4 完成 | 快照集成和 OpLog 清理完成 | +| **M5** | Phase 5 完成 | 所有优化和完善完成 | + +## 风险评估 + +### 高风险项 + +1. **etcd 性能瓶颈** + - **风险**：大量 OpLog 写入可能导致 etcd 性能下降 + - **缓解**：批量写入、压缩、定期清理 + - **监控**：etcd 写入速率、延迟、存储大小 + +2. **Watch 断开和重连** + - **风险**：Watch 断开可能导致数据丢失 + - **缓解**：自动重连、断点续传、从 etcd 重新读取 + - **监控**：Watch 断开次数、重连时间 + +3. **序列号乱序** + - **风险**：乱序可能导致数据不一致 + - **缓解**：回滚和重放机制、监控告警 + - **监控**：乱序频率、回滚次数 + +### 中风险项 + +1. **Standby 提升时的数据迁移** + - **风险**：metadata 迁移可能失败 + - **缓解**：完善的错误处理、回滚机制 + - **监控**：提升成功率、迁移耗时 + +2. **快照和 OpLog 的一致性** + - **风险**：快照和 OpLog 可能不一致 + - **缓解**：快照时记录 sequence_id、验证机制 + - **监控**：快照和 OpLog 的一致性检查 + +## 测试策略 + +### 单元测试 + +- [ ] EtcdOpLogStore 的所有方法 +- [ ] OpLogWatcher 的 Watch 功能 +- [ ] OpLogApplier 的应用逻辑 +- [ ] 时序检查逻辑 +- [ ] 回滚和重放逻辑 + +### 集成测试 + +- [ ] Primary 写入 → etcd → Standby 同步 +- [ ] Standby 初始同步（快照 + OpLog） +- [ ] Standby 提升为 Primary +- [ ] OpLog 清理机制 +- [ ] 错误恢复场景 + +### 端到端测试 + +- [ ] 完整的主备切换流程 +- [ ] 长时间运行稳定性测试 +- [ ] 高负载下的性能测试 +- [ ] 故障注入测试 + +### 性能测试 + +- [ ] OpLog 写入性能（目标：> 1000 ops/s） +- [ ] Standby 同步延迟（目标：< 100ms） +- [ ] etcd 存储大小（目标：10 分钟内 < 1GB） +- [ ] 回滚和重放性能 + +## 文档要求 + +### 必须完成的文档 + +- [x] 主设计文档：`doc/zh/rfc-oplog-via-etcd-complete-design.md` +- [x] Standby 服务集成：`doc/zh/rfc-standby-service-integration.md` +- [x] Lease 初始化：`doc/zh/rfc-standby-promotion-lease-initialization.md` +- [x] OpLog 清理：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md` +- [x] 回滚和重放：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` +- [x] 实施计划：`doc/zh/rfc-oplog-implementation-plan.md`（本文档） + +### 可选文档 + +- [ ] API 文档：各个类的接口说明 +- [ ] 运维文档：部署和运维指南 +- [ ] 故障排查文档：常见问题和解决方案 + +## 时间估算 + +### 总体时间 + +- **Phase 1**：2-3 周（P0） +- **Phase 2**：2-3 周（P0） +- **Phase 3**：2-3 周（P1） +- **Phase 4**：1-2 周（P2） +- **Phase 5**：1-2 周（P3） + +**总计**：8-13 周（约 2-3 个月） + +### 关键路径 + +``` +Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 +``` + +**最短时间**：8 周（如果所有阶段都按最短时间完成） + +### 并行开发可能性 + +- **Phase 1 和 Phase 2**：可以部分并行（Phase 2 的 OpLogApplier 可以在 Phase 1 完成后开始） +- **Phase 3 和 Phase 4**：可以部分并行（快照集成和时序保证相对独立） +- **Phase 5**：可以在 Phase 1-4 完成后开始 + +## 优先级说明 + +### P0（最高优先级） + +- **Phase 1**：基础框架，所有后续功能都依赖于此 +- **Phase 2**：Standby 服务集成，核心功能 + +**必须完成**：这两个阶段是核心功能，必须优先完成。 + +### P1（高优先级） + +- **Phase 3**：时序保证和容错，确保数据一致性 + +**重要**：这个阶段确保数据一致性，应该在 Phase 1-2 完成后尽快完成。 + +### P2（中优先级） + +- **Phase 4**：快照集成和清理，减少存储压力 + +**可选但推荐**：这个阶段可以减少 etcd 存储压力，建议完成。 + +### P3（低优先级） + +- **Phase 5**：优化和完善，提升系统稳定性 + +**可选**：这个阶段是优化，可以在系统稳定运行后再完成。 + +## 实施建议 + +### 第一步：完成 Phase 1 + +1. **先实现 `EtcdOpLogStore` 的基础功能**（写入和读取） + - 确保可以成功写入和读取 OpLog + - 完成单元测试 + +2. **集成到 `OpLogManager`** + - 确保 Primary 可以写入 OpLog + - 完成集成测试 + +3. **扩展 `EtcdHelper`** + - 支持所有需要的操作 + - 完成单元测试 + +4. **完成测试** + - 单元测试、集成测试、性能测试 + +### 第二步：完成 Phase 2 + +1. **实现 `OpLogWatcher`** + - 支持 Watch etcd + - 完成单元测试 + +2. **实现 `OpLogApplier` 基础功能** + - 可以应用 OpLog + - 完成单元测试 + +3. **修改 `HotStandbyService`** + - 使用 etcd Watch + - 完成集成测试 + +4. **修改 `MasterServiceSupervisor`** + - 支持 Standby 模式 + - 完成端到端测试 + +### 第三步：完成 Phase 3 + +1. **实现序列号不连续处理** + - 缓存待处理条目 + - 从 etcd 读取缺失条目 + +2. **实现回滚和重放机制** + - 检测乱序 + - 回滚和重放 + +3. **完善错误处理和恢复** + - Watch 重连 + - 错误重试 + +4. **完成压力测试** + +### 第四步：完成 Phase 4 和 Phase 5 + +1. **实现快照集成和 OpLog 清理** + - 快照时记录 sequence_id + - 清理旧的 OpLog + +2. **实现 Standby 提升时的 Lease 初始化** + - 初始化所有对象的 lease + - 执行驱逐清理 + +3. **优化性能和完善监控** + - 批量写入（可选） + - OpLog 压缩（可选） + - 监控和告警 + +4. **完成所有测试** + +## 关键成功因素 + +### 1. 代码质量 + +- **代码审查**：每个阶段完成后进行代码审查 +- **单元测试覆盖率**：目标 > 80% +- **集成测试**：确保各组件正确集成 + +### 2. 性能要求 + +- **OpLog 写入性能**：> 1000 ops/s +- **Standby 同步延迟**：< 100ms +- **etcd 存储大小**：10 分钟内 < 1GB + +### 3. 稳定性要求 + +- **错误处理**：所有错误都有完善的处理 +- **自动恢复**：Watch 断开、读取失败等可以自动恢复 +- **监控告警**：关键指标都有监控和告警 + +### 4. 文档要求 + +- **设计文档**：所有设计都有详细文档 +- **API 文档**：所有接口都有文档 +- **运维文档**：部署和运维指南 + +## 总结 + +本实施计划按照依赖关系和重要性，将整个项目分为 5 个阶段。**Phase 1 和 Phase 2 是核心功能，必须优先完成**。Phase 3 确保数据一致性，Phase 4 和 Phase 5 是优化和完善。 + +**建议按照阶段顺序实施，每个阶段完成后进行充分测试，确保稳定性后再进入下一阶段。** + +### 关键要点 + +1. **优先级明确**：P0 > P1 > P2 > P3 +2. **依赖关系清晰**：Phase 1 → Phase 2 → Phase 3 → Phase 4 → Phase 5 +3. **测试充分**：每个阶段都有对应的测试要求 +4. **风险可控**：识别了高风险项并提供了缓解措施 +5. **时间合理**：总计 8-13 周，符合项目时间要求 + +### 下一步行动 + +1. **评审本计划**：与团队评审实施计划 +2. **分配任务**：根据计划分配开发任务 +3. **开始 Phase 1**：从基础框架开始实施 +4. **定期检查**：每周检查进度，确保按计划进行 + diff --git a/doc/zh/rfc-oplog-key-sequence-map-cleanup.md b/doc/zh/rfc-oplog-key-sequence-map-cleanup.md new file mode 100644 index 0000000000..6d954d46dc --- /dev/null +++ b/doc/zh/rfc-oplog-key-sequence-map-cleanup.md @@ -0,0 +1,253 @@ +# OpLogApplier key_sequence_map_ 清理策略 + +## 问题描述 + +在 Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`，以确保 OpLog 的顺序正确性。当 metadata 被删除（REMOVE 操作）后，`key_sequence_map_` 中的条目仍然保留，用于检测可能的乱序操作。 + +### 内存泄漏风险 + +如果 `key_sequence_map_` 中的条目一直不删除，长期运行可能导致内存泄漏： + +- **内存占用**：每个条目约 90 字节（string key + uint64_t value + hash map 开销） +- **累积效应**：系统长期运行，可能有数百万个不同的 key 曾经存在过 +- **极端场景**：如果每天创建 10 万个新 key，运行 100 天，累计 1000 万个不同的 key，内存占用可达 900MB + +### 清理需求 + +需要在保证功能正确性的前提下，实现内存清理机制。 + +## 解决方案 + +### 核心策略 + +**清理条件**： +1. 最后一次操作是 `REMOVE`（DELETE） +2. 距离当前时间超过 1 小时 + +**清理频率**：每小时扫描一次 + +**保留策略**： +- `PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） +- 即使超过 1 小时，只要最后操作不是 `REMOVE`，也保留 + +### 设计原理 + +1. **乱序检测时间窗口**：乱序检测一般只需要秒级的时间窗口，1 小时的保留时间足够处理网络延迟、重传等情况 +2. **只清理 DELETE 操作**：因为 DELETE 操作的 metadata 已经不存在，且超过 1 小时后不太可能再出现乱序 +3. **保留 PUT 操作**：PUT 操作的 metadata 可能仍存在，需要保留用于顺序检查 + +## 实现设计 + +### 数据结构 + +```cpp +class OpLogApplier { +private: + struct KeySequenceInfo { + uint64_t sequence_id{0}; + OpType last_op_type{OpType::PUT_END}; + std::chrono::steady_clock::time_point last_op_time; + + KeySequenceInfo() + : last_op_time(std::chrono::steady_clock::now()) {} + }; + + std::unordered_map key_sequence_map_; + mutable std::mutex key_sequence_mutex_; + + // 清理配置 + static constexpr std::chrono::hours kCleanupInterval{1}; // 每小时清理一次 + static constexpr std::chrono::hours kStaleThreshold{1}; // 1小时未访问则清理（仅限DELETE） + + std::chrono::steady_clock::time_point last_cleanup_time_; +}; +``` + +### 核心方法 + +#### 1. 定期清理检查 + +```cpp +void OpLogApplier::PeriodicCleanup() { + auto now = std::chrono::steady_clock::now(); + if (now - last_cleanup_time_ < kCleanupInterval) { + return; // 还没到清理时间 + } + + CleanupStaleKeySequences(); + last_cleanup_time_ = now; +} +``` + +#### 2. 清理过期条目 + +```cpp +void OpLogApplier::CleanupStaleKeySequences() { + std::lock_guard lock(key_sequence_mutex_); + auto now = std::chrono::steady_clock::now(); + auto threshold = now - kStaleThreshold; + + size_t cleaned = 0; + for (auto it = key_sequence_map_.begin(); + it != key_sequence_map_.end();) { + const auto& info = it->second; + + // 清理条件： + // 1. 最后一次操作是 REMOVE（DELETE） + // 2. 且距离当前超过1小时 + if (info.last_op_type == OpType::REMOVE && + info.last_op_time < threshold) { + it = key_sequence_map_.erase(it); + cleaned++; + } else { + ++it; + } + } + + if (cleaned > 0) { + LOG(INFO) << "Cleaned up " << cleaned + << " stale key_sequence_map entries " + << "(REMOVE operations older than 1 hour)"; + } +} +``` + +#### 3. 应用 OpLog 时更新 + +```cpp +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 检查顺序 + if (!CheckSequenceOrder(entry)) { + // 处理乱序... + return false; + } + + // 2. 应用操作 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 3. 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + auto& info = key_sequence_map_[entry.object_key]; + info.sequence_id = entry.key_sequence_id; + info.last_op_type = entry.op_type; + info.last_op_time = std::chrono::steady_clock::now(); + } + + // 4. 定期清理（每次应用时检查，避免额外线程） + PeriodicCleanup(); + + return true; +} +``` + +## 关键设计要点 + +### 1. 清理时机 + +- **触发方式**：在 `ApplyOpLogEntry` 中检查，无需额外线程 +- **清理频率**：每小时执行一次 +- **清理条件**：只清理 `REMOVE` 操作且超过 1 小时的条目 + +### 2. 安全性保证 + +- **保留 PUT 操作**：`PUT_END` 和 `PUT_REVOKE` 的 key 不清理，因为 metadata 可能仍存在 +- **1 小时窗口**：足够处理网络延迟、重传等异常情况 +- **线程安全**：使用 mutex 保护 `key_sequence_map_` 的访问 + +### 3. 内存占用控制 + +**清理前**： +- 假设系统长期运行，有 100 万个不同的 key 曾经存在过 +- 内存占用：100万 × 90字节 ≈ 90MB + +**清理后**： +- 假设系统每小时处理 10 万个 OpLog，其中 10% 是 REMOVE 操作 +- `key_sequence_map_` 中最多保留： + - 最近 1 小时的 REMOVE key：约 1 万个 + - 所有 PUT_END/PUT_REVOKE 的 key：取决于实际 metadata 数量 +- 内存占用：约 `(活跃key数量 + 1万) × 90字节` + +**内存节省**：从 90MB 降低到约 `(活跃key数量 + 1万) × 90字节`，通常远小于不清理的情况。 + +## 使用场景示例 + +### 场景 1：正常 REMOVE 操作 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} + +2. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=6, op_type=REMOVE + → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:05} + → metadata 被删除 + +3. 1小时后（11:05），清理扫描 + → 检测到 "obj1" 的 last_op_type=REMOVE 且超过1小时 + → 清理 key_sequence_map_["obj1"] +``` + +### 场景 2：乱序 REMOVE 操作 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:5, op:PUT_END, time:10:00} + +2. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END + → 应用成功，key_sequence_map_["obj1"] = {seq:6, op:PUT_END, time:10:02} + +3. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE + → 检测到乱序：entry.key_sequence_id(5) <= current(6) + → 触发回滚和重放 + → key_sequence_map_["obj1"] 保留用于重放 +``` + +### 场景 3：删除后重新创建 + +``` +时间线： +1. key="obj1" 被 REMOVE，key_sequence_map_["obj1"] = {seq:6, op:REMOVE, time:10:00} + +2. 30分钟后（10:30），Standby 收到 OpLog: sequence_id=200, key="obj1", key_sequence_id=7, op_type=PUT_END + → 检查：key_sequence_map_["obj1"] 存在，seq=6（期望） + → 应用成功，key_sequence_map_["obj1"] = {seq:7, op:PUT_END, time:10:30} + → 不会被清理（因为 last_op_type=PUT_END） +``` + +## 配置参数 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `kCleanupInterval` | 1 小时 | 清理检查的间隔时间 | +| `kStaleThreshold` | 1 小时 | REMOVE 操作超过此时间后可以清理 | + +## 优势 + +1. **内存控制**：及时清理已删除且超过 1 小时的 key，有效控制内存占用 +2. **安全性**：保留最近删除的 key，确保乱序检测的正确性 +3. **简单高效**：无需额外线程，在应用 OpLog 时检查，实现简单 +4. **精确清理**：只清理符合条件的条目，不影响活跃 key + +## 注意事项 + +1. **清理时机**：清理在 `ApplyOpLogEntry` 中触发，如果长时间没有 OpLog，可能不会及时清理 +2. **时间精度**：使用 `std::chrono::steady_clock`，不受系统时间调整影响 +3. **线程安全**：所有对 `key_sequence_map_` 的访问都需要加锁保护 + +## 相关文档 + +- [OpLog 主备同步完整方案](./rfc-oplog-via-etcd-complete-design.md) +- [OpLog 序列号乱序时的回滚和重放方案](./rfc-oplog-rollback-replay-on-sequence-violation.md) + diff --git a/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md b/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md new file mode 100644 index 0000000000..5b3338521f --- /dev/null +++ b/doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md @@ -0,0 +1,653 @@ +# OpLog 序列号乱序时的回滚和重放方案 + +## 问题描述 + +当 Standby 检测到某个 key 的 `key_sequence_id` 乱序时（例如：收到了 `key_sequence_id=5`，但之前已经处理了 `key_sequence_id=6`），说明该 key 的 metadata 可能已经不一致。 + +### 问题场景 + +``` +时间线： +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5, op_type=PUT_END +2. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 5 +3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6, op_type=PUT_END +4. Standby 应用成功，metadata 中 obj1 的 key_sequence_id = 6 +5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5, op_type=REMOVE + ❌ 乱序！key_sequence_id=5 < 当前值 6 +``` + +**问题**： +- 该 key 的 metadata 可能已经不一致 +- 需要修复该 key 的数据状态 + +## 解决方案 + +### 核心思路 + +**对于乱序的 key，执行回滚和重放**： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ OpLogApplier::ApplyOpLogEntry() │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 检查 key_sequence_id 是否递增 │ +│ - 如果乱序 → 触发回滚和重放 │ +└─────────────────────────────────────────────────────────┘ + │ + ├─ 正常顺序 + │ │ + │ ▼ + │ ┌─────────────────────────────────────────┐ + │ │ 正常应用 OpLog │ + │ └─────────────────────────────────────────┘ + │ + └─ 乱序 + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. 回滚：删除该 key 的 metadata │ +│ - metadata_store_->RemoveKey(key) │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 2. 从 etcd 重新读取该 key 的所有 OpLog │ +│ - ReadOpLogForKey(key, first_seq_id) │ +│ - 过滤出该 key 的条目 │ +│ - 按 sequence_id 排序 │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 3. 按顺序重新应用所有 OpLog │ +│ - 跳过时序检查（因为已经排序） │ +│ - 重新构建 metadata │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现设计 + +### 1. 方案 A：基于 etcd 的完整重放（推荐） + +**优点**： +- 数据准确：从 etcd 读取保证数据正确 +- 实现简单：不需要维护操作历史 +- 内存友好：不需要额外存储 +- 容错性好：即使本地状态丢失也能恢复 + +**缺点**： +- 需要从 etcd 读取：可能有网络 I/O 开销 +- 可能较慢：如果该 key 的操作很多 + +#### 实现代码 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的首次 sequence_id（用于回滚） + std::unordered_map key_first_sequence_id_; + std::mutex key_first_sequence_mutex_; + + // 记录正在回滚的 key（防止并发回滚） + std::set keys_under_rollback_; + std::mutex rollback_mutex_; + + EtcdOpLogStore* etcd_oplog_store_; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 记录首次 sequence_id + { + std::lock_guard lock(key_first_sequence_mutex_); + if (key_first_sequence_id_.count(entry.object_key) == 0) { + key_first_sequence_id_[entry.object_key] = entry.sequence_id; + } + } + + // 2. 检查 key 级别的时序性 + if (!CheckSequenceOrder(entry)) { + LOG(ERROR) << "Key-level sequence order violation for key: " + << entry.object_key + << ", entry_seq=" << entry.key_sequence_id + << ", current_seq=" << GetKeySequenceId(entry.object_key); + + // 3. 触发回滚和重放（异步执行，不阻塞） + std::thread([this, key = entry.object_key]() { + RollbackAndReplayKey(key); + }).detach(); + + // 暂时跳过这个条目，等待回滚完成 + return false; + } + + // 4. 正常应用 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 5. 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[entry.object_key] = entry.key_sequence_id; + } + + return true; + } + +private: + bool RollbackAndReplayKey(const std::string& key) { + // 1. 检查是否正在回滚（防止并发回滚） + { + std::lock_guard lock(rollback_mutex_); + if (keys_under_rollback_.count(key) > 0) { + LOG(WARNING) << "Key is already under rollback: " << key; + return false; + } + keys_under_rollback_.insert(key); + } + + // 2. 获取该 key 的首次 sequence_id + uint64_t first_seq_id; + { + std::lock_guard lock(key_first_sequence_mutex_); + auto it = key_first_sequence_id_.find(key); + if (it == key_first_sequence_id_.end()) { + LOG(ERROR) << "Cannot find first sequence_id for key: " << key; + std::lock_guard lock2(rollback_mutex_); + keys_under_rollback_.erase(key); + return false; + } + first_seq_id = it->second; + } + + // 3. 回滚：从 metadata_store_ 中删除该 key + LOG(INFO) << "Rolling back key: " << key + << ", removing from metadata_store"; + metadata_store_->RemoveKey(key); + + // 4. 从 etcd 重新读取该 key 的所有 OpLog + LOG(INFO) << "Re-reading OpLog for key: " << key + << " from sequence_id: " << first_seq_id; + + std::vector key_entries; + if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { + LOG(ERROR) << "Failed to read OpLog for key: " << key; + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + return false; + } + + // 5. 按顺序重新应用所有 OpLog + LOG(INFO) << "Replaying " << key_entries.size() + << " OpLog entries for key: " << key; + + for (const auto& entry : key_entries) { + // 重新应用（跳过时序检查，因为我们已经从 etcd 读取了正确的顺序） + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 更新 key_sequence_map_ + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[key] = entry.key_sequence_id; + } + } + + // 6. 清除回滚标记 + { + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + } + + LOG(INFO) << "Successfully replayed OpLog for key: " << key; + return true; + } + + bool ReadOpLogForKey(const std::string& key, + uint64_t start_seq_id, + std::vector& entries) { + // 从 etcd 读取从 start_seq_id 开始的所有 OpLog + std::vector all_entries; + const uint32_t batch_size = 10000; // 批量读取 + + uint64_t current_seq_id = start_seq_id; + while (true) { + std::vector batch; + if (!etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch)) { + LOG(ERROR) << "Failed to read OpLog from etcd"; + return false; + } + + if (batch.empty()) { + break; // 没有更多条目 + } + + // 过滤出该 key 的条目 + for (const auto& entry : batch) { + if (entry.object_key == key) { + entries.push_back(entry); + } + } + + // 更新 current_seq_id + if (batch.size() < batch_size) { + break; // 已读取完所有条目 + } + current_seq_id = batch.back().sequence_id + 1; + } + + // 按 sequence_id 排序（确保顺序正确） + std::sort(entries.begin(), entries.end(), + [](const OpLogEntry& a, const OpLogEntry& b) { + return a.sequence_id < b.sequence_id; + }); + + return true; + } +}; +``` + +### 2. 方案 B：基于操作历史的回滚（可选） + +**优点**： +- 快速：不需要网络 I/O +- 高效：直接从内存读取 + +**缺点**： +- 需要额外内存：存储操作历史 +- 实现复杂：需要维护历史记录 +- 容错性差：如果历史丢失，无法恢复 + +#### 实现代码 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的操作历史（用于回滚） + struct KeyOperationHistory { + std::vector operations; // 按顺序记录的操作 + uint64_t first_sequence_id{0}; + }; + std::unordered_map key_history_; + std::mutex key_history_mutex_; + + // 限制历史记录的大小（避免内存无限增长） + static constexpr size_t kMaxHistorySize = 1000; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 记录操作历史 + { + std::lock_guard lock(key_history_mutex_); + auto& history = key_history_[entry.object_key]; + if (history.operations.empty()) { + history.first_sequence_id = entry.sequence_id; + } + + // 限制历史记录大小 + if (history.operations.size() < kMaxHistorySize) { + history.operations.push_back(entry); + } else { + // 如果超过限制，只保留最近的操作 + history.operations.erase(history.operations.begin()); + history.operations.push_back(entry); + } + } + + // 2. 检查时序性 + if (!CheckSequenceOrder(entry)) { + return RollbackAndReplayKey(entry.object_key); + } + + // 3. 正常应用 + // ... + } + +private: + bool RollbackAndReplayKey(const std::string& key) { + std::lock_guard lock(key_history_mutex_); + + auto it = key_history_.find(key); + if (it == key_history_.end()) { + LOG(ERROR) << "Cannot find history for key: " << key; + return false; + } + + // 1. 回滚：删除该 key 的 metadata + metadata_store_->RemoveKey(key); + + // 2. 重新应用所有操作（从历史记录中） + for (const auto& entry : it->second.operations) { + // 重新应用 + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + } + + // 更新 key_sequence_map_ + { + std::lock_guard lock2(key_sequence_mutex_); + key_sequence_map_[key] = entry.key_sequence_id; + } + } + + return true; + } +}; +``` + +## 关键设计点 + +### 1. 回滚起点的确定 + +**方案 A（推荐）**： +- 维护 `key_first_sequence_id_` 记录每个 key 第一次出现的 sequence_id +- 从该 sequence_id 开始重新读取所有 OpLog + +**方案 B**： +- 维护操作历史，从历史记录中获取所有操作 + +### 2. 并发处理 + +**问题**：回滚期间，如果收到新的 OpLog 怎么办？ + +**解决方案**： +- 使用 `keys_under_rollback_` 标记正在回滚的 key +- 回滚期间，新的 OpLog 暂时跳过（返回 false） +- 回滚完成后，新的 OpLog 可以正常处理 + +```cpp +bool ApplyOpLogEntry(const OpLogEntry& entry) { + // 检查是否正在回滚 + { + std::lock_guard lock(rollback_mutex_); + if (keys_under_rollback_.count(entry.object_key) > 0) { + LOG(WARNING) << "Key is under rollback, skipping entry: " + << entry.sequence_id; + return false; // 暂时跳过，等待回滚完成 + } + } + + // 正常处理 + // ... +} +``` + +### 3. 性能优化 + +#### 3.1 异步回滚 + +**问题**：回滚和重放可能耗时，会阻塞新 OpLog 的处理 + +**解决方案**：异步执行回滚，不阻塞正常处理 + +```cpp +if (!CheckSequenceOrder(entry)) { + // 异步回滚（不阻塞） + std::thread([this, key = entry.object_key]() { + RollbackAndReplayKey(key); + }).detach(); + + return false; // 暂时跳过 +} +``` + +#### 3.2 批量读取 + +**问题**：从 etcd 读取大量 OpLog 可能较慢 + +**解决方案**：批量读取，减少网络往返 + +```cpp +bool ReadOpLogForKey(const std::string& key, + uint64_t start_seq_id, + std::vector& entries) { + const uint32_t batch_size = 10000; // 批量读取 + uint64_t current_seq_id = start_seq_id; + + while (true) { + std::vector batch; + etcd_oplog_store_->ReadOpLogSince(current_seq_id, batch_size, batch); + // ... + } +} +``` + +#### 3.3 限制回滚范围 + +**问题**：如果该 key 的操作非常多，回滚可能很耗时 + +**解决方案**：限制回滚范围，只回滚最近的操作 + +```cpp +bool RollbackAndReplayKey(const std::string& key) { + // 只回滚最近 N 个操作 + const uint64_t max_rollback_ops = 1000; + + // 从 etcd 读取时，限制范围 + uint64_t start_seq_id = std::max( + first_seq_id, + GetLatestSequenceId() - max_rollback_ops + ); + + // ... +} +``` + +### 4. 错误处理 + +#### 4.1 回滚失败 + +**场景**：从 etcd 读取 OpLog 失败 + +**处理**： +- 记录错误日志 +- 清除回滚标记 +- 可以考虑触发全量同步 + +```cpp +if (!ReadOpLogForKey(key, first_seq_id, key_entries)) { + LOG(ERROR) << "Failed to read OpLog for key: " << key; + + // 清除回滚标记 + { + std::lock_guard lock(rollback_mutex_); + keys_under_rollback_.erase(key); + } + + // 可选：触发全量同步 + // TriggerFullSync(); + + return false; +} +``` + +#### 4.2 重复回滚 + +**场景**：同一个 key 多次触发回滚 + +**处理**： +- 使用 `keys_under_rollback_` 防止并发回滚 +- 如果正在回滚，跳过新的回滚请求 + +### 5. 监控和告警 + +#### 5.1 记录乱序频率 + +```cpp +class OpLogApplier { +private: + // 记录每个 key 的乱序次数 + std::unordered_map key_violation_count_; + std::mutex violation_count_mutex_; + + // 乱序阈值 + static constexpr uint64_t kMaxViolationsPerKey = 10; + +public: + bool ApplyOpLogEntry(const OpLogEntry& entry) { + if (!CheckSequenceOrder(entry)) { + // 记录乱序次数 + { + std::lock_guard lock(violation_count_mutex_); + key_violation_count_[entry.object_key]++; + + if (key_violation_count_[entry.object_key] > kMaxViolationsPerKey) { + LOG(ERROR) << "Too many violations for key: " + << entry.object_key + << ", count: " + << key_violation_count_[entry.object_key]; + + // 触发全量同步 + TriggerFullSync(); + return false; + } + } + + // 触发回滚 + // ... + } + } +}; +``` + +#### 5.2 性能指标 + +- 回滚次数 +- 回滚耗时 +- 回滚成功率 +- 乱序频率 + +## 方案对比 + +| 特性 | 方案 A（基于 etcd） | 方案 B（基于历史） | +|------|-------------------|------------------| +| **数据准确性** | 高（从 etcd 读取） | 中（依赖历史记录） | +| **实现复杂度** | 低 | 高 | +| **内存开销** | 低 | 高（需要存储历史） | +| **性能** | 中（需要网络 I/O） | 高（内存操作） | +| **容错性** | 高（可以从 etcd 恢复） | 低（历史可能丢失） | +| **适用场景** | 乱序不频繁、数据准确性要求高 | 乱序频繁、性能要求高 | + +## 推荐方案 + +**推荐使用方案 A（基于 etcd 的完整重放）**，原因： + +1. **数据准确性高**：从 etcd 读取保证数据正确 +2. **实现简单**：不需要维护操作历史 +3. **内存友好**：不需要额外存储 +4. **容错性好**：即使本地状态丢失也能恢复 + +**优化建议**： +1. **异步回滚**：不阻塞正常处理 +2. **批量读取**：减少网络往返 +3. **限制范围**：只回滚最近的操作 +4. **监控告警**：记录乱序频率，超过阈值时触发全量同步 + +## 测试场景 + +### 1. 正常乱序检测和回滚 + +``` +1. Standby 收到 OpLog: sequence_id=100, key="obj1", key_sequence_id=5 +2. 应用成功 +3. Standby 收到 OpLog: sequence_id=102, key="obj1", key_sequence_id=6 +4. 应用成功 +5. Standby 收到 OpLog: sequence_id=101, key="obj1", key_sequence_id=5 +6. 检测到乱序，触发回滚 +7. 从 etcd 重新读取 obj1 的所有 OpLog +8. 按顺序重新应用 +9. 验证 metadata 正确 +``` + +### 2. 并发回滚保护 + +``` +1. 检测到 key="obj1" 乱序，开始回滚 +2. 回滚期间，收到新的 OpLog: key="obj1" +3. 检测到正在回滚，跳过新 OpLog +4. 回滚完成后，新的 OpLog 可以正常处理 +``` + +### 3. 回滚失败处理 + +``` +1. 检测到乱序，触发回滚 +2. 从 etcd 读取 OpLog 失败 +3. 记录错误日志 +4. 清除回滚标记 +5. 可选：触发全量同步 +``` + +### 4. 频繁乱序处理 + +``` +1. 某个 key 频繁乱序（超过阈值） +2. 记录告警 +3. 触发全量同步 +4. 避免频繁回滚影响性能 +``` + +## 总结 + +### 核心方案 + +**对于乱序的 key，执行回滚和重放**： +1. 回滚：删除该 key 的 metadata +2. 重放：从 etcd 重新读取该 key 的所有 OpLog +3. 重写：按正确顺序重新应用所有 OpLog + +### 关键实现 + +1. **回滚起点**：维护 `key_first_sequence_id_` 记录首次 sequence_id +2. **并发保护**：使用 `keys_under_rollback_` 防止并发回滚 +3. **异步执行**：回滚在后台线程执行，不阻塞正常处理 +4. **批量读取**：从 etcd 批量读取 OpLog，减少网络往返 +5. **监控告警**：记录乱序频率，超过阈值时触发全量同步 + +### 优势 + +1. **数据准确性**：从 etcd 读取保证数据正确 +2. **局部修复**：只影响乱序的 key，不影响其他 key +3. **自动恢复**：自动检测和修复数据不一致 +4. **性能友好**：异步执行，不阻塞正常处理 + +### 注意事项 + +1. **性能影响**：回滚和重放可能耗时，需要异步执行 +2. **并发处理**：回滚期间需要防止并发处理该 key +3. **范围限制**：可以限制回滚范围，只回滚最近的操作 +4. **监控告警**：需要监控乱序频率，超过阈值时考虑全量同步 + diff --git a/doc/zh/rfc-oplog-via-etcd-complete-design.md b/doc/zh/rfc-oplog-via-etcd-complete-design.md new file mode 100644 index 0000000000..58060f3098 --- /dev/null +++ b/doc/zh/rfc-oplog-via-etcd-complete-design.md @@ -0,0 +1,738 @@ +# 基于 etcd 的 OpLog 主备同步完整方案 + +## 方案概述 + +使用 etcd 作为中间可靠性组件，实现 Primary Master 和 Standby Master 之间的 OpLog 同步。OpLog 只记录 PUT 和 DELETE 事件，通过 etcd 的 Watch 机制实现实时同步。 + +## 核心设计原则 + +1. **OpLog 只记录 PUT 和 DELETE 事件**：不记录 LEASE_RENEW，减少 OpLog 大小 +2. **etcd 作为中间存储**：利用 etcd 的强一致性和 Watch 机制 +3. **快照集成**：与现有快照机制集成，快照后可以清理旧的 OpLog +4. **时序保证**：通过 sequence_id 和 key 级别的版本控制保证时序 + +## 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Primary Master │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ MasterService│ │ OpLogManager │ │ +│ │ │ │ │ │ +│ │ PutEnd() │─────▶│ Append() │ │ +│ │ Remove() │ │ │ │ +│ │ Eviction │ └──────────────┘ │ +│ └──────────────┘ │ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ EtcdOpLogStore│ │ +│ │ │ │ +│ │ WriteOpLog() │ │ +│ └──────────────┘ │ +│ │ │ +│ │ 写入 etcd │ +│ ▼ │ +│ ┌──────────────┐ │ +│ │ etcd │ │ +│ │ │ │ +│ │ /oplog/{seq} │ │ +│ └──────────────┘ │ +└─────────────────────────────────────────────────────────┘ + │ + │ Watch + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Standby Masters │ +│ │ +│ ┌──────────────────────────────────────────────────┐ │ +│ │ MasterServiceSupervisor │ │ +│ │ - 检测 leader │ │ +│ │ - 启动/停止 HotStandbyService │ │ +│ └──────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ OpLogWatcher │ │ OpLogApplier │ │ +│ │ │ │ │ │ +│ │ WatchEtcd() │─────▶│ ApplyOpLog() │ │ +│ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ │ +│ │ │ │ +│ │ ▼ │ +│ │ ┌──────────────┐ │ +│ │ │ MetadataStore│ │ +│ │ │ │ │ +│ │ │ 更新 metadata │ │ +│ │ └──────────────┘ │ +│ │ │ +│ └──────────────────────────────────────────────┘ +└─────────────────────────────────────────────────────────┘ +``` + +## etcd Key 设计 + +### 1. OpLog Entry Key + +``` +mooncake-store/oplog/{cluster_id}/{sequence_id} +``` + +**示例**： +``` +mooncake-store/oplog/mooncake_cluster/1 +mooncake-store/oplog/mooncake_cluster/2 +mooncake-store/oplog/mooncake_cluster/3 +... +``` + +**设计考虑**： +- 使用 `sequence_id` 作为 key 的一部分，保证顺序 +- 支持按 sequence_id 范围查询 +- 易于清理（删除指定 sequence_id 之前的 key） + +### 2. 最新 Sequence ID Key + +``` +mooncake-store/oplog/{cluster_id}/latest +``` + +**用途**： +- 存储当前最新的 sequence_id +- Standby 可以快速获取最新的 sequence_id +- 用于快照时记录 OpLog 的 sequence_id + +### 3. 快照 Sequence ID Key + +``` +mooncake-store/oplog/{cluster_id}/snapshot/{snapshot_id}/sequence_id +``` + +**用途**： +- 记录每个快照对应的 sequence_id +- 用于确定可以清理的 OpLog 范围 + +## OpLog Entry 数据结构 + +```cpp +struct OpLogEntry { + uint64_t sequence_id{0}; // 全局递增序列号 + uint64_t timestamp_ms{0}; // 时间戳（毫秒） + OpType op_type{OpType::PUT_END}; // PUT_END, PUT_REVOKE, REMOVE + std::string object_key; // 对象 key + std::string payload; // 可选负载（用于 PUT_END 时携带 replica 信息） + uint32_t checksum{0}; // 校验和 + uint32_t prefix_hash{0}; // key 前缀哈希 + uint64_t key_sequence_id{0}; // 该 key 的操作序列号（用于时序保证） +}; +``` + +**JSON 序列化格式**： +```json +{ + "sequence_id": 12345, + "timestamp": 1704110400123, + "op_type": "PUT_END", + "key": "object_key_123", + "payload": "optional_payload", + "checksum": 1234567890, + "prefix_hash": 987654321, + "key_sequence_id": 5 +} +``` + +## Primary 端实现 + +### 1. EtcdOpLogStore 类 + +```cpp +class EtcdOpLogStore { +public: + EtcdOpLogStore(const std::string& etcd_endpoints, + const std::string& cluster_id); + + // 写入 OpLog 到 etcd + bool WriteOpLog(const OpLogEntry& entry); + + // 批量写入 OpLog（可选优化） + bool WriteOpLogBatch(const std::vector& entries); + + // 更新最新的 sequence_id + bool UpdateLatestSequenceId(uint64_t sequence_id); + + // 记录快照对应的 sequence_id + bool RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + // 清理指定 sequence_id 之前的 OpLog + bool CleanupOpLogBefore(uint64_t sequence_id); + +private: + std::string BuildOpLogKey(uint64_t sequence_id); + std::string SerializeOpLogEntry(const OpLogEntry& entry); + OpLogEntry DeserializeOpLogEntry(const std::string& data); + + std::string etcd_prefix_; + std::string cluster_id_; + // etcd client +}; +``` + +### 2. 集成到 OpLogManager + +```cpp +class OpLogManager { +public: + // 设置 EtcdOpLogStore（可选，如果不设置则只写入内存） + void SetEtcdOpLogStore(EtcdOpLogStore* store); + + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()) { + OpLogEntry entry; + // ... 填充 entry ... + + // 写入内存 buffer + buffer_.emplace_back(entry); + + // 写入 etcd（如果设置了） + if (etcd_store_) { + etcd_store_->WriteOpLog(entry); + etcd_store_->UpdateLatestSequenceId(entry.sequence_id); + } + + return entry.sequence_id; + } + +private: + EtcdOpLogStore* etcd_store_{nullptr}; + // ... 其他成员 ... +}; +``` + +### 3. 驱逐时记录 DELETE 事件 + +```cpp +void MasterService::BatchEvict(...) { + // ... 驱逐逻辑 ... + + if (it->second.lease_timeout <= target_timeout) { + std::string evicted_key = it->first; + + // 驱逐对象 + total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + it->second.EraseReplica(ReplicaType::MEMORY); + + if (it->second.IsValid() == false) { + // 对象完全无效，记录 DELETE 事件 + AppendOpLogAndNotify(OpType::REMOVE, evicted_key); + it = shard.metadata.erase(it); + } else { + ++it; + } + } +} +``` + +## Standby 端实现 + +### 0. Standby 服务集成 + +**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 + +**核心流程**： +1. `MasterServiceSupervisor` 检测到有 leader 时，启动 `HotStandbyService` +2. `HotStandbyService` 启动 `OpLogWatcher` watch etcd OpLog +3. 实时应用 OpLog 到本地 metadata store +4. 选举成功后，停止 Standby 服务并提升为 Primary + +**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` + +### 1. OpLogWatcher 类 + +```cpp +class OpLogWatcher { +public: + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, + OpLogApplier* applier); + + // 启动 Watch + void Start(); + + // 停止 Watch + void Stop(); + + // 从指定 sequence_id 开始读取历史 OpLog + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); + +private: + // Watch etcd OpLog 变化 + void WatchOpLog(); + + // 处理 Watch 事件 + void HandleWatchEvent(const WatchEvent& event); + + std::string etcd_prefix_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + uint64_t last_processed_sequence_id_{0}; +}; +``` + +### 2. OpLogApplier 类（时序保证） + +```cpp +class OpLogApplier { +public: + OpLogApplier(MetadataStore* metadata_store); + + // 应用 OpLog Entry（带时序检查） + bool ApplyOpLogEntry(const OpLogEntry& entry); + + // 获取 key 的当前 sequence_id + uint64_t GetKeySequenceId(const std::string& key) const; + + // 恢复处理状态 + void Recover(uint64_t last_applied_sequence_id); + +private: + // 检查时序性 + bool CheckSequenceOrder(const OpLogEntry& entry); + + // 应用 PUT_END + void ApplyPutEnd(const OpLogEntry& entry); + + // 应用 PUT_REVOKE + void ApplyPutRevoke(const OpLogEntry& entry); + + // 应用 REMOVE + void ApplyRemove(const OpLogEntry& entry); + + MetadataStore* metadata_store_; + + // 记录每个 key 的最后 sequence_id（用于时序检查） + std::unordered_map key_sequence_map_; + std::mutex key_sequence_mutex_; + + // 记录待处理的条目（用于处理序列号不连续的情况） + std::map pending_entries_; + uint64_t expected_sequence_id_{1}; + std::mutex pending_mutex_; +}; +``` + +### 3. 时序保证机制 + +```cpp +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // 1. 检查全局序列号连续性 + if (entry.sequence_id != expected_sequence_id_) { + if (entry.sequence_id > expected_sequence_id_) { + // 序列号不连续，缓存待处理 + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + + // 等待一段时间，看是否有缺失的条目到达 + ScheduleWaitForMissingEntries(entry.sequence_id); + return false; + } else { + // 序列号小于期望值（可能是重复或乱序） + LOG(WARNING) << "Received out-of-order OpLog entry: " + << "expected=" << expected_sequence_id_ + << ", received=" << entry.sequence_id; + return false; + } + } + + // 2. 检查 key 级别的时序性 + if (!CheckSequenceOrder(entry)) { + LOG(ERROR) << "Key-level sequence order violation for key: " + << entry.object_key + << ", entry_seq=" << entry.key_sequence_id + << ", current_seq=" << GetKeySequenceId(entry.object_key); + + // 触发回滚和重放（异步执行） + // 详细设计请参考：doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md + RollbackAndReplayKey(entry.object_key); + return false; + } + + // 3. 应用 OpLog + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(WARNING) << "Unknown OpType: " + << static_cast(entry.op_type); + return false; + } + + // 4. 更新状态 + { + std::lock_guard lock(key_sequence_mutex_); + key_sequence_map_[entry.object_key] = entry.key_sequence_id; + } + + expected_sequence_id_++; + + // 5. 处理待处理的条目 + ProcessPendingEntries(); + + return true; +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + std::lock_guard lock(key_sequence_mutex_); + + auto it = key_sequence_map_.find(entry.object_key); + if (it == key_sequence_map_.end()) { + // 新 key，允许 + return true; + } + + // 检查 key_sequence_id 是否递增 + if (entry.key_sequence_id <= it->second) { + // 序列号乱序，需要回滚和重放 + return false; + } + + return true; +} +``` + +### 4. 初始同步流程 + +```cpp +class StandbyInitialSync { +public: + void PerformInitialSync() { + // Step 1: 从 Primary 获取快照 + MetadataSnapshot snapshot = RequestSnapshotFromPrimary(); + + // Step 2: 获取快照对应的 sequence_id + uint64_t snapshot_seq_id = snapshot.last_oplog_sequence_id; + + // Step 3: 应用快照 + metadata_store_->ImportSnapshot(snapshot); + + // Step 4: 从 etcd 读取快照后的 OpLog + std::vector entries; + op_log_watcher_->ReadOpLogSince(snapshot_seq_id + 1, entries); + + // Step 5: 应用历史 OpLog + for (const auto& entry : entries) { + applier_->ApplyOpLogEntry(entry); + } + + // Step 6: 开始 Watch 增量 OpLog + op_log_watcher_->Start(); + } +}; +``` + +## 快照集成 + +### 1. 快照时记录 Sequence ID + +```cpp +class SnapshotManager { +public: + MetadataSnapshot CreateSnapshot() { + MetadataSnapshot snapshot; + + // 1. 导出 metadata + snapshot.metadata = ExportMetadata(); + + // 2. 记录当前的 OpLog sequence_id + snapshot.last_oplog_sequence_id = oplog_manager_->GetLastSequenceId(); + + // 3. 将快照信息写入 etcd + std::string snapshot_id = GenerateSnapshotId(); + etcd_oplog_store_->RecordSnapshotSequenceId( + snapshot_id, snapshot.last_oplog_sequence_id); + + // 4. 清理旧的 OpLog + etcd_oplog_store_->CleanupOpLogBefore( + snapshot.last_oplog_sequence_id); + + return snapshot; + } +}; +``` + +### 2. OpLog 清理策略 + +**方案：从 etcd 查询最小的 sequence_id，然后使用 DeleteRange 删除** + +```cpp +bool EtcdOpLogStore::CleanupOpLogBefore(uint64_t target_sequence_id) { + if (target_sequence_id <= 1) { + return true; // 没有需要清理的 + } + + // 1. 从 etcd 查询最小的 sequence_id + uint64_t min_seq_id = GetMinSequenceId(); + + // 2. 如果 min_seq_id >= target_sequence_id，无需清理 + if (min_seq_id >= target_sequence_id) { + return true; + } + + // 3. 执行 DeleteRange + std::string start_key = BuildOpLogKey(min_seq_id); + std::string end_key = BuildOpLogKey(target_sequence_id); + + int64_t deleted_count = 0; + auto err = EtcdHelper::DeleteRange( + start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size(), + deleted_count); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cleanup OpLog"; + return false; + } + + LOG(INFO) << "Cleaned up " << deleted_count + << " OpLog entries from " << min_seq_id + << " to " << target_sequence_id; + return true; +} + +uint64_t EtcdOpLogStore::GetMinSequenceId() const { + // 从 etcd 查询最小的 OpLog sequence_id + std::string prefix = etcd_prefix_ + "/" + cluster_id_ + "/"; + std::string first_key, first_value; + + auto err = EtcdHelper::GetFirstKeyWithPrefix( + prefix.c_str(), prefix.size(), + first_key, first_value); + + if (err == ErrorCode::OK) { + // 从 key 中提取 sequence_id + uint64_t min_seq_id = ExtractSequenceIdFromKey(first_key); + if (min_seq_id > 0) { + return min_seq_id; + } + } + + // Fallback：从快照记录获取 + uint64_t last_snapshot_seq_id = GetLastSnapshotSequenceId(); + if (last_snapshot_seq_id > 0) { + return last_snapshot_seq_id; + } + + // 保守策略：从 1 开始 + return 1; +} +``` + +**详细实现请参考：`doc/zh/rfc-oplog-cleanup-start-sequence-id.md`** + +## 时序保证机制详解 + +### 1. 全局序列号（sequence_id） + +- **作用**：保证所有 OpLog 事件的全局顺序 +- **生成**：Primary 端 OpLogManager 全局递增 +- **检查**：Standby 端检查 sequence_id 是否连续 + +### 2. Key 级别序列号（key_sequence_id） + +- **作用**：保证同一个 key 的操作顺序 +- **生成**：Primary 端为每个 key 维护独立的序列号 +- **检查**：Standby 端检查 key_sequence_id 是否递增 + +### 3. 序列号不连续处理 + +```cpp +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq) { + // 等待一段时间（如 1 秒） + std::this_thread::sleep_for(std::chrono::seconds(1)); + + // 如果缺失的条目仍未到达，需要从 etcd 读取 + if (pending_entries_.find(missing_seq) == pending_entries_.end()) { + RequestMissingOpLog(missing_seq); + } +} + +void OpLogApplier::RequestMissingOpLog(uint64_t missing_seq) { + // 从 etcd 读取缺失的 OpLog + OpLogEntry entry; + if (ReadOpLogFromEtcd(missing_seq, entry)) { + ApplyOpLogEntry(entry); + } else { + LOG(ERROR) << "Failed to read missing OpLog: seq=" << missing_seq; + // 触发重新同步 + TriggerResync(); + } +} +``` + +## 实现步骤 + +### Phase 1：基础框架（优先级：高） + +1. **实现 EtcdOpLogStore** + - 写入 OpLog 到 etcd + - 更新最新 sequence_id + - 读取 OpLog 从 etcd + +2. **集成到 OpLogManager** + - 添加 EtcdOpLogStore 成员 + - 在 Append 时写入 etcd + +3. **实现 OpLogWatcher** + - Watch etcd OpLog 变化 + - 处理 Watch 事件 + +### Phase 2：Standby 端处理（优先级：高） + +1. **实现 OpLogApplier** + - 应用 OpLog Entry + - 时序检查逻辑 + - 处理序列号不连续 + +2. **实现初始同步** + - 从 Primary 获取快照 + - 读取历史 OpLog + - 应用快照和 OpLog + +### Phase 3：快照集成（优先级：中） + +1. **快照时记录 sequence_id** + - 在快照中记录 last_oplog_sequence_id + - 写入 etcd + +2. **OpLog 清理** + - 实现 CleanupOpLogBefore + - 定期清理旧的 OpLog + +### Phase 4：优化（优先级：低） + +1. **批量写入** + - 实现 WriteOpLogBatch + - 减少 etcd 写入次数 + +2. **压缩** + - OpLog Entry 压缩 + - 减少 etcd 存储大小 + +## 关键设计要点 + +### 1. etcd Key 设计 + +- 使用顺序 Key：`mooncake-store/oplog/{cluster_id}/{sequence_id}` +- 支持按 sequence_id 范围查询 +- 易于清理（删除指定 sequence_id 之前的 key） + +### 2. 时序保证 + +- **全局序列号**：保证所有事件的全局顺序 +- **Key 级别序列号**：保证同一 key 的操作顺序 +- **序列号不连续处理**：检测并处理序列号不连续的情况 +- **序列号乱序处理**：检测到 key 级别乱序时，执行回滚和重放（详细设计请参考：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md`） + +### 3. 快照集成 + +- 快照时记录 sequence_id +- 快照后清理旧的 OpLog +- Standby 从快照点开始应用增量 OpLog + +### 4. 故障恢复 + +- Standby 持久化处理状态 +- 支持断点续传 +- 发现不一致时触发重新同步 + +### 5. Standby 服务集成 + +**问题**：现有代码中，Standby 在等待 leader 选举期间只是阻塞等待，没有运行 Standby 服务来同步 OpLog。 + +**解决方案**：在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata。 + +**详细设计请参考**：`doc/zh/rfc-standby-service-integration.md` + +### 6. Standby 提升为 Primary 时的 Lease 初始化 + +**问题**：Standby 上的对象 lease 都是 0（因为 OpLog 只包含 PUT_END，不包含续约信息），提升为 Primary 后所有对象会立即过期。 + +**解决方案**：在 `Promote()` 时，给所有 lease 为 0 的对象授予默认租约时间（`default_kv_lease_ttl`）。 + +**详细设计请参考**：`doc/zh/rfc-standby-promotion-lease-initialization.md` + +### 7. 序列号乱序时的回滚和重放 + +**问题**：当检测到某个 key 的 `key_sequence_id` 乱序时，该 key 的 metadata 可能已经不一致。 + +**解决方案**：对于乱序的 key，执行回滚和重放： +1. **回滚**：从 metadata_store 中删除该 key 的所有状态 +2. **重放**：从该 key 第一次出现的 sequence_id 开始，从 etcd 重新读取所有 OpLog +3. **重写**：按正确顺序重新应用所有 OpLog，重建 metadata + +**关键设计**： +- 异步执行回滚，不阻塞正常处理 +- 使用 `keys_under_rollback_` 防止并发回滚 +- 从 etcd 批量读取 OpLog，减少网络往返 +- 监控乱序频率，超过阈值时触发全量同步 + +**详细设计请参考**：`doc/zh/rfc-oplog-rollback-replay-on-sequence-violation.md` + +### 8. key_sequence_map_ 内存清理策略 + +**问题**：Standby 端的 `OpLogApplier` 中，`key_sequence_map_` 用于跟踪每个 key 的 `key_sequence_id`。当 metadata 被删除后，这些条目仍然保留用于乱序检测，长期运行可能导致内存泄漏。 + +**解决方案**：实现定期清理机制： +1. **清理条件**：最后一次操作是 `REMOVE` 且距离当前超过 1 小时 +2. **清理频率**：每小时扫描一次 +3. **保留策略**：`PUT_END` 和 `PUT_REVOKE` 操作的 key 不清理（metadata 可能仍存在） + +**关键设计**： +- 在 `ApplyOpLogEntry` 中触发清理检查，无需额外线程 +- 只清理 `REMOVE` 操作且超过 1 小时的条目 +- 1 小时的时间窗口足够处理网络延迟、重传等异常情况 +- 有效控制内存占用，从潜在的 90MB+ 降低到约 `(活跃key数量 + 1万) × 90字节` + +**详细设计请参考**：`doc/zh/rfc-oplog-key-sequence-map-cleanup.md` + +## 与现有方案对比 + +| 特性 | 当前方案（gRPC 推送） | etcd Watch 方案 | +|------|----------------------|----------------| +| **时序保证** | 依赖网络顺序 | etcd 保证顺序 | +| **可靠性** | 需要 ACK 机制 | etcd 保证可靠性 | +| **断点续传** | 需要实现 | etcd 原生支持 | +| **数据持久化** | 需要额外实现 | etcd 自动持久化 | +| **快照集成** | 需要额外实现 | 易于集成 | +| **实现复杂度** | 高 | 中等 | + +## 实施计划 + +详细的实施计划、优先级和时间估算请参考：`doc/zh/rfc-oplog-implementation-plan.md` + +**实施阶段总览**： +- **Phase 1**：基础框架（P0，2-3 周） +- **Phase 2**：Standby 服务集成（P0，2-3 周） +- **Phase 3**：时序保证和容错（P1，2-3 周） +- **Phase 4**：快照集成和清理（P2，1-2 周） +- **Phase 5**：优化和完善（P3，1-2 周） + +**总计**：8-13 周（约 2-3 个月） + +## 总结 + +本方案利用 etcd 的强一致性和 Watch 机制，实现了可靠的 OpLog 同步。通过只记录 PUT 和 DELETE 事件，大幅减少了 OpLog 大小。通过全局和 key 级别的序列号，保证了时序性。通过与快照机制集成，实现了高效的 OpLog 清理。 + diff --git a/doc/zh/rfc-standby-no-response-handling.md b/doc/zh/rfc-standby-no-response-handling.md new file mode 100644 index 0000000000..a46ab68d8d --- /dev/null +++ b/doc/zh/rfc-standby-no-response-handling.md @@ -0,0 +1,355 @@ +# Standby Master 无响应处理方案 + +## 问题分析 + +当 OpLog 从 Primary Master 同步到 Standby Master 时，如果 Standby 一直不响应，会导致以下问题： + +### 1. **内存压力** +- `OpLogManager` 的 buffer 有上限（`kMaxBufferEntries_ = 100000`），但即使有上限，也可能导致： + - 内存占用持续增长 + - 无法及时 truncate，导致 buffer 长期占用 + - 如果多个 Standby 都无响应，问题会放大 + +### 2. **数据丢失风险** +- 如果 buffer 满了，最老的 OpLog 会被丢弃（`pop_front()`） +- 如果 Standby 后来恢复，可能无法完整同步历史数据 + +### 3. **性能影响** +- 持续尝试发送失败的消息会消耗 CPU +- 阻塞其他正常 Standby 的同步（如果实现不当） + +### 4. **故障检测缺失** +- 当前实现无法区分： + - **网络分区**：Standby 节点正常，但网络不通 + - **节点故障**：Standby 节点宕机 + - **处理慢**：Standby 节点正常，但处理速度慢 + +## 解决方案设计 + +### 方案 1: 超时检测 + 故障隔离（推荐） + +#### 1.1 添加超时检测机制 + +```cpp +struct StandbyState { + std::shared_ptr stream; + uint64_t acked_seq_id{0}; + std::chrono::steady_clock::time_point last_ack_time; + std::chrono::steady_clock::time_point last_send_time; // 新增 + std::vector pending_batch; + + // 新增：超时和重试状态 + enum class State { + HEALTHY, // 正常状态 + SLOW, // 响应慢，但还在处理 + TIMEOUT, // 超时，可能故障 + DISCONNECTED // 已断开连接 + }; + State state{State::HEALTHY}; + uint32_t consecutive_failures{0}; // 连续失败次数 +}; +``` + +#### 1.2 实现超时检测逻辑 + +```cpp +class ReplicationService { +private: + // 配置参数 + static constexpr uint32_t kAckTimeoutMs = 5000; // ACK 超时时间（5秒） + static constexpr uint32_t kSendTimeoutMs = 3000; // 发送超时时间（3秒） + static constexpr uint32_t kMaxConsecutiveFailures = 3; // 最大连续失败次数 + static constexpr uint32_t kHealthCheckIntervalMs = 1000; // 健康检查间隔（1秒） + + // 定期检查 Standby 健康状态 + void CheckStandbyHealth(); + + // 标记 Standby 为故障状态 + void MarkStandbyUnhealthy(const std::string& standby_id); + + // 尝试恢复 Standby 连接 + void TryRecoverStandby(const std::string& standby_id); +}; +``` + +#### 1.3 故障隔离策略 + +**策略 A: 暂停发送（推荐）** +- 当 Standby 超时或连续失败时，暂停向该 Standby 发送新的 OpLog +- 继续向其他健康的 Standby 发送 +- 保留该 Standby 的 `acked_seq_id`，等待恢复后从断点继续 + +**策略 B: 降级处理** +- 将 Standby 标记为 `SLOW` 状态 +- 降低发送频率（例如：每 10 个 OpLog 发送一次） +- 如果持续超时，再升级为 `TIMEOUT` 状态 + +#### 1.4 实现示例 + +```cpp +void ReplicationService::CheckStandbyHealth() { + std::unique_lock lock(mutex_); + auto now = std::chrono::steady_clock::now(); + + for (auto& [standby_id, state] : standbys_) { + // 检查连接状态 + if (!state.stream || !state.stream->IsConnected()) { + state.state = StandbyState::State::DISCONNECTED; + continue; + } + + // 检查 ACK 超时 + auto ack_age = std::chrono::duration_cast( + now - state.last_ack_time).count(); + + if (ack_age > kAckTimeoutMs) { + state.consecutive_failures++; + + if (state.consecutive_failures >= kMaxConsecutiveFailures) { + state.state = StandbyState::State::TIMEOUT; + LOG(WARNING) << "Standby " << standby_id + << " marked as TIMEOUT after " + << state.consecutive_failures << " failures"; + // 暂停向该 Standby 发送 + } else { + state.state = StandbyState::State::SLOW; + LOG(WARNING) << "Standby " << standby_id + << " is slow (ack_age=" << ack_age << "ms)"; + } + } else { + // 恢复正常 + if (state.state != StandbyState::State::HEALTHY) { + LOG(INFO) << "Standby " << standby_id << " recovered"; + state.state = StandbyState::State::HEALTHY; + state.consecutive_failures = 0; + } + } + } +} + +void ReplicationService::BroadcastEntry(const OpLogEntry& entry) { + std::shared_lock lock(mutex_); + + for (auto& [standby_id, state] : standbys_) { + // 跳过故障的 Standby + if (state.state == StandbyState::State::TIMEOUT || + state.state == StandbyState::State::DISCONNECTED) { + continue; + } + + state.pending_batch.push_back(entry); + + if (state.pending_batch.size() >= kBatchSize) { + SendBatch(standby_id, state.pending_batch); + state.pending_batch.clear(); + } + } +} +``` + +### 方案 2: 真正的 ACK 机制 + +当前实现中，`acked_seq_id` 的更新是假设 `Send()` 成功就更新，这是不正确的。应该： + +1. **发送时记录待确认的序列号** +2. **等待 Standby 的 ACK 响应** +3. **只有收到 ACK 后才更新 `acked_seq_id`** + +```cpp +struct StandbyState { + // ... + std::map pending_acks; // seq_id -> send_time + uint64_t last_sent_seq_id{0}; // 最后发送的序列号 +}; + +void ReplicationService::SendBatch(const std::string& standby_id, + const std::vector& entries) { + // ... 发送逻辑 ... + + if (success && !entries.empty()) { + uint64_t last_seq = entries.back().sequence_id; + state.last_sent_seq_id = last_seq; + // 记录待确认的序列号 + state.pending_acks[last_seq] = std::chrono::steady_clock::now(); + // 注意：这里不更新 acked_seq_id，等收到 ACK 再更新 + } +} + +// 处理 Standby 的 ACK 响应 +void ReplicationService::OnAck(const std::string& standby_id, uint64_t acked_seq_id) { + std::unique_lock lock(mutex_); + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + return; + } + + auto& state = it->second; + if (acked_seq_id > state.acked_seq_id) { + state.acked_seq_id = acked_seq_id; + state.last_ack_time = std::chrono::steady_clock::now(); + state.consecutive_failures = 0; // 重置失败计数 + + // 清理已确认的 pending_acks + auto ack_it = state.pending_acks.begin(); + while (ack_it != state.pending_acks.end()) { + if (ack_it->first <= acked_seq_id) { + ack_it = state.pending_acks.erase(ack_it); + } else { + ++ack_it; + } + } + } +} +``` + +### 方案 3: 流控（Backpressure）机制 + +如果 Standby 处理慢，应该限制发送速度，避免 Standby 内存溢出： + +```cpp +struct StandbyState { + // ... + size_t in_flight_bytes{0}; // 正在传输的字节数 + size_t max_in_flight_bytes{10 * 1024 * 1024}; // 最大 10MB + uint32_t pending_batch_count{0}; // 待确认的批次数量 + uint32_t max_pending_batches{10}; // 最大待确认批次 +}; + +bool ReplicationService::CanSendToStandby(const StandbyState& state) const { + // 检查流控条件 + if (state.in_flight_bytes >= state.max_in_flight_bytes) { + return false; // 超过流量限制 + } + if (state.pending_batch_count >= state.max_pending_batches) { + return false; // 超过批次限制 + } + return true; +} +``` + +### 方案 4: OpLog Truncate 策略 + +只有当**所有健康的 Standby** 都 ACK 了某个序列号后，才能安全地 truncate： + +```cpp +uint64_t ReplicationService::GetMinAckedSequenceId() const { + std::shared_lock lock(mutex_); + + if (standbys_.empty()) { + // 没有 Standby，可以 truncate 所有 + return oplog_manager_.GetLastSequenceId(); + } + + uint64_t min_acked = UINT64_MAX; + for (const auto& [standby_id, state] : standbys_) { + // 只考虑健康的 Standby + if (state.state == StandbyState::State::HEALTHY || + state.state == StandbyState::State::SLOW) { + min_acked = std::min(min_acked, state.acked_seq_id); + } + } + + return (min_acked == UINT64_MAX) ? 0 : min_acked; +} + +// 定期调用，清理已确认的 OpLog +void ReplicationService::TruncateOpLog() { + uint64_t min_acked = GetMinAckedSequenceId(); + if (min_acked > 0) { + oplog_manager_.TruncateBefore(min_acked); + } +} +``` + +### 方案 5: 重连和恢复机制 + +当 Standby 恢复后，应该能够从断点继续同步： + +```cpp +void ReplicationService::TryRecoverStandby(const std::string& standby_id) { + std::unique_lock lock(mutex_); + auto it = standbys_.find(standby_id); + if (it == standbys_.end()) { + return; + } + + auto& state = it->second; + + // 检查连接是否恢复 + if (state.stream && state.stream->IsConnected()) { + // 从上次 ACK 的位置开始重新发送 + uint64_t start_seq = state.acked_seq_id + 1; + auto entries = oplog_manager_.GetEntriesSince(start_seq, 1000); + + if (!entries.empty()) { + LOG(INFO) << "Recovering Standby " << standby_id + << " from seq_id=" << start_seq + << ", entries=" << entries.size(); + SendBatch(standby_id, entries); + state.state = StandbyState::State::HEALTHY; + } + } +} +``` + +## 实施优先级 + +### Phase 1: 基础超时检测（必须） +1. 添加 `StandbyState::State` 枚举 +2. 实现 `CheckStandbyHealth()` 定期检查 +3. 在 `BroadcastEntry()` 中跳过故障 Standby +4. 添加配置参数（超时时间、最大失败次数） + +### Phase 2: 真正的 ACK 机制（重要） +1. 修改 `SendBatch()` 不立即更新 `acked_seq_id` +2. 添加 `OnAck()` 方法处理 ACK 响应 +3. 实现 `pending_acks` 跟踪机制 + +### Phase 3: 流控和 Truncate（优化） +1. 实现流控机制 +2. 实现安全的 OpLog truncate +3. 添加监控指标（replication lag、failure rate） + +### Phase 4: 恢复机制（完善） +1. 实现重连检测 +2. 实现断点续传 +3. 添加恢复日志 + +## 配置参数建议 + +```cpp +struct ReplicationConfig { + uint32_t ack_timeout_ms = 5000; // ACK 超时时间 + uint32_t send_timeout_ms = 3000; // 发送超时时间 + uint32_t max_consecutive_failures = 3; // 最大连续失败次数 + uint32_t health_check_interval_ms = 1000; // 健康检查间隔 + size_t max_in_flight_bytes = 10 * 1024 * 1024; // 最大传输字节数 + uint32_t max_pending_batches = 10; // 最大待确认批次 + bool enable_backpressure = true; // 是否启用流控 +}; +``` + +## 监控指标 + +建议添加以下监控指标： + +1. **Replication Lag**: 每个 Standby 的延迟（`primary_seq_id - acked_seq_id`） +2. **Failure Rate**: Standby 的失败率 +3. **Timeout Count**: 超时次数 +4. **Recovery Count**: 恢复次数 +5. **OpLog Buffer Size**: OpLog buffer 当前大小 +6. **Truncate Rate**: OpLog truncate 频率 + +## 总结 + +Standby 无响应是一个复杂的分布式系统问题，需要多层次的解决方案： + +1. **超时检测**：及时发现故障 +2. **故障隔离**：避免影响其他 Standby +3. **真正的 ACK**：准确跟踪同步进度 +4. **流控**：保护 Standby 不被压垮 +5. **安全 Truncate**：避免数据丢失 +6. **恢复机制**：支持断点续传 + +建议先实施 Phase 1 和 Phase 2，这两个是最关键的。 + diff --git a/doc/zh/rfc-standby-promotion-lease-initialization.md b/doc/zh/rfc-standby-promotion-lease-initialization.md new file mode 100644 index 0000000000..b10daa8921 --- /dev/null +++ b/doc/zh/rfc-standby-promotion-lease-initialization.md @@ -0,0 +1,439 @@ +# Standby 提升为 Primary 时的 Lease 初始化方案 + +## 问题描述 + +当 Standby Master 被提升为 Primary Master 时，存在一个关键问题：**所有对象的 lease 都是 0（已过期）**。 + +### 问题根源 + +1. **OpLog 中只包含 PUT 和 DELETE 事件** + - `PUT_END` 事件：在 Primary 上创建对象时，`lease_timeout` 被初始化为 0（立即过期） + - `DELETE` 事件：删除对象 + - **不包含** `LEASE_RENEW` 事件（已从 OpLog 中移除） + +2. **Standby 上的对象状态** + - Standby 从 Primary 同步 OpLog，只收到 `PUT_END` 事件 + - 因此 Standby 上所有对象的 `lease_timeout` 都是 0（epoch） + - Standby 不执行驱逐，所以不会检查 lease 是否过期 + +3. **提升为 Primary 后的影响** + - 新 Primary 开始执行驱逐逻辑 + - 由于所有对象的 `lease_timeout` 都是 0，所有对象都会立即被判定为过期 + - 这会导致所有对象被立即驱逐，系统无法正常工作 + +### 问题场景示例 + +``` +时间线： +1. Primary: PutEnd(key="obj1") → lease_timeout = 0 +2. Primary: ExistKey(key="obj1") → lease_timeout = now + 5s (续约) +3. Standby: 同步 PUT_END 事件 → lease_timeout = 0 (没有续约信息) +4. Primary 崩溃 +5. Standby 提升为 Primary +6. 新 Primary: 执行驱逐 → 所有对象 lease_timeout = 0 → 全部被驱逐 ❌ +``` + +## 解决方案 + +### 方案：在提升时给所有对象授予默认租约 + +**核心思路**：当 Standby 被提升为 Primary 时，遍历所有 metadata，给每个对象授予一个默认的租约时间。 + +### 实现设计 + +#### 1. 在 `HotStandbyService::Promote()` 中添加 Lease 初始化逻辑 + +```cpp +std::unique_ptr HotStandbyService::Promote() { + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion"; + return nullptr; + } + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << applied_seq_id_.load(); + + // Stop replication + Stop(); + + // 1. 创建新的 MasterService 实例 + auto master_service = std::make_unique(/* config */); + + // 2. 从 metadata_store_ 恢复 metadata 到新的 MasterService + RestoreMetadataToMasterService(*master_service); + + // 3. 【关键】给所有对象授予默认租约 + InitializeLeasesForAllObjects(*master_service); + + // 4. 执行一次完整的驱逐清理（清理真正过期的对象） + PerformFullEvictionCleanup(*master_service); + + LOG(INFO) << "Standby promoted to Primary successfully"; + return master_service; +} +``` + +#### 2. 实现 `InitializeLeasesForAllObjects()` + +```cpp +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + LOG(INFO) << "Initializing leases for all objects after promotion"; + + uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); + uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); + + size_t initialized_count = 0; + + // 遍历所有 shard 中的所有 metadata + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + // 检查 lease 是否过期（lease_timeout = 0 表示过期） + if (metadata.IsLeaseExpired()) { + // 授予默认租约 + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); + initialized_count++; + + VLOG(2) << "Initialized lease for key: " << key + << ", lease_ttl=" << default_lease_ttl; + } + } + } + + LOG(INFO) << "Initialized leases for " << initialized_count + << " objects after promotion"; +} +``` + +#### 3. 实现 `PerformFullEvictionCleanup()` + +```cpp +void HotStandbyService::PerformFullEvictionCleanup(MasterService& master_service) { + LOG(INFO) << "Performing full eviction cleanup after promotion"; + + // 执行一次完整的驱逐，清理真正过期的对象 + // 注意：此时所有对象的 lease 都已经初始化，只有真正过期的对象才会被驱逐 + master_service.BatchEvict(); + + LOG(INFO) << "Full eviction cleanup completed"; +} +``` + +### 关键设计点 + +#### 1. 默认租约时间的选择 + +**选项 A：使用配置的 `default_kv_lease_ttl`** +- **优点**：简单，与正常操作一致 +- **缺点**：可能给已经很久没有访问的对象也授予租约，导致内存浪费 + +**选项 B：使用较短的租约时间（如 1-2 秒）** +- **优点**：快速淘汰真正不活跃的对象 +- **缺点**：可能误杀活跃对象 + +**推荐：选项 A（使用 `default_kv_lease_ttl`）** + +**理由**： +1. 保守策略，避免误杀活跃对象 +2. 如果对象真的不活跃，会在下次驱逐时被清理 +3. 与正常操作一致，行为可预测 + +#### 2. 何时执行 Lease 初始化 + +**时机**：在 `Promote()` 方法中，在恢复 metadata 之后、开始服务请求之前 + +**流程**： +``` +1. 停止 Standby 的复制循环 +2. 创建新的 MasterService 实例 +3. 恢复 metadata 到新的 MasterService +4. 【关键】初始化所有对象的 lease +5. 执行一次完整的驱逐清理 +6. 开始服务请求 +``` + +#### 3. 与驱逐清理的配合 + +**问题**：如果先初始化 lease，再执行驱逐，那么所有对象都有 lease，不会被驱逐？ + +**解答**： +- 初始化 lease 的目的是**防止误杀活跃对象** +- 驱逐清理的目的是**清理真正过期的对象**（基于 `put_start_time` 等条件） +- 实际上，在 Standby 提升时，所有对象都是"新"的（从 OpLog 恢复），所以应该都保留 +- 如果某些对象在 Primary 崩溃前就已经过期，那么它们应该已经被 Primary 驱逐并产生 DELETE 事件，Standby 上不应该有这些对象 + +**更准确的驱逐逻辑**： +- 在 Standby 提升时，不应该基于 lease 进行驱逐 +- 应该基于其他条件（如 `put_start_time` + `put_start_release_timeout_sec_`）进行清理 +- 或者，在提升时**不执行驱逐**，让正常的驱逐循环来处理 + +**修正后的方案**： + +```cpp +void HotStandbyService::Promote() { + // ... 前面的步骤 ... + + // 3. 给所有对象授予默认租约 + InitializeLeasesForAllObjects(*master_service); + + // 4. 【可选】执行一次清理，但只清理明显无效的对象 + // 注意：不基于 lease 进行驱逐，因为所有对象的 lease 都是 0 + // 可以清理：put_start_time 过期的对象、没有完整 replica 的对象等 + CleanupInvalidObjects(*master_service); + + // 5. 启动 MasterService 的驱逐循环 + // 正常的驱逐循环会基于 lease 和其他条件进行驱逐 +} +``` + +## 实现细节 + +### 1. 在 `MasterService` 中添加辅助方法 + +```cpp +class MasterService { +public: + // 获取默认租约 TTL + uint64_t GetDefaultLeaseTtl() const { return default_kv_lease_ttl_; } + + // 获取默认 Soft Pin TTL + uint64_t GetDefaultSoftPinTtl() const { return default_kv_soft_pin_ttl_; } + + // 获取 metadata shards（用于遍历） + std::vector& GetMetadataShards() { return metadata_shards_; } + + // ... 其他方法 ... +}; +``` + +### 2. 在 `HotStandbyService` 中实现 Lease 初始化 + +```cpp +class HotStandbyService { +private: + void InitializeLeasesForAllObjects(MasterService& master_service); + void CleanupInvalidObjects(MasterService& master_service); + + // ... 其他成员 ... +}; + +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + LOG(INFO) << "Initializing leases for all objects after promotion"; + + uint64_t default_lease_ttl = master_service.GetDefaultLeaseTtl(); + uint64_t default_soft_pin_ttl = master_service.GetDefaultSoftPinTtl(); + + size_t initialized_count = 0; + size_t skipped_count = 0; + + // 遍历所有 shard + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + // 只初始化 lease 为 0 的对象 + if (metadata.IsLeaseExpired()) { + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); + initialized_count++; + } else { + // 如果 lease 已经有效，说明可能是从快照恢复的，保留原值 + skipped_count++; + } + } + } + + LOG(INFO) << "Lease initialization completed: " + << initialized_count << " objects initialized, " + << skipped_count << " objects skipped"; +} +``` + +### 3. 清理无效对象（可选） + +```cpp +void HotStandbyService::CleanupInvalidObjects(MasterService& master_service) { + LOG(INFO) << "Cleaning up invalid objects after promotion"; + + size_t cleaned_count = 0; + auto now = std::chrono::steady_clock::now(); + + // 遍历所有 shard + for (auto& shard : master_service.GetMetadataShards()) { + std::unique_lock lock(shard.mutex); + + auto it = shard.metadata.begin(); + while (it != shard.metadata.end()) { + auto& [key, metadata] = *it; + + // 清理条件： + // 1. put_start_time 过期且没有完整 replica + // 2. 所有 replica 都无效 + bool should_cleanup = false; + + if (!metadata.HasCompletedReplicas() && + metadata.put_start_time + + master_service.GetPutStartReleaseTimeout() < now) { + should_cleanup = true; + } else if (!metadata.IsValid()) { + should_cleanup = true; + } + + if (should_cleanup) { + VLOG(1) << "Cleaning up invalid object: " << key; + it = shard.metadata.erase(it); + cleaned_count++; + } else { + ++it; + } + } + } + + LOG(INFO) << "Cleaned up " << cleaned_count << " invalid objects"; +} +``` + +## 边界情况处理 + +### 1. 从快照恢复的场景 + +**场景**：Standby 从快照恢复，快照中可能包含 lease 信息 + +**处理**： +- 如果快照中包含 lease 信息，保留原值 +- 如果快照中 lease 为 0，则初始化 + +**实现**： +```cpp +if (metadata.IsLeaseExpired()) { + // lease 为 0，需要初始化 + metadata.GrantLease(default_lease_ttl, default_soft_pin_ttl); +} else { + // lease 已有效，可能是从快照恢复的，保留原值 + skipped_count++; +} +``` + +### 2. 提升过程中的并发访问 + +**场景**：提升过程中，可能有其他线程访问 metadata + +**处理**： +- 使用 `std::unique_lock` 保护每个 shard +- 提升过程应该是原子的（停止 Standby，创建 Primary） + +### 3. 提升失败的处理 + +**场景**：提升过程中发生错误 + +**处理**： +- 记录错误日志 +- 返回 `nullptr`，表示提升失败 +- Standby 继续运行，等待下次提升机会 + +## 性能考虑 + +### 1. 遍历所有对象的开销 + +**影响**： +- 如果对象数量很大（如 100 万），遍历所有对象可能需要几秒 + +**优化**： +- 使用多线程并行处理不同 shard +- 批量处理，减少锁竞争 + +**实现**： +```cpp +void HotStandbyService::InitializeLeasesForAllObjects(MasterService& master_service) { + auto& shards = master_service.GetMetadataShards(); + + // 并行处理所有 shard + std::vector threads; + for (size_t i = 0; i < shards.size(); ++i) { + threads.emplace_back([&shards, i, &master_service]() { + auto& shard = shards[i]; + std::unique_lock lock(shard.mutex); + + for (auto& [key, metadata] : shard.metadata) { + if (metadata.IsLeaseExpired()) { + metadata.GrantLease( + master_service.GetDefaultLeaseTtl(), + master_service.GetDefaultSoftPinTtl()); + } + } + }); + } + + for (auto& t : threads) { + t.join(); + } +} +``` + +### 2. 提升时间窗口 + +**影响**： +- 提升过程需要时间，期间系统不可用 + +**优化**： +- 尽量减少提升时间 +- 可以考虑在 Standby 阶段就预先初始化 lease（但这样 Standby 也需要维护 lease） + +## 测试场景 + +### 1. 正常提升场景 + +``` +1. Standby 同步了 1000 个对象的 PUT_END 事件 +2. 所有对象的 lease_timeout = 0 +3. Primary 崩溃 +4. Standby 提升为 Primary +5. 验证：所有对象的 lease_timeout > now +6. 验证：系统可以正常服务请求 +``` + +### 2. 从快照恢复的场景 + +``` +1. Standby 从快照恢复，快照中包含 lease 信息 +2. 部分对象的 lease_timeout > 0（从快照恢复） +3. 部分对象的 lease_timeout = 0（新同步的） +4. Standby 提升为 Primary +5. 验证：lease_timeout = 0 的对象被初始化 +6. 验证：lease_timeout > 0 的对象保留原值 +``` + +### 3. 大量对象的场景 + +``` +1. Standby 同步了 100 万个对象 +2. Standby 提升为 Primary +3. 验证：所有对象的 lease 都被初始化 +4. 验证：提升时间在可接受范围内（< 10 秒） +``` + +## 总结 + +### 核心方案 + +**在 Standby 提升为 Primary 时，给所有 lease 为 0 的对象授予默认租约时间** + +### 关键点 + +1. **时机**：在 `Promote()` 中，恢复 metadata 之后、开始服务之前 +2. **租约时间**：使用 `default_kv_lease_ttl`（保守策略） +3. **清理**：可选，清理明显无效的对象（不基于 lease） +4. **性能**：并行处理多个 shard，减少提升时间 + +### 优势 + +1. **简单可靠**：逻辑清晰，易于实现和测试 +2. **保守策略**：避免误杀活跃对象 +3. **与现有机制兼容**：使用现有的 `GrantLease` 方法 + +### 注意事项 + +1. **提升时间**：如果对象数量很大，提升可能需要几秒 +2. **内存影响**：给所有对象授予租约，可能暂时保留一些不活跃对象 +3. **后续清理**：正常的驱逐循环会在后续清理不活跃对象 + diff --git a/doc/zh/rfc-standby-service-integration.md b/doc/zh/rfc-standby-service-integration.md new file mode 100644 index 0000000000..fa16d2cafa --- /dev/null +++ b/doc/zh/rfc-standby-service-integration.md @@ -0,0 +1,673 @@ +# Standby 服务集成方案 + +## 问题描述 + +在现有代码实现中，Standby Master 在 `MasterServiceSupervisor::Start()` 中只是阻塞等待 leader 失效（`WatchUntilDeleted`），没有运行 Standby 服务来同步 OpLog 和恢复 metadata。 + +### 现有代码的问题 + +```cpp +// MasterServiceSupervisor::Start() +mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); +// 这里会阻塞等待 leader 失效，期间 Standby 什么都不做 +``` + +**问题**： +1. Standby 在等待期间不执行任何操作 +2. 没有 watch etcd 的 OpLog +3. 没有实时恢复 metadata +4. 提升为 Primary 时，metadata 可能不完整 + +### 我们方案的需求 + +根据基于 etcd 的 OpLog 同步方案，Standby 需要： +1. **Watch etcd 的 OpLog**：实时接收 Primary 写入的 OpLog 事件 +2. **实时恢复 metadata**：将 OpLog 应用到本地 metadata store +3. **在等待选举期间持续运行**：即使不是 leader，也要保持数据同步 + +## 解决方案 + +### 核心思路 + +**在 Standby 模式下并行运行 Standby 服务**： +- 检测到有 leader 时，启动 Standby 服务 +- Standby 服务 watch etcd OpLog 并实时恢复 metadata +- 选举成功后，停止 Standby 服务并提升为 Primary + +### 架构设计 + +``` +┌─────────────────────────────────────────────────────────┐ +│ MasterServiceSupervisor::Start() │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 1. 检查当前是否有 leader │ +│ - GetMasterView() │ +│ - 如果有 leader 且不是自己 → Standby 模式 │ +│ - 如果没有 leader → 直接选举 │ +└─────────────────────────────────────────────────────────┘ + │ + ├─ 有 leader (Standby 模式) + │ │ + │ ▼ + │ ┌─────────────────────────────────────────────┐ + │ │ 2. 启动 Standby 服务 │ + │ │ - 创建 HotStandbyService │ + │ │ - 启动 ReplicationLoop (watch etcd) │ + │ │ - 启动 VerificationLoop │ + │ └─────────────────────────────────────────────┘ + │ │ + │ ▼ + │ ┌─────────────────────────────────────────────┐ + │ │ 3. 阻塞等待 leader 失效 │ + │ │ - ElectLeader() (WatchUntilDeleted) │ + │ │ - 期间 Standby 服务持续运行 │ + │ └─────────────────────────────────────────────┘ + │ + └─ 没有 leader (直接选举) + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 4. 选举成功 │ +│ - 停止 Standby 服务（如果正在运行） │ +│ - 检查是否准备好提升 │ +│ - 等待 5 秒防止 split-brain │ +└─────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ 5. 提升为 Primary │ +│ - 调用 HotStandbyService::Promote() │ +│ - 初始化所有对象的 lease │ +│ - 创建 WrappedMasterService │ +│ - 启动 RPC 服务器 │ +└─────────────────────────────────────────────────────────┘ +``` + +## 实现设计 + +### 1. 修改 MasterServiceSupervisor::Start() + +```cpp +int MasterServiceSupervisor::Start() { + while (true) { + LOG(INFO) << "Init master service..."; + coro_rpc::coro_rpc_server server( + config_.rpc_thread_num, config_.rpc_port, config_.rpc_address, + config_.rpc_conn_timeout, config_.rpc_enable_tcp_no_delay); + const char* value = std::getenv("MC_RPC_PROTOCOL"); + if (value && std::string_view(value) == "rdma") { + server.init_ibv(); + } + + LOG(INFO) << "Init leader election helper..."; + MasterViewHelper mv_helper; + if (mv_helper.ConnectToEtcd(config_.etcd_endpoints) != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd endpoints: " + << config_.etcd_endpoints; + return -1; + } + + // 【新增】检查当前是否有 leader + ViewVersionId current_version = 0; + std::string current_master; + auto ret = mv_helper.GetMasterView(current_master, current_version); + + // 【新增】如果有 leader 且不是自己，启动 Standby 服务 + std::unique_ptr standby_service = nullptr; + if (ret == ErrorCode::OK && current_master != config_.local_hostname) { + LOG(INFO) << "Current leader: " << current_master + << ", starting Standby service..."; + + // 创建并启动 Standby 服务 + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_master; + standby_config.etcd_endpoints = config_.etcd_endpoints; + standby_config.cluster_id = config_.cluster_id; + standby_config.enable_verification = true; + standby_config.max_replication_lag_entries = 1000; + + standby_service = std::make_unique(standby_config); + auto err = standby_service->Start(current_master); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service.reset(); + } else { + LOG(INFO) << "Standby service started, watching OpLog from etcd"; + } + } + + // 尝试选举（如果有 leader，会阻塞等待；如果没有，立即选举） + LOG(INFO) << "Trying to elect self as leader..."; + EtcdLeaseId lease_id = 0; + ViewVersionId view_version = 0; + mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + + // 【新增】停止 Standby 服务（如果正在运行） + if (standby_service) { + LOG(INFO) << "Stopping Standby service before promotion..."; + standby_service->Stop(); + + // 【新增】检查是否准备好提升 + if (!standby_service->IsReadyForPromotion()) { + LOG(WARNING) << "Standby is not ready for promotion, " + << "lag: " << standby_service->GetSyncStatus().lag_entries + << " entries, but proceeding anyway due to leader election"; + } + } + + // 防止 split-brain + const int waiting_time = ETCD_MASTER_VIEW_LEASE_TTL; + std::this_thread::sleep_for(std::chrono::seconds(waiting_time)); + + LOG(INFO) << "Starting master service as Primary..."; + + // 【新增】如果 Standby 服务存在，使用它来初始化 MasterService + std::unique_ptr promoted_service = nullptr; + if (standby_service) { + promoted_service = standby_service->Promote(); + if (!promoted_service) { + LOG(ERROR) << "Failed to promote Standby to Primary"; + // 继续使用新的 MasterService，但 metadata 可能不完整 + } else { + LOG(INFO) << "Successfully promoted Standby to Primary"; + } + } + + // 创建 WrappedMasterService + // 注意：这里需要将 promoted_service 的 metadata 复制到新的 MasterService + // 或者修改 WrappedMasterService 的构造方式，支持从 promoted_service 初始化 + mooncake::WrappedMasterService wrapped_master_service( + mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // TODO: 如果 promoted_service 存在，需要将其 metadata 复制到 wrapped_master_service + // 这需要修改 WrappedMasterService 或 MasterService 的接口 + + mooncake::RegisterRpcService(server, wrapped_master_service); + + // Start a thread to keep the leader alive + auto keep_leader_thread = + std::thread([&server, &mv_helper, lease_id]() { + mv_helper.KeepLeader(lease_id); + LOG(INFO) << "Trying to stop server..."; + server.stop(); + }); + + async_simple::Future ec = + server.async_start(); + if (ec.hasResult()) { + LOG(ERROR) << "Failed to start master service: " + << ec.result().value(); + auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); + if (etcd_err != ErrorCode::OK) { + LOG(ERROR) << "Failed to cancel keep leader alive: " + << etcd_err; + } + keep_leader_thread.join(); + return -1; + } + + // Block until the server is stopped + auto server_err = std::move(ec).get(); + LOG(ERROR) << "Master service stopped: " << server_err; + + // If the server is closed due to internal errors, we need to manually + // stop keep leader alive. + auto etcd_err = EtcdHelper::CancelKeepAlive(lease_id); + LOG(INFO) << "Cancel keep leader alive: " << etcd_err; + keep_leader_thread.join(); + } + return 0; +} +``` + +### 2. 修改 HotStandbyService::ReplicationLoop() + +```cpp +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started"; + + // 【新增】创建 OpLogWatcher（使用 etcd Watch） + OpLogWatcher oplog_watcher( + config_.etcd_endpoints, + config_.cluster_id, + this); // HotStandbyService 作为 OpLogApplier + + // 【新增】从上次处理的 sequence_id 开始读取历史 OpLog + uint64_t start_seq_id = applied_seq_id_.load() + 1; + if (start_seq_id > 1) { + std::vector historical_entries; + if (oplog_watcher.ReadOpLogSince(start_seq_id, historical_entries)) { + LOG(INFO) << "Read " << historical_entries.size() + << " historical OpLog entries from sequence_id " + << start_seq_id; + + // 应用历史 OpLog + for (const auto& entry : historical_entries) { + ApplyOpLogEntry(entry); + } + } else { + LOG(WARNING) << "Failed to read historical OpLog, " + << "may need to perform full snapshot sync"; + } + } + + // 【新增】启动 etcd Watch + oplog_watcher.Start(); + is_connected_.store(true); + LOG(INFO) << "OpLog watcher started, watching etcd for new OpLog entries"; + + while (running_.load()) { + // OpLogWatcher 会在后台线程中处理 Watch 事件 + // 当收到新 OpLog 时，会调用 ApplyOpLogEntry() + + // 定期检查同步状态 + auto status = GetSyncStatus(); + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Replication lag is high: " + << status.lag_entries << " entries"; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + // 【新增】停止 Watch + oplog_watcher.Stop(); + is_connected_.store(false); + LOG(INFO) << "Replication loop stopped"; +} +``` + +### 3. 实现 OpLogWatcher（基于 etcd Watch） + +```cpp +class OpLogWatcher { +public: + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, + OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), + cluster_id_(cluster_id), + applier_(applier), + etcd_oplog_store_(etcd_endpoints, cluster_id) { + etcd_prefix_ = "mooncake-store/oplog/" + cluster_id + "/"; + } + + void Start() { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return; + } + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLogThreadFunc, this); + LOG(INFO) << "OpLogWatcher started"; + } + + void Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + LOG(INFO) << "OpLogWatcher stopped"; + } + + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { + return etcd_oplog_store_.ReadOpLogSince(start_seq_id, 1000, entries); + } + +private: + void WatchOpLogThreadFunc() { + LOG(INFO) << "OpLog watch thread started"; + + // 从上次处理的 sequence_id 开始 Watch + uint64_t start_seq_id = last_processed_sequence_id_ + 1; + std::string watch_prefix = etcd_prefix_; + + while (running_.load()) { + try { + // 使用 etcd Watch 监听 OpLog 变化 + // 这里需要使用 etcd 的 Watch API + // 假设 EtcdHelper 提供了 WatchWithPrefix 方法 + auto watch_result = EtcdHelper::WatchWithPrefix( + watch_prefix.c_str(), + watch_prefix.size(), + [this](const EtcdWatchEvent& event) { + HandleWatchEvent(event); + }); + + if (!watch_result) { + LOG(ERROR) << "Watch failed, retrying..."; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } catch (const std::exception& e) { + LOG(ERROR) << "Exception in watch thread: " << e.what(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; + } + + void HandleWatchEvent(const EtcdWatchEvent& event) { + if (event.type == EtcdWatchEventType::PUT) { + // 解析 OpLog Entry + OpLogEntry entry; + if (DeserializeOpLogEntry(event.value, entry)) { + // 应用 OpLog + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_ = entry.sequence_id; + VLOG(2) << "Applied OpLog entry: sequence_id=" + << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + LOG(WARNING) << "Failed to apply OpLog entry: sequence_id=" + << entry.sequence_id; + } + } else { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " + << event.key; + } + } else if (event.type == EtcdWatchEventType::DELETE) { + // OpLog 被清理，记录日志 + VLOG(1) << "OpLog entry deleted: " << event.key; + } + } + + std::string etcd_endpoints_; + std::string cluster_id_; + std::string etcd_prefix_; + OpLogApplier* applier_; + EtcdOpLogStore etcd_oplog_store_; + + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; +}; +``` + +### 4. HotStandbyService 实现 OpLogApplier 接口 + +```cpp +class HotStandbyService : public OpLogApplier { +public: + // 实现 OpLogApplier 接口 + bool ApplyOpLogEntry(const OpLogEntry& entry) override { + std::lock_guard lock(mutex_); + + // 检查时序性 + if (!CheckSequenceOrder(entry)) { + LOG(WARNING) << "Sequence order violation for entry: " + << entry.sequence_id; + return false; + } + + // 应用 OpLog + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(WARNING) << "Unknown OpType: " + << static_cast(entry.op_type); + return false; + } + + applied_seq_id_.store(entry.sequence_id); + return true; + } + +private: + void ApplyPutEnd(const OpLogEntry& entry) { + // 从 metadata_store_ 创建或更新 metadata + // 这里需要实现完整的 metadata 恢复逻辑 + if (metadata_store_) { + metadata_store_->entry_count++; + } + } + + void ApplyPutRevoke(const OpLogEntry& entry) { + // 处理 PUT_REVOKE + // ... + } + + void ApplyRemove(const OpLogEntry& entry) { + // 从 metadata_store_ 删除 metadata + if (metadata_store_ && metadata_store_->entry_count > 0) { + metadata_store_->entry_count--; + } + } + + bool CheckSequenceOrder(const OpLogEntry& entry) { + // 检查全局序列号 + if (entry.sequence_id <= applied_seq_id_.load()) { + LOG(WARNING) << "Received out-of-order entry: " + << "expected > " << applied_seq_id_.load() + << ", got " << entry.sequence_id; + return false; + } + + // 检查 key 级别的序列号 + // 这里需要维护 key_sequence_map_ + // ... + + return true; + } +}; +``` + +## 关键设计点 + +### 1. Standby 服务生命周期 + +``` +启动阶段： +1. 检测到有 leader → 创建 HotStandbyService +2. 启动 ReplicationLoop → 读取历史 OpLog → 启动 Watch +3. 启动 VerificationLoop（可选） + +运行阶段： +1. OpLogWatcher 持续 watch etcd +2. 收到新 OpLog → 调用 ApplyOpLogEntry +3. 实时更新 metadata_store_ + +提升阶段： +1. 选举成功 → 停止 Standby 服务 +2. 调用 Promote() → 初始化 lease → 创建 MasterService +3. 启动 Primary 服务 +``` + +### 2. 历史 OpLog 读取 + +**策略**： +- 从 `applied_seq_id_ + 1` 开始读取 +- 如果 `applied_seq_id_` 为 0，说明是首次启动，需要从快照开始 +- 批量读取（每次 1000 条），避免一次性读取过多 + +**实现**： +```cpp +uint64_t start_seq_id = applied_seq_id_.load() + 1; +if (start_seq_id == 1) { + // 首次启动，需要从快照恢复 + // 或者从 sequence_id = 1 开始读取所有历史 +} +std::vector entries; +oplog_watcher.ReadOpLogSince(start_seq_id, entries); +``` + +### 3. etcd Watch 实现 + +**关键点**： +- 使用 `WatchWithPrefix` 监听 OpLog 前缀 +- 处理 Watch 断开和重连 +- 处理序列号不连续的情况 + +**Watch 前缀**： +``` +mooncake-store/oplog/{cluster_id}/ +``` + +### 4. 时序保证 + +**全局序列号**： +- 检查 `entry.sequence_id > applied_seq_id_` +- 如果序列号不连续，缓存待处理 + +**Key 级别序列号**： +- 维护 `key_sequence_map_` 记录每个 key 的最后 sequence_id +- 检查 `entry.key_sequence_id > key_sequence_map_[key]` + +## 与现有方案的集成 + +### 1. 与快照机制集成 + +**场景**：Standby 首次启动或需要全量同步 + +**流程**： +1. 检测到 `applied_seq_id_ == 0` 或 lag 过大 +2. 请求 Primary 的快照 +3. 应用快照 +4. 从快照的 `last_oplog_sequence_id` 开始读取增量 OpLog + +### 2. 与提升机制集成 + +**流程**： +1. 选举成功 +2. 停止 Standby 服务 +3. 检查同步状态（`IsReadyForPromotion()`） +4. 调用 `Promote()` → 初始化 lease +5. 创建 MasterService 并启动 + +### 3. 与 OpLog 清理集成 + +**场景**：OpLog 被清理后，Watch 可能收到 DELETE 事件 + +**处理**： +- 记录警告日志 +- 如果发现大量 OpLog 被删除，可能需要重新同步 + +## 错误处理和容错 + +### 1. Watch 断开 + +**处理**： +- 自动重连 +- 从上次处理的 sequence_id 重新 Watch +- 如果重连失败，记录错误并重试 + +### 2. 序列号不连续 + +**处理**： +- 缓存待处理的条目 +- 等待一段时间看是否有缺失的条目到达 +- 如果超时，请求 Primary 或从 etcd 读取缺失的条目 + +### 3. 应用失败 + +**处理**： +- 记录错误日志 +- 不更新 `applied_seq_id_` +- 继续处理后续条目（但可能影响一致性） + +## 性能考虑 + +### 1. Watch 性能 + +- etcd Watch 是高效的，不会产生大量网络开销 +- 批量处理 Watch 事件，减少锁竞争 + +### 2. 历史 OpLog 读取 + +- 批量读取（每次 1000 条） +- 并行应用（如果支持） + +### 3. Metadata 更新 + +- 使用适当的锁粒度 +- 考虑使用无锁数据结构（如果可能） + +## 测试场景 + +### 1. 正常 Standby 运行 + +``` +1. 启动 Standby，检测到有 leader +2. 启动 Standby 服务 +3. Watch etcd OpLog +4. 实时应用 OpLog 到 metadata +5. 验证 metadata 与 Primary 一致 +``` + +### 2. Standby 提升为 Primary + +``` +1. Standby 正在运行 +2. Primary 失效 +3. Standby 选举成功 +4. 停止 Standby 服务 +5. 提升为 Primary +6. 验证 metadata 完整性 +``` + +### 3. Watch 断开重连 + +``` +1. Standby 正在 Watch +2. etcd 连接断开 +3. 自动重连 +4. 从上次处理的 sequence_id 继续 +5. 验证没有丢失 OpLog +``` + +### 4. 历史 OpLog 读取 + +``` +1. Standby 重启 +2. applied_seq_id_ = 1000 +3. 读取 sequence_id >= 1001 的历史 OpLog +4. 应用历史 OpLog +5. 启动 Watch 监听新 OpLog +``` + +## 总结 + +### 核心方案 + +**在 Standby 模式下并行运行 Standby 服务，watch etcd OpLog 并实时恢复 metadata** + +### 关键实现 + +1. **MasterServiceSupervisor**：检测 leader，启动/停止 Standby 服务 +2. **HotStandbyService**：实现 OpLogApplier 接口，管理 Standby 生命周期 +3. **OpLogWatcher**：watch etcd OpLog，处理 Watch 事件 +4. **时序保证**：全局和 key 级别的序列号检查 + +### 优势 + +1. **实时同步**：Standby 实时接收并应用 OpLog +2. **数据完整性**：提升时 metadata 已完整 +3. **自动恢复**：Watch 断开自动重连 +4. **与现有方案兼容**：不影响现有的选举和提升逻辑 + +### 注意事项 + +1. **Watch 性能**：需要确保 etcd Watch 的性能 +2. **序列号不连续**：需要处理缺失的 OpLog +3. **Metadata 恢复**：需要完整实现 metadata 的恢复逻辑 +4. **提升时的数据迁移**：需要将 Standby 的 metadata 迁移到 Primary + diff --git a/mooncake-common/etcd/etcd_wrapper.go b/mooncake-common/etcd/etcd_wrapper.go index e57ae53628..ff2f0060fe 100644 --- a/mooncake-common/etcd/etcd_wrapper.go +++ b/mooncake-common/etcd/etcd_wrapper.go @@ -9,9 +9,11 @@ import "C" import ( "context" + "encoding/json" "strings" "sync" "time" + "unsafe" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -20,18 +22,21 @@ import ( // and can be configured separately. var ( // etcd client for transform engine - globalClient *clientv3.Client - globalMutex sync.Mutex - globalRefCount int + globalClient *clientv3.Client + globalMutex sync.Mutex + globalRefCount int // etcd client for store - storeClient *clientv3.Client - storeMutex sync.Mutex + storeClient *clientv3.Client + storeMutex sync.Mutex // keep alive contexts for store - storeKeepAliveCtx = make(map[int64]context.CancelFunc) - storeKeepAliveMutex sync.Mutex + storeKeepAliveCtx = make(map[int64]context.CancelFunc) + storeKeepAliveMutex sync.Mutex // watch contexts for store - storeWatchCtx = make(map[string]context.CancelFunc) - storeWatchMutex sync.Mutex + storeWatchCtx = make(map[string]context.CancelFunc) + storeWatchMutex sync.Mutex + // watch contexts for prefix watch + storePrefixWatchCtx = make(map[string]context.CancelFunc) + storePrefixWatchMutex sync.Mutex ) //export NewEtcdClient @@ -43,30 +48,30 @@ func NewEtcdClient(endpoints *C.char, errMsg **C.char) int { return 0 } - MaxMsgSize := 32*1024*1024 - endpointStr := C.GoString(endpoints) - // Support multiple endpoints separated by comma or semicolon - // Normalize separators to semicolon first, then split - endpointStr = strings.ReplaceAll(endpointStr, ",", ";") - parts := strings.Split(endpointStr, ";") - var validEndpoints []string - for _, ep := range parts { - ep = strings.TrimSpace(ep) - if ep != "" { - validEndpoints = append(validEndpoints, ep) - } - } - if len(validEndpoints) == 0 { - *errMsg = C.CString("no valid endpoints provided") - return -1 - } - - cli, err := clientv3.New(clientv3.Config{ - Endpoints: validEndpoints, - DialTimeout: 5 * time.Second, - MaxCallSendMsgSize: MaxMsgSize, - MaxCallRecvMsgSize: MaxMsgSize, - }) + MaxMsgSize := 32 * 1024 * 1024 + endpointStr := C.GoString(endpoints) + // Support multiple endpoints separated by comma or semicolon + // Normalize separators to semicolon first, then split + endpointStr = strings.ReplaceAll(endpointStr, ",", ";") + parts := strings.Split(endpointStr, ";") + var validEndpoints []string + for _, ep := range parts { + ep = strings.TrimSpace(ep) + if ep != "" { + validEndpoints = append(validEndpoints, ep) + } + } + if len(validEndpoints) == 0 { + *errMsg = C.CString("no valid endpoints provided") + return -1 + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: validEndpoints, + DialTimeout: 5 * time.Second, + MaxCallSendMsgSize: MaxMsgSize, + MaxCallRecvMsgSize: MaxMsgSize, + }) if err != nil { *errMsg = C.CString(err.Error()) @@ -160,7 +165,7 @@ func NewStoreEtcdClient(endpoints *C.char, errMsg **C.char) int { endpointStr := C.GoString(endpoints) endpointList := strings.Split(endpointStr, ";") - + // Filter out any empty strings that might result from splitting var validEndpoints []string for _, ep := range endpointList { @@ -235,37 +240,37 @@ func EtcdStoreGrantLeaseWrapper(ttl int64, leaseId *int64, errMsg **C.char) int //export EtcdStoreCreateWithLeaseWrapper func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, leaseId int64, revisionId *int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - v := C.GoStringN(value, valueSize) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // Create a transaction - txn := storeClient.Txn(ctx) - - // Only put the key if it does not exist - resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). - Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). - Commit() - - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // If the key already existed, resp.Succeeded will be false - // If we created the key, resp.Succeeded will be true - if resp.Succeeded { - *revisionId = resp.Header.Revision - return 0; - } else { - *errMsg = C.CString("etcd transaction failed") - return -2 - } + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Create a transaction + txn := storeClient.Txn(ctx) + + // Only put the key if it does not exist + resp, err := txn.If(clientv3.Compare(clientv3.CreateRevision(k), "=", 0)). + Then(clientv3.OpPut(k, v, clientv3.WithLease(clientv3.LeaseID(leaseId)))). + Commit() + + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // If the key already existed, resp.Succeeded will be false + // If we created the key, resp.Succeeded will be true + if resp.Succeeded { + *revisionId = resp.Header.Revision + return 0 + } else { + *errMsg = C.CString("etcd transaction failed") + return -2 + } } /* @@ -274,77 +279,77 @@ func EtcdStoreCreateWithLeaseWrapper(key *C.char, keySize C.int, value *C.char, * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteWatch(k string) int { - storeWatchMutex.Lock() - defer storeWatchMutex.Unlock() - - if cancel, exists := storeWatchCtx[k]; exists { - cancel() - delete(storeWatchCtx, k) - return 0 - } + storeWatchMutex.Lock() + defer storeWatchMutex.Unlock() + + if cancel, exists := storeWatchCtx[k]; exists { + cancel() + delete(storeWatchCtx, k) + return 0 + } return -1 } //export EtcdStoreWatchUntilDeletedWrapper func EtcdStoreWatchUntilDeletedWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - k := C.GoStringN(key, keySize) - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeWatchMutex.Lock() - if _, exists := storeWatchCtx[k]; exists { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeWatchMutex.Lock() + if _, exists := storeWatchCtx[k]; exists { storeWatchMutex.Unlock() - *errMsg = C.CString("This key is already being watched") - return -1 - } - storeWatchCtx[k] = cancel - storeWatchMutex.Unlock() + *errMsg = C.CString("This key is already being watched") + return -1 + } + storeWatchCtx[k] = cancel + storeWatchMutex.Unlock() // Make sure to delete from the map before returning defer cancelAndDeleteWatch(k) - // Start watching the key - watchChan := storeClient.Watch(ctx, k) - - // Wait for the key to be deleted - for { - select { - case watchResp, ok := <-watchChan: - if !ok { - // Channel closed unexpectedly - *errMsg = C.CString("watch channel closed unexpectedly") - return -1 - } - for _, event := range watchResp.Events { - if event.Type == clientv3.EventTypeDelete { - // Clean up the context when done - return 0 - } - } - case <-ctx.Done(): - // Context was cancelled + // Start watching the key + watchChan := storeClient.Watch(ctx, k) + + // Wait for the key to be deleted + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + *errMsg = C.CString("watch channel closed unexpectedly") + return -1 + } + for _, event := range watchResp.Events { + if event.Type == clientv3.EventTypeDelete { + // Clean up the context when done + return 0 + } + } + case <-ctx.Done(): + // Context was cancelled *errMsg = C.CString("watch context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelWatchWrapper func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) int { - k := C.GoStringN(key, keySize) - if cancelAndDeleteWatch(k) == -1 { - *errMsg = C.CString("no watch context found for the given key") - return -1 - } - return 0 + k := C.GoStringN(key, keySize) + if cancelAndDeleteWatch(k) == -1 { + *errMsg = C.CString("no watch context found for the given key") + return -1 + } + return 0 } /* @@ -353,76 +358,526 @@ func EtcdStoreCancelWatchWrapper(key *C.char, keySize C.int, errMsg **C.char) in * other than the one we want to delete. In that case, that context will * be deleted before being cancelled and will not be able to be cancelled * anymore. -*/ + */ func cancelAndDeleteKeepAlive(leaseId int64) int { - storeKeepAliveMutex.Lock() - defer storeKeepAliveMutex.Unlock() - - if cancel, exists := storeKeepAliveCtx[leaseId]; exists { - cancel() - delete(storeKeepAliveCtx, leaseId) - return 0 - } + storeKeepAliveMutex.Lock() + defer storeKeepAliveMutex.Unlock() + + if cancel, exists := storeKeepAliveCtx[leaseId]; exists { + cancel() + delete(storeKeepAliveCtx, leaseId) + return 0 + } return -1 } //export EtcdStoreKeepAliveWrapper func EtcdStoreKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if storeClient == nil { - *errMsg = C.CString("etcd client not initialized") - return -1 - } - - // Create a context with cancel function - ctx, cancel := context.WithCancel(context.Background()) - - // Store the cancel function - storeKeepAliveMutex.Lock() + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storeKeepAliveMutex.Lock() if _, exists := storeKeepAliveCtx[leaseId]; exists { storeKeepAliveMutex.Unlock() - *errMsg = C.CString("This lease id is already being kept alive") - return -1 - } - storeKeepAliveCtx[leaseId] = cancel - storeKeepAliveMutex.Unlock() + *errMsg = C.CString("This lease id is already being kept alive") + return -1 + } + storeKeepAliveCtx[leaseId] = cancel + storeKeepAliveMutex.Unlock() // Make sure to delete from the map before returning - defer cancelAndDeleteKeepAlive(leaseId) - - // Start keep alive - keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) - if err != nil { - *errMsg = C.CString(err.Error()) - return -1 - } - - // Wait for keep alive responses - for { - select { - case resp, ok := <-keepAliveChan: - if !ok { - *errMsg = C.CString("keep alive channel closed") - return -1 - } - if resp == nil { - *errMsg = C.CString("keep alive response is nil") - return -1 - } - // Keep alive successful, continue - case <-ctx.Done(): + defer cancelAndDeleteKeepAlive(leaseId) + + // Start keep alive + keepAliveChan, err := storeClient.KeepAlive(ctx, clientv3.LeaseID(leaseId)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + // Wait for keep alive responses + for { + select { + case resp, ok := <-keepAliveChan: + if !ok { + *errMsg = C.CString("keep alive channel closed") + return -1 + } + if resp == nil { + *errMsg = C.CString("keep alive response is nil") + return -1 + } + // Keep alive successful, continue + case <-ctx.Done(): // Context cancelled *errMsg = C.CString("keep alive context cancelled") - return -2 - } - } + return -2 + } + } } //export EtcdStoreCancelKeepAliveWrapper func EtcdStoreCancelKeepAliveWrapper(leaseId int64, errMsg **C.char) int { - if cancelAndDeleteKeepAlive(leaseId) == -1 { - *errMsg = C.CString("no keep alive context found for the given lease ID") - return -1 - } - return 0 + if cancelAndDeleteKeepAlive(leaseId) == -1 { + *errMsg = C.CString("no keep alive context found for the given lease ID") + return -1 + } + return 0 +} + +//export EtcdStorePutWrapper +func EtcdStorePutWrapper(key *C.char, keySize C.int, value *C.char, valueSize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + k := C.GoStringN(key, keySize) + v := C.GoStringN(value, valueSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := storeClient.Put(ctx, k, v) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +//export EtcdStoreGetWithPrefixWrapper +func EtcdStoreGetWithPrefixWrapper(prefix *C.char, prefixSize C.int, keys **C.char, keySizes **C.int, values **C.char, valueSizes **C.int, count *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if len(resp.Kvs) == 0 { + *count = 0 + return 0 + } + + // Allocate arrays for keys and values + keyCount := len(resp.Kvs) + *count = C.int(keyCount) + + // Allocate memory for arrays + keysArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + keySizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + valuesArray := (*[1 << 30]*C.char)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof((*C.char)(nil))))) + valueSizesArray := (*[1 << 30]C.int)(C.malloc(C.size_t(keyCount) * C.size_t(unsafe.Sizeof(C.int(0))))) + + for i, kv := range resp.Kvs { + keysArray[i] = C.CString(string(kv.Key)) + keySizesArray[i] = C.int(len(kv.Key)) + valuesArray[i] = C.CString(string(kv.Value)) + valueSizesArray[i] = C.int(len(kv.Value)) + } + + *keys = (*C.char)(unsafe.Pointer(keysArray)) + *keySizes = (*C.int)(unsafe.Pointer(keySizesArray)) + *values = (*C.char)(unsafe.Pointer(valuesArray)) + *valueSizes = (*C.int)(unsafe.Pointer(valueSizesArray)) + + return 0 +} + +//export EtcdStoreGetRangeAsJsonWrapper +func EtcdStoreGetRangeAsJsonWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, limit C.int, outJson **C.char, outJsonSize *C.int, revisionId *C.longlong, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + opts := []clientv3.OpOption{ + clientv3.WithRange(end), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), + } + if limit > 0 { + opts = append(opts, clientv3.WithLimit(int64(limit))) + } + resp, err := storeClient.Get(ctx, start, opts...) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + + if resp != nil && resp.Header != nil { + *revisionId = C.longlong(resp.Header.Revision) + } else { + *revisionId = 0 + } + + type kvPair struct { + Key string `json:"key"` + Value string `json:"value"` + } + kvs := make([]kvPair, 0, len(resp.Kvs)) + for _, kv := range resp.Kvs { + kvs = append(kvs, kvPair{Key: string(kv.Key), Value: string(kv.Value)}) + } + b, jerr := json.Marshal(kvs) + if jerr != nil { + *errMsg = C.CString(jerr.Error()) + return -1 + } + + *outJson = C.CString(string(b)) + *outJsonSize = C.int(len(b)) + return 0 +} + +//export EtcdStoreGetFirstKeyWithPrefixWrapper +func EtcdStoreGetFirstKeyWithPrefixWrapper(prefix *C.char, prefixSize C.int, firstKey **C.char, firstKeySize *C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + resp, err := storeClient.Get(ctx, p, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), clientv3.WithLimit(1)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + if len(resp.Kvs) == 0 { + *errMsg = C.CString("no key found with prefix") + return -2 + } + kv := resp.Kvs[0] + *firstKey = C.CString(string(kv.Key)) + *firstKeySize = C.int(len(kv.Key)) + return 0 +} + +//export EtcdStoreDeleteRangeWrapper +func EtcdStoreDeleteRangeWrapper(startKey *C.char, startKeySize C.int, endKey *C.char, endKeySize C.int, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + start := C.GoStringN(startKey, startKeySize) + end := C.GoStringN(endKey, endKeySize) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := storeClient.Delete(ctx, start, clientv3.WithRange(end)) + if err != nil { + *errMsg = C.CString(err.Error()) + return -1 + } + return 0 +} + +//export EtcdStoreWatchWithPrefixWrapper +func EtcdStoreWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + // Start watching in a goroutine + go func() { + defer cancelAndDeletePrefixWatch(p) + + // Start watching the prefix + watchChan := storeClient.Watch(ctx, p, clientv3.WithPrefix()) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly + return + } + if watchResp.Err() != nil { + // Watch error, stop watching + return + } + + // Process each event + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) // WatchEventTypePut + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) // WatchEventTypeDelete + valuePtr = nil + valueSize = 0 + } + + // Call the C callback function + // Convert unsafe.Pointer to function pointer type and call it + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + // Free the C strings + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + // Context was cancelled + return + } + } + }() + + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionWrapper +func EtcdStoreWatchWithPrefixFromRevisionWrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + // Create a context with cancel function + ctx, cancel := context.WithCancel(context.Background()) + + // Store the cancel function + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + return + } + if watchResp.Err() != nil { + return + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + +//export EtcdStoreWatchWithPrefixFromRevisionV2Wrapper +func EtcdStoreWatchWithPrefixFromRevisionV2Wrapper(prefix *C.char, prefixSize C.int, startRevision C.longlong, callbackContext unsafe.Pointer, callbackFunc unsafe.Pointer, errMsg **C.char) int { + if storeClient == nil { + *errMsg = C.CString("etcd client not initialized") + return -1 + } + if callbackFunc == nil { + *errMsg = C.CString("callback function is nil") + return -1 + } + p := C.GoStringN(prefix, prefixSize) + + ctx, cancel := context.WithCancel(context.Background()) + + storePrefixWatchMutex.Lock() + if _, exists := storePrefixWatchCtx[p]; exists { + storePrefixWatchMutex.Unlock() + *errMsg = C.CString("This prefix is already being watched") + return -1 + } + storePrefixWatchCtx[p] = cancel + storePrefixWatchMutex.Unlock() + + go func() { + defer cancelAndDeletePrefixWatch(p) + + opts := []clientv3.OpOption{clientv3.WithPrefix()} + if startRevision > 0 { + opts = append(opts, clientv3.WithRev(int64(startRevision))) + } + watchChan := storeClient.Watch(ctx, p, opts...) + + for { + select { + case watchResp, ok := <-watchChan: + if !ok { + // Channel closed unexpectedly. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + if watchResp.Err() != nil { + // Watch error, stop watching. Notify C++ watcher to reconnect. + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, nil, 0, nil, 0, C.int(2) /*WATCH_BROKEN*/, C.longlong(0)) + return + } + + // Use response-level revision as a more stable resume point. + // (It can be >= individual event's ModRevision.) + // Note: watchResp.Header is a value type, not a pointer, so we can directly access it. + respRev := int64(0) + if watchResp.Header.Revision > 0 { + respRev = watchResp.Header.Revision + } + + for _, event := range watchResp.Events { + keyStr := string(event.Kv.Key) + keyPtr := C.CString(keyStr) + keySize := C.size_t(len(keyStr)) + + var valuePtr *C.char + var valueSize C.size_t + var eventType C.int + + if event.Type == clientv3.EventTypePut { + eventType = C.int(0) + valueStr := string(event.Kv.Value) + valuePtr = C.CString(valueStr) + valueSize = C.size_t(len(valueStr)) + } else if event.Type == clientv3.EventTypeDelete { + eventType = C.int(1) + valuePtr = nil + valueSize = 0 + } + + modRev := C.longlong(0) + if event.Kv != nil { + evRev := event.Kv.ModRevision + if respRev > evRev { + evRev = respRev + } + modRev = C.longlong(evRev) + } else if respRev > 0 { + modRev = C.longlong(respRev) + } + + // Callback signature: + // void cb(void* ctx, char* key, size_t keySize, char* value, size_t valueSize, int eventType, long long modRev) + callbackType := (*func(unsafe.Pointer, *C.char, C.size_t, *C.char, C.size_t, C.int, C.longlong))(callbackFunc) + (*callbackType)(callbackContext, keyPtr, keySize, valuePtr, valueSize, eventType, modRev) + + C.free(unsafe.Pointer(keyPtr)) + if valuePtr != nil { + C.free(unsafe.Pointer(valuePtr)) + } + } + case <-ctx.Done(): + return + } + } + }() + + return 0 +} + +func cancelAndDeletePrefixWatch(p string) int { + storePrefixWatchMutex.Lock() + defer storePrefixWatchMutex.Unlock() + + if cancel, exists := storePrefixWatchCtx[p]; exists { + cancel() + delete(storePrefixWatchCtx, p) + return 0 + } + return -1 +} + +//export EtcdStoreCancelWatchWithPrefixWrapper +func EtcdStoreCancelWatchWithPrefixWrapper(prefix *C.char, prefixSize C.int, errMsg **C.char) int { + p := C.GoStringN(prefix, prefixSize) + if cancelAndDeletePrefixWatch(p) == -1 { + *errMsg = C.CString("no watch context found for the given prefix") + return -1 + } + return 0 } func main() {} diff --git a/mooncake-store/include/etcd_helper.h b/mooncake-store/include/etcd_helper.h index 1f272142ac..e9a6044a76 100644 --- a/mooncake-store/include/etcd_helper.h +++ b/mooncake-store/include/etcd_helper.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include "types.h" @@ -90,6 +92,124 @@ class EtcdHelper { */ static ErrorCode CancelKeepAlive(EtcdLeaseId lease_id); + /* + * @brief Put a key-value pair to etcd. + * @param key: The key to put. + * @param key_size: The size of the key in bytes. + * @param value: The value to put. + * @param value_size: The size of the value in bytes. + * @return: Error code. + */ + static ErrorCode Put(const char* key, const size_t key_size, + const char* value, const size_t value_size); + + /* + * @brief Get all key-value pairs with a given prefix. + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param keys: Output param, vector of keys. + * @param values: Output param, vector of values. + * @return: Error code. + */ + static ErrorCode GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values); + + /* + * @brief Range get in etcd and return result as a JSON array string. + * This avoids complex cross-language memory management for key/value arrays. + * @param start_key: Start key (inclusive). + * @param start_key_size: Size in bytes. + * @param end_key: End key (exclusive). + * @param end_key_size: Size in bytes. + * @param limit: Maximum number of kvs to return (0 means no limit). + * @param json: Output JSON string, format: [{"key":"...","value":"..."}] + * @param revision_id: Output etcd revision of this read (resp.Header.Revision). + */ + static ErrorCode GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id); + + /* + * @brief Get the first key with a given prefix (sorted by key). + * @param prefix: The prefix to search for. + * @param prefix_size: The size of the prefix in bytes. + * @param first_key: Output param, the first key found. + * @return: Error code. ETCD_KEY_NOT_EXIST if no key found. + */ + static ErrorCode GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key); + + /* + * @brief Delete a range of keys from etcd. + * @param start_key: The start key (inclusive). + * @param start_key_size: The size of the start key in bytes. + * @param end_key: The end key (exclusive). + * @param end_key_size: The size of the end key in bytes. + * @return: Error code. + */ + static ErrorCode DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size); + + /* + * @brief Watch all keys with a given prefix for changes. + * This is a non-blocking function that starts watching in a background + * goroutine. Events are delivered via the callback function. + * @param prefix: The prefix to watch. + * @param prefix_size: The size of the prefix in bytes. + * @param callback_context: User context passed to the callback function. + * @param callback_func: Callback function called for each watch event. + * Signature: void callback(void* context, const char* key, size_t key_size, + * const char* value, size_t value_size, int event_type) + * event_type: 0 = PUT, 1 = DELETE + * @return: Error code. + */ + static ErrorCode WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)); + + /* + * @brief Watch all keys with a given prefix from a specific etcd revision. + * This is used to close the "read historical -> start watch" gap. + * @param start_revision: Watch events with revision >= start_revision (0 means from now). + */ + static ErrorCode WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)); + + /* + * @brief Watch all keys with a given prefix from a specific etcd revision (V2). + * V2 callback includes `mod_revision` for precise resume. + * (Implementation may pass max(event.ModRevision, watchResp.Header.Revision).) + * @param callback_func: void cb(void* ctx, const char* key, size_t key_size, + * const char* value, size_t value_size, + * int event_type, int64_t mod_revision) + * event_type: 0=PUT, 1=DELETE, 2=WATCH_BROKEN (watch ended; reconnect) + */ + static ErrorCode WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)); + + /* + * @brief Cancel watching a prefix. + * @param prefix: The prefix to stop watching. + * @param prefix_size: The size of the prefix in bytes. + * @return: Error code. + */ + static ErrorCode CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size); + private: // Variables that are used to ensure the etcd client // is only connected once. diff --git a/mooncake-store/include/etcd_oplog_store.h b/mooncake-store/include/etcd_oplog_store.h new file mode 100644 index 0000000000..55efa78d9d --- /dev/null +++ b/mooncake-store/include/etcd_oplog_store.h @@ -0,0 +1,193 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +/** + * @brief Store for OpLog entries in etcd. + * + * This class is responsible for writing OpLog entries to etcd and reading them back. + * OpLog entries are stored with keys in the format: + * /oplog/{cluster_id}/{sequence_id} + * + * The latest sequence_id is also stored at: + * /oplog/{cluster_id}/latest + */ +class EtcdOpLogStore { + public: + /** + * @brief Constructor. + * @param cluster_id: The cluster ID for this OpLog store. + * @param enable_latest_seq_batch_update: Whether to start background thread + * to batch-update `/latest`. Readers (Standby) should set this to false + * to avoid unnecessary thread creation. + */ + explicit EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update = false); + + /** + * @brief Write an OpLog entry to etcd. + * @param entry: The OpLog entry to write. + * @return: Error code. + */ + ErrorCode WriteOpLog(const OpLogEntry& entry); + + /** + * @brief Read an OpLog entry from etcd by sequence_id. + * @param sequence_id: The sequence ID of the entry to read. + * @param entry: Output param, the OpLog entry. + * @return: Error code. + */ + ErrorCode ReadOpLog(uint64_t sequence_id, OpLogEntry& entry); + + /** + * @brief Read OpLog entries starting from a given sequence_id. + * @param start_sequence_id: The starting sequence ID (exclusive). + * @param limit: Maximum number of entries to read (default: 1000). + * @param entries: Output param, vector of OpLog entries. + * @return: Error code. + */ + ErrorCode ReadOpLogSince(uint64_t start_sequence_id, size_t limit, + std::vector& entries); + + // Like ReadOpLogSince, but also returns the etcd revision for consistent + // "read then watch(from revision+1)" startup. + ErrorCode ReadOpLogSinceWithRevision(uint64_t start_sequence_id, size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id); + + /** + * @brief Get the latest sequence_id from etcd. + * @param sequence_id: Output param, the latest sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if no OpLog exists yet. + */ + ErrorCode GetLatestSequenceId(uint64_t& sequence_id); + + /** + * @brief Update the latest sequence_id in etcd. + * @param sequence_id: The latest sequence_id to update. + * @return: Error code. + */ + ErrorCode UpdateLatestSequenceId(uint64_t sequence_id); + + /** + * @brief Record the sequence_id corresponding to a snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: The sequence_id at which the snapshot was taken. + * @return: Error code. + */ + ErrorCode RecordSnapshotSequenceId(const std::string& snapshot_id, + uint64_t sequence_id); + + /** + * @brief Get the sequence_id for a given snapshot. + * @param snapshot_id: The snapshot ID. + * @param sequence_id: Output param, the sequence_id. + * @return: Error code. ETCD_KEY_NOT_EXIST if snapshot not found. + */ + ErrorCode GetSnapshotSequenceId(const std::string& snapshot_id, + uint64_t& sequence_id); + + /** + * @brief Clean up OpLog entries before a given sequence_id. + * @param before_sequence_id: All entries with sequence_id < before_sequence_id + * will be deleted. + * @return: Error code. + */ + ErrorCode CleanupOpLogBefore(uint64_t before_sequence_id); + + /** + * @brief Destructor - stops batch update thread. + */ + ~EtcdOpLogStore(); + + private: + /** + * @brief Build the etcd key for an OpLog entry. + * @param sequence_id: The sequence ID. + * @return: The etcd key. + */ + std::string BuildOpLogKey(uint64_t sequence_id) const; + + /** + * @brief Build the etcd key for the latest sequence_id. + * @return: The etcd key. + */ + std::string BuildLatestKey() const; + + /** + * @brief Build the etcd key for a snapshot sequence_id. + * @param snapshot_id: The snapshot ID. + * @return: The etcd key. + */ + std::string BuildSnapshotKey(const std::string& snapshot_id) const; + + // Best-effort: find the minimum existing OpLog sequence_id in etcd. + // Used for robust cleanup (Scheme 3) so we don't rely on a persisted + // "cleaned_upto" marker. + std::optional GetMinSequenceId() const; + + /** + * @brief Serialize an OpLogEntry to JSON string. + * @param entry: The OpLog entry to serialize. + * @return: The JSON string. + */ + std::string SerializeOpLogEntry(const OpLogEntry& entry) const; + + /** + * @brief Deserialize a JSON string to OpLogEntry. + * @param json_str: The JSON string. + * @param entry: Output param, the OpLog entry. + * @return: true if successful, false otherwise. + */ + bool DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const; + + /** + * @brief Batch update thread function. + * Periodically updates latest_sequence_id in etcd. + */ + void BatchUpdateThread(); + + /** + * @brief Trigger immediate batch update if threshold is reached. + */ + void TriggerBatchUpdateIfNeeded(); + + /** + * @brief Perform the actual batch update to etcd. + */ + void DoBatchUpdate(); + + std::string cluster_id_; + static constexpr const char* kOpLogPrefix = "/oplog/"; + static constexpr const char* kLatestSuffix = "/latest"; + static constexpr const char* kSnapshotPrefix = "/oplog/"; + static constexpr const char* kSnapshotSuffix = "/snapshot/"; + + // Batch update mechanism for latest_sequence_id + const bool enable_latest_seq_batch_update_{false}; + std::atomic pending_latest_seq_id_{0}; + std::atomic pending_count_{0}; + std::atomic batch_update_running_{false}; + std::mutex batch_update_mutex_; + std::thread batch_update_thread_; + std::chrono::steady_clock::time_point last_update_time_; + + // Batch update configuration + static constexpr size_t kBatchSize = 100; // Update every 100 entries + static constexpr int kBatchIntervalMs = 1000; // Or every 1 second +}; + +} // namespace mooncake diff --git a/mooncake-store/include/ha_helper.h b/mooncake-store/include/ha_helper.h index 897ba53a5c..f6774d6a93 100644 --- a/mooncake-store/include/ha_helper.h +++ b/mooncake-store/include/ha_helper.h @@ -3,12 +3,15 @@ #include +#include +#include #include #include #include -#include "types.h" +#include "hot_standby_service.h" #include "master_config.h" +#include "types.h" namespace mooncake { @@ -77,10 +80,27 @@ class MasterServiceSupervisor { ~MasterServiceSupervisor(); private: + /** + * @brief Start HotStandbyService when there is an existing leader + * @param mv_helper MasterViewHelper instance + * @param current_leader Current leader address + */ + void StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader); + + /** + * @brief Stop HotStandbyService + */ + void StopStandbyService(); + // coro_rpc server thread std::thread server_thread_; MasterServiceSupervisorConfig config_; + + // HotStandbyService for standby mode + std::unique_ptr standby_service_; + std::atomic standby_running_{false}; }; } // namespace mooncake diff --git a/mooncake-store/include/hot_standby_service.h b/mooncake-store/include/hot_standby_service.h new file mode 100644 index 0000000000..7b7752d66a --- /dev/null +++ b/mooncake-store/include/hot_standby_service.h @@ -0,0 +1,220 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "metadata_store.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" +#include "snapshot_provider.h" +#include "types.h" + +namespace mooncake { + +// Forward declarations +class MasterService; +class ReplicationStream; + +/** + * @brief Configuration for HotStandbyService + */ +struct HotStandbyConfig { + std::string standby_id; + std::string primary_address; + uint32_t replication_port{0}; + uint32_t verification_interval_sec{30}; + uint32_t max_replication_lag_entries{1000}; + bool enable_verification{true}; + + // Snapshot bootstrap (optional): + // If provided, Standby will try to load a snapshot first, then replay OpLog + // from snapshot_sequence_id. + bool enable_snapshot_bootstrap{false}; +}; + +/** + * @brief Sync status information for HotStandbyService + */ +struct StandbySyncStatus { + uint64_t applied_seq_id{0}; + uint64_t primary_seq_id{0}; + uint64_t lag_entries{0}; + std::chrono::milliseconds lag_time{0}; + bool is_syncing{false}; + bool is_connected{false}; +}; + +/** + * @brief HotStandbyService manages standby replication and promotion + * + * This service runs on Standby Master nodes and is responsible for: + * - Connecting to Primary and receiving OpLog entries + * - Applying OpLog entries to local metadata store + * - Periodically verifying data consistency with Primary + * - Promoting to Primary when elected as new Leader + * + * For now, this is a skeleton implementation without actual network + * communication. The gRPC integration will be added later. + */ +class HotStandbyService { + public: + explicit HotStandbyService(const HotStandbyConfig& config); + ~HotStandbyService(); + + /** + * @brief Start connecting to Primary and begin replication + * @param primary_address Address of the Primary Master (not used with etcd-based sync) + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier for OpLog path + * @return ErrorCode::OK on success + */ + ErrorCode Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id); + + /** + * @brief Stop replication and disconnect from Primary + */ + void Stop(); + + /** + * @brief Get current synchronization status + * @return StandbySyncStatus with current sync state + */ + StandbySyncStatus GetSyncStatus() const; + + /** + * @brief Check if standby is ready for promotion + * @return true if replication lag is within threshold + */ + bool IsReadyForPromotion() const; + + /** + * @brief Promote this standby to Primary + * + * This method should be called after successful leader election. + * It returns a MasterService instance initialized with the replicated + * metadata, ready to serve as the new Primary. + * + * @return Unique pointer to MasterService, or nullptr on failure + */ + std::unique_ptr Promote(); + + /** + * @brief Get the number of metadata entries in the local store + */ + size_t GetMetadataCount() const; + + /** + * @brief Get the latest applied sequence ID after promotion + * + * This should be called after Promote() to get the sequence_id + * that the new Primary's OpLogManager should start from. + * + * @return Latest applied sequence ID, or 0 if not available + */ + uint64_t GetLatestAppliedSequenceId() const; + + // Export a point-in-time snapshot of all replicated metadata. + // This is used by MasterServiceSupervisor to initialize the new Primary + // after leader election (fast recovery). + bool ExportMetadataSnapshot( + std::vector>& out) const; + + // Inject a snapshot provider (from external snapshot implementation). + void SetSnapshotProvider(std::unique_ptr provider); + + private: + /** + * @brief Main replication loop (runs in background thread) + */ + void ReplicationLoop(); + + /** + * @brief Verification loop (runs in background thread) + */ + void VerificationLoop(); + + /** + * @brief Apply a single OpLog entry to local metadata store + * @param entry The OpLog entry to apply + * @deprecated Use OpLogApplier instead + */ + void ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Connect to Primary and establish replication stream + * @return true on success, false on failure + */ + bool ConnectToPrimary(); + + /** + * @brief Disconnect from Primary + */ + void DisconnectFromPrimary(); + + /** + * @brief Process a batch of OpLog entries received from Primary + * @param entries Batch of OpLog entries + */ + void ProcessOpLogBatch(const std::vector& entries); + + HotStandbyConfig config_; + + // Simple in-memory metadata store implementation + class StandbyMetadataStore : public MetadataStore { + public: + bool PutMetadata(const std::string& key, + const StandbyObjectMetadata& metadata) override; + bool Put(const std::string& key, + const std::string& payload = std::string()) override; + const StandbyObjectMetadata* GetMetadata(const std::string& key) const override; + bool Remove(const std::string& key) override; + bool Exists(const std::string& key) const override; + size_t GetKeyCount() const override; + + // Snapshot for promotion/restore. + void Snapshot( + std::vector>& out) const; + + private: + mutable std::mutex mutex_; + std::unordered_map store_; + }; + std::unique_ptr metadata_store_; + std::unique_ptr snapshot_provider_{std::make_unique()}; + + // OpLog replication components + std::unique_ptr oplog_applier_; + std::unique_ptr oplog_watcher_; + + // Configuration for etcd-based OpLog sync + std::string etcd_endpoints_; + std::string cluster_id_; + + // Replication state + std::shared_ptr replication_stream_; + std::atomic applied_seq_id_{0}; + std::atomic primary_seq_id_{0}; + std::atomic running_{false}; + std::atomic is_connected_{false}; + + // Background threads + std::thread replication_thread_; + std::thread verification_thread_; + + // Synchronization + mutable std::mutex mutex_; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 35e728f0c5..7b42a3c3e1 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -4,7 +4,10 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -13,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -25,11 +29,16 @@ #include "master_config.h" #include "rpc_types.h" #include "replica.h" +#include "oplog_manager.h" +#include "metadata_store.h" namespace mooncake { // Forward declarations class AllocationStrategy; class EvictionStrategy; +class BufferAllocatorBase; +struct StandbyObjectMetadata; +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead /* * @brief MasterService is the main class for the master server. @@ -94,6 +103,13 @@ class MasterService { */ auto GetAllKeys() -> tl::expected, ErrorCode>; + // Restore metadata from a Standby snapshot (fast failover). + // NOTE: This is used only on the node that was running HotStandbyService + // right before it was promoted to leader. + void RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + /** * @brief Fetch all segments, each node has a unique real client with fixed * segment name : segment name, preferred format : {ip}:{port}, bad format : @@ -271,6 +287,14 @@ class MasterService { */ tl::expected GetStorageConfig() const; + /** + * @brief Get OpLogManager reference for external access + * @return Reference to the OpLogManager instance + */ + OpLogManager& GetOpLogManager(); + + // SetReplicationService removed - using etcd-based OpLog sync instead + /** * @brief Mounts a file storage segment into the master. * @param enable_offloading If true, enables offloading (write-to-file). @@ -301,6 +325,21 @@ class MasterService { -> tl::expected; private: + /** + * @brief Helper function to append OpLog entry + * @param type Operation type + * @param key Object key + * @param payload Optional payload data + */ + void AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Durable OpLog append: must succeed (write to etcd) for operations that may + // free/reuse memory (e.g. REMOVE). See OpLogManager::AppendAndPersist. + auto AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload = std::string()) + -> tl::expected; + // Resolve the key to a sanitized format for storage std::string SanitizeKey(const std::string& key) const; std::string ResolvePath(const std::string& key) const; @@ -477,6 +516,95 @@ class MasterService { } }; + /** + * @brief Serialize ObjectMetadata to JSON string for OpLog payload + * @param metadata The metadata to serialize + * @return JSON string containing the serialized metadata + */ + std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; + + // Serialize metadata but exclude MEMORY replicas. + // Used for eviction: when memory replicas are freed/reused, Standby must not + // keep stale memory descriptors. We persist a PUT_END containing only + // remaining (DISK/LOCAL_DISK) replicas before freeing memory. + std::string SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const; + + // Serialize metadata from a caller-provided replica descriptor list. + // This is used when we need to persist an updated replica set *before* + // mutating local replicas (which may free/reuse memory). + std::string SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const; + + // Pending durable mutations (etcd write retry queue) + // -------------------------------------------------- + // In HA mode, freeing/reusing MEMORY replicas before Standby observes the + // corresponding OpLog update can cause stale descriptors on Standby. + // If durable etcd write fails, we enqueue a pending mutation and retry + // asynchronously to avoid long-term memory retention. + enum class PendingMutationKind : uint8_t { + EVICT_MEM_REPLICAS = 1, // drop MEMORY replicas; persist PUT_END or REMOVE + CLEAR_ALL_REPLICAS = 2, // remove the whole key; persist REMOVE + CLEAR_REPLICAS_ON_SEGMENT = 3, // remove COMPLETE replicas on segment; persist PUT_END/REMOVE + }; + struct PendingMutation { + PendingMutationKind kind{PendingMutationKind::EVICT_MEM_REPLICAS}; + std::string key; + std::string segment_name; // only for CLEAR_REPLICAS_ON_SEGMENT + // OpLog entry to persist. If sequence_id==0, this is a deferred action and + // the worker will allocate a new OpLogEntry at execution time. + // If sequence_id>0, sequence_id is pre-allocated and MUST be persisted as-is + // (implements: "enqueue time seq_id fixed and smaller"). + OpLogEntry oplog_entry; + uint32_t attempt{0}; + std::chrono::steady_clock::time_point next_retry_at{}; + }; + + void EnqueuePendingMutation(PendingMutation m); + void PendingMutationWorker(); + bool ProcessPendingMutationOnce(PendingMutation& m); + + // Helper for etcd durable write (HA only): + // - Persist a pre-allocated OpLogEntry with small synchronous retries. + // - On failure, enqueue a PendingMutation (caller decides whether to proceed + // with local state changes; we do NOT block per-key). + ErrorCode PersistOpLogEntryWithSyncRetries(const OpLogEntry& entry) const; + void EnqueueRetryOnPersistFailure(const char* ctx, const OpLogEntry& entry, + ErrorCode persist_err, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Higher-level helper that also handles: + // - STORE_USE_ETCD compile-time switch + // - enable_ha_ runtime switch + // + // Behavior: + // - HA + STORE_USE_ETCD: AllocateEntry -> Persist (sync retries) -> enqueue on failure + // - non-HA: Append to in-memory OpLog buffer only + // - HA but STORE_USE_ETCD disabled: no-op (best-effort; see constructor warning) + void AppendOrPersistOrEnqueue(const char* ctx, OpType type, + const std::string& key, + const std::string& payload, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // Lazy-payload variant: payload is computed only when needed. + // This is useful to avoid expensive metadata serialization when: + // - HA is enabled but STORE_USE_ETCD is disabled at compile time (no-op), or + // - the branch will not publish OpLog at all. + void AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, + const std::string& segment_name = std::string()); + + // NOTE: + // We intentionally do NOT block subsequent operations for the same key when a + // durable OpLog write fails. Failed entries are retried asynchronously with the + // original pre-allocated sequence_id, and Standby handles gaps via timeout + + // late-arrival policy (apply late REMOVE/PUT_REVOKE, discard late PUT_END). + static constexpr size_t kNumShards = 1024; // Number of metadata shards // Sharded metadata maps and their mutexes @@ -631,8 +759,29 @@ class MasterService { // Segment management SegmentManager segment_manager_; BufferAllocatorType memory_allocator_type_; + + // Keep dummy allocators alive for memory replicas restored from standby. + // AllocatedBuffer stores allocator as weak_ptr; without an owning shared_ptr, + // the allocator would expire immediately and transport_endpoint_ would be lost + // when re-serializing Replica descriptors. + std::unordered_map> + standby_allocator_keepalive_; + + // Operation log manager for hot-standby replication. It records + // state-changing operations so that a standby master can replay them. + OpLogManager oplog_manager_; + + // ReplicationService removed - using etcd-based OpLog sync instead + std::shared_ptr allocation_strategy_; + // Pending durable mutation retry queue (HA only). + std::mutex pending_mutations_mutex_; + std::condition_variable pending_mutations_cv_; + std::deque pending_mutations_; + std::atomic pending_mutations_running_{false}; + std::thread pending_mutations_thread_; + // Discarded replicas management const std::chrono::seconds put_start_discard_timeout_sec_; const std::chrono::seconds put_start_release_timeout_sec_; diff --git a/mooncake-store/include/metadata_store.h b/mooncake-store/include/metadata_store.h new file mode 100644 index 0000000000..33842238f4 --- /dev/null +++ b/mooncake-store/include/metadata_store.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include + +#include "replica.h" +#include "types.h" +#include "ylt/struct_json/json_reader.h" +#include "ylt/struct_json/json_writer.h" + +namespace mooncake { + +/** + * @brief Metadata structure for Standby to store and restore object information + * + * This structure contains all essential metadata information needed by Standby + * to immediately serve as Primary when promoted. + */ +struct StandbyObjectMetadata { + UUID client_id{0, 0}; + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information is NOT stored because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + uint64_t last_sequence_id{0}; // Last OpLog sequence ID that modified this key + + StandbyObjectMetadata() = default; + + // Check if this metadata has valid replicas + bool HasReplicas() const { return !replicas.empty(); } +}; + +/** + * @brief Payload structure for JSON serialization/deserialization + * + * Uses separate fields for UUID since std::pair cannot be directly serialized. + */ +struct MetadataPayload { + uint64_t client_id_first{0}; // UUID.first + uint64_t client_id_second{0}; // UUID.second + uint64_t size{0}; + std::vector replicas; + // NOTE: Lease information removed - not needed by Standby + + YLT_REFL(MetadataPayload, client_id_first, client_id_second, size, replicas); + + // Convert to StandbyObjectMetadata + StandbyObjectMetadata ToStandbyMetadata(uint64_t sequence_id) const { + StandbyObjectMetadata meta; + meta.client_id = {client_id_first, client_id_second}; + meta.size = size; + meta.replicas = replicas; + meta.last_sequence_id = sequence_id; + return meta; + } +}; + +/** + * @brief Abstract interface for metadata storage on Standby + * + * This interface provides basic operations for storing and managing object metadata. + * In a full implementation, this would mirror MasterService's metadata_shards_ structure. + */ +class MetadataStore { + public: + virtual ~MetadataStore() = default; + + /** + * @brief Put or update metadata for a key with structured metadata + * @param key Object key + * @param metadata Structured metadata object + * @return true on success, false on failure + */ + virtual bool PutMetadata(const std::string& key, const StandbyObjectMetadata& metadata) = 0; + + /** + * @brief Put or update metadata for a key (legacy interface for backward compatibility) + * @param key Object key + * @param payload Optional payload data (JSON serialized metadata) + * @return true on success, false on failure + */ + virtual bool Put(const std::string& key, const std::string& payload = std::string()) = 0; + + /** + * @brief Get metadata for a key + * @param key Object key + * @return Pointer to metadata if found, nullptr otherwise + */ + virtual const StandbyObjectMetadata* GetMetadata(const std::string& key) const = 0; + + /** + * @brief Remove metadata for a key + * @param key Object key + * @return true if key was found and removed, false otherwise + */ + virtual bool Remove(const std::string& key) = 0; + + /** + * @brief Check if a key exists + * @param key Object key + * @return true if key exists, false otherwise + */ + virtual bool Exists(const std::string& key) const = 0; + + /** + * @brief Get the count of keys in the store + * @return Number of keys + */ + virtual size_t GetKeyCount() const = 0; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_applier.h b/mooncake-store/include/oplog_applier.h new file mode 100644 index 0000000000..cbab4623a8 --- /dev/null +++ b/mooncake-store/include/oplog_applier.h @@ -0,0 +1,169 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "metadata_store.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +/** + * @brief Apply OpLog entries to Standby metadata store with ordering guarantee + * + * This class applies OpLog entries to the Standby metadata store, + * ensuring both global and per-key ordering. + */ +class OpLogApplier { + public: + /** + * @brief Constructor + * @param metadata_store Metadata store to apply changes to + * @param cluster_id Cluster ID for accessing etcd OpLog (optional, for requesting missing OpLog) + */ + explicit OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id = std::string()); + + /** + * @brief Apply a single OpLog entry (with ordering checks) + * @param entry OpLog entry to apply + * @return true on success, false on failure or ordering violation + */ + bool ApplyOpLogEntry(const OpLogEntry& entry); + + /** + * @brief Apply multiple OpLog entries + * @param entries OpLog entries to apply + * @return Number of successfully applied entries + */ + size_t ApplyOpLogEntries(const std::vector& entries); + + /** + * @brief Get the current sequence ID for a key (DEPRECATED) + * @param key Object key + * @return Always returns 0 - key_sequence_id is no longer tracked + * @deprecated Use global sequence_id for ordering + */ + uint64_t GetKeySequenceId(const std::string& key) const; + + /** + * @brief Get the expected global sequence ID + * @return Expected global sequence ID + */ + uint64_t GetExpectedSequenceId() const; + + /** + * @brief Recover from a given sequence ID + * @param last_applied_sequence_id Last applied sequence ID + */ + void Recover(uint64_t last_applied_sequence_id); + + /** + * @brief Process pending entries (entries with non-continuous sequence IDs) + * @return Number of entries processed + */ + size_t ProcessPendingEntries(); + + // Promotion helper: + // Try to resolve current gaps ONCE (no waiting) by fetching missing/skipped + // sequence_ids from etcd. If an entry arrives late: + // - REMOVE / PUT_REVOKE: delete the key + // - PUT_END: discard + // + // This is used during Standby promotion so we don't block promotion on gaps, + // but still best-effort clean up potentially stale metadata. + struct GapResolveResult { + size_t attempted{0}; + size_t fetched{0}; + size_t applied_deletes{0}; + }; + GapResolveResult TryResolveGapsOnceForPromotion(size_t max_ids = 1024); + + private: + /** + * @brief Check if the entry's sequence order is valid + * @param entry OpLog entry + * @return true if order is valid, false otherwise + */ + bool CheckSequenceOrder(const OpLogEntry& entry); + + /** + * @brief Apply PUT_END operation + * @param entry OpLog entry + */ + void ApplyPutEnd(const OpLogEntry& entry); + + /** + * @brief Apply PUT_REVOKE operation + * @param entry OpLog entry + */ + void ApplyPutRevoke(const OpLogEntry& entry); + + /** + * @brief Apply REMOVE operation + * @param entry OpLog entry + */ + void ApplyRemove(const OpLogEntry& entry); + + /** + * @brief Request missing OpLog entry from etcd + * @param missing_seq_id Missing sequence ID + * @return true if entry was found and applied, false otherwise + */ + bool RequestMissingOpLog(uint64_t missing_seq_id); + + /** + * @brief Schedule wait for missing entries + * @param missing_seq_id Missing sequence ID + */ + void ScheduleWaitForMissingEntries(uint64_t missing_seq_id); + + MetadataStore* metadata_store_; + + // EtcdOpLogStore for requesting missing OpLog entries (optional) + std::string cluster_id_; + mutable std::mutex etcd_oplog_store_mutex_; + mutable std::unique_ptr etcd_oplog_store_; + + /** + * @brief Get or create EtcdOpLogStore instance (lazy initialization) + * @return Pointer to EtcdOpLogStore, or nullptr if cluster_id is not set + */ + EtcdOpLogStore* GetEtcdOpLogStore() const; + + // Note: key_sequence_map_ has been removed. + // Global sequence_id is sufficient for ordering guarantee. + + // Track pending entries (entries with non-continuous sequence IDs) + mutable std::mutex pending_mutex_; + std::map pending_entries_; + + // Track missing sequence IDs that we're waiting for + std::map missing_sequence_ids_; + + // Sequence IDs we chose to skip (gap-timeout). If the late entry arrives: + // - REMOVE / PUT_REVOKE: delete the key (safe) + // - PUT_END: discard (do not resurrect potentially stale metadata) + std::map skipped_sequence_ids_; + + // Next expected global sequence_id. Read frequently from monitoring thread, + // updated by watch/apply thread. Use atomic to avoid data races. + std::atomic expected_sequence_id_{1}; + + // Constants for missing entry handling + static constexpr int kMissingEntryWaitSeconds = 5; // Wait 5 seconds before requesting + static constexpr int kMissingEntrySkipSeconds = 3; // Wait 3 seconds then skip (avoid global stall) + static constexpr int kMaxPendingEntries = 1000; // Max pending entries before giving up +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/oplog_manager.h b/mooncake-store/include/oplog_manager.h new file mode 100644 index 0000000000..3d9a0504b5 --- /dev/null +++ b/mooncake-store/include/oplog_manager.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "types.h" + +namespace mooncake { + +// Forward declaration +class EtcdOpLogStore; + +// Operation types for hot-standby replication. +// This is a minimal subset that can be extended later. +enum class OpType : uint8_t { + PUT_END = 1, + PUT_REVOKE = 2, + REMOVE = 3, + // Deprecated: LEASE_RENEW is intentionally not recorded in OpLog in the + // current etcd-based hot-standby design (Standby relies on Primary DELETE operations). + LEASE_RENEW = 4, +}; + +// A single operation log entry. +// Note: Payload contains JSON serialized MetadataPayload (defined in metadata_store.h) +// for PUT_END operations, allowing Standby to restore complete metadata. +struct OpLogEntry { + uint64_t sequence_id{0}; // Monotonically increasing global sequence + uint64_t timestamp_ms{0}; // Logical timestamp in milliseconds + OpType op_type{OpType::PUT_END}; + std::string object_key; // Target object key + std::string payload; // Serialized extra data (optional) + uint32_t checksum{0}; // Checksum of payload (implementation-defined) + uint32_t prefix_hash{0}; // Hash of the entire key (for verification and optimization) + // Deprecated: key_sequence_id is kept for backward compatibility only. + // Ordering is guaranteed by global sequence_id. + uint64_t key_sequence_id{0}; +}; + +/** + * @brief In-memory operation log manager. + * + * This class is intentionally simple: it keeps a bounded deque of OpLogEntry + * and provides append / get-since primitives. It can later be extended to + * or to spill to disk if needed. In the new etcd-based design, OpLog will be written to etcd. + */ +class OpLogManager { + public: + OpLogManager(); + + // Set the EtcdOpLogStore for writing OpLog to etcd (optional). + // If not set, OpLog will only be stored in memory buffer. + void SetEtcdOpLogStore(std::shared_ptr etcd_oplog_store); + + // Append a new entry and return the assigned sequence_id. + uint64_t Append(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Allocate a new OpLogEntry with a reserved sequence_id, append it to the + // in-memory buffer, and return the full entry. + // + // IMPORTANT: This will advance last_seq_id_ even if the caller later fails + // to persist it to etcd. This supports "seq pre-allocation" semantics where + // retries use the same (smaller) sequence_id. + OpLogEntry AllocateEntry(OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Persist an already-allocated entry to etcd using its sequence_id. + // Does NOT modify sequence counters. + ErrorCode PersistEntryToEtcd(const OpLogEntry& entry) const; + + // Append a new entry and durably persist it to etcd (if EtcdOpLogStore is set). + // + // This is intended for operations that may free/reuse memory (e.g. REMOVE), + // where best-effort replication is unsafe: Standby must observe the DELETE + // before promotion, otherwise it may return stale descriptors that point to + // reused memory and cause silent data corruption. + // + // Design (updated for seq pre-allocation): + // - sequence_id is allocated first and never reused. + // - If etcd write fails, caller may retry PersistEntryToEtcd with the same + // entry (sequence_id fixed and "smaller" than later entries). + tl::expected AppendAndPersist( + OpType type, const std::string& key, + const std::string& payload = std::string()); + + // Get the latest assigned sequence id. Returns 0 if no entry exists. + uint64_t GetLastSequenceId() const; + + // Set the initial sequence ID (used when promoting Standby to Primary). + // This ensures the new Primary's OpLogManager continues from the correct sequence_id. + void SetInitialSequenceId(uint64_t sequence_id); + + // Current number of entries in the buffer. + size_t GetEntryCount() const; + + + private: + static uint64_t NowMs(); + static uint32_t ComputeChecksum(const std::string& data); + static uint32_t ComputePrefixHash(const std::string& key); + + mutable std::shared_mutex mutex_; + std::deque buffer_; + uint64_t first_seq_id_{1}; // sequence_id of buffer_.front() + uint64_t last_seq_id_{0}; // last assigned sequence_id + + // Note: We removed key_sequence_map_ and key_remove_time_map_. + // Global sequence_id is sufficient for ordering guarantee. + // All operations are applied in sequence_id order, which ensures consistency. + + // Optional etcd OpLog store for persistent storage + std::shared_ptr etcd_oplog_store_; + + // Simple bounds to avoid unbounded memory growth. + static constexpr size_t kMaxBufferEntries_ = 100000; +}; + +} // namespace mooncake + + diff --git a/mooncake-store/include/oplog_watcher.h b/mooncake-store/include/oplog_watcher.h new file mode 100644 index 0000000000..f0be684fd2 --- /dev/null +++ b/mooncake-store/include/oplog_watcher.h @@ -0,0 +1,149 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "oplog_manager.h" +#include "types.h" + +namespace mooncake { + +// Forward declaration +class OpLogApplier; + +/** + * @brief Watch etcd for OpLog changes and apply them to Standby + * + * This class watches etcd for new OpLog entries and forwards them + * to OpLogApplier for processing. + */ +class OpLogWatcher { + public: + /** + * @brief Constructor + * @param etcd_endpoints Comma-separated etcd endpoints + * @param cluster_id Cluster identifier + * @param applier OpLog applier to process entries + */ + OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier); + + ~OpLogWatcher(); + + /** + * @brief Start watching etcd for OpLog changes + */ + void Start(); + + /** + * @brief Start from a known last-applied sequence_id. + * + * It will read historical OpLogs at a consistent etcd revision, then start + * watch from revision+1 to close the gap between "read" and "watch". + */ + bool StartFromSequenceId(uint64_t start_seq_id); + + /** + * @brief Stop watching + */ + void Stop(); + + /** + * @brief Read OpLog entries from etcd since a given sequence ID + * @param start_seq_id Starting sequence ID (exclusive) + * @param entries Output vector of OpLog entries + * @return true on success, false on failure + */ + bool ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries); + + /** + * @brief Get the last processed sequence ID + * @return Last processed sequence ID + */ + uint64_t GetLastProcessedSequenceId() const; + + private: + bool ReadOpLogSinceWithRevision(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id); + /** + * @brief Static callback function for etcd Watch + * @param context OpLogWatcher instance (passed as void*) + * @param key etcd key + * @param key_size key size + * @param value etcd value + * @param value_size value size + * @param event_type event type (0 = PUT, 1 = DELETE) + */ + static void WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type); + + // V2 callback includes etcd KV mod_revision for precise resume. + static void WatchCallbackV2(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type, + int64_t mod_revision); + + /** + * @brief Watch etcd OpLog changes (runs in background thread) + */ + void WatchOpLog(); + + /** + * @brief Process a Watch event + * @param key etcd key + * @param value etcd value (JSON string for PUT events, empty for DELETE events) + * @param event_type Event type (0 = PUT, 1 = DELETE) + */ + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type); + void HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision); + + /** + * @brief Deserialize OpLogEntry from JSON string + * @param json_str JSON string + * @param entry Output OpLog entry + * @return true on success, false on failure + */ + bool DeserializeOpLogEntry(const std::string& json_str, OpLogEntry& entry); + + /** + * @brief Attempt to reconnect after watch failure + */ + void TryReconnect(); + + /** + * @brief Sync missed OpLog entries after reconnection + * @return true if sync was successful + */ + bool SyncMissedEntries(); + + // Next watch revision (0 means from now). Updated by consistent reads. + std::atomic next_watch_revision_{0}; + + std::string etcd_endpoints_; + std::string cluster_id_; + OpLogApplier* applier_; + std::atomic running_{false}; + std::thread watch_thread_; + std::atomic last_processed_sequence_id_{0}; + + // Error handling and recovery + std::atomic consecutive_errors_{0}; + std::atomic reconnect_count_{0}; + std::atomic watch_healthy_{false}; + + // Constants for error handling + static constexpr int kMaxConsecutiveErrors = 10; + static constexpr int kReconnectDelayMs = 1000; + static constexpr int kMaxReconnectDelayMs = 30000; + static constexpr int kSyncBatchSize = 1000; +}; + +} // namespace mooncake + diff --git a/mooncake-store/include/replica.h b/mooncake-store/include/replica.h index 5793e90f96..997259ca0b 100644 --- a/mooncake-store/include/replica.h +++ b/mooncake-store/include/replica.h @@ -131,10 +131,21 @@ struct DiskDescriptor { }; struct LocalDiskDescriptor { - UUID client_id; + uint64_t client_id_first{0}; // UUID.first - split for JSON serialization + uint64_t client_id_second{0}; // UUID.second - split for JSON serialization uint64_t object_size = 0; std::string transport_endpoint; - YLT_REFL(LocalDiskDescriptor, client_id, object_size, transport_endpoint); + + // Constructor from UUID for convenience + LocalDiskDescriptor() = default; + LocalDiskDescriptor(UUID client_id, uint64_t object_size, const std::string& transport_endpoint) + : client_id_first(client_id.first), client_id_second(client_id.second), + object_size(object_size), transport_endpoint(transport_endpoint) {} + + // Get UUID (for backward compatibility) + UUID GetClientId() const { return {client_id_first, client_id_second}; } + + YLT_REFL(LocalDiskDescriptor, client_id_first, client_id_second, object_size, transport_endpoint); }; class Replica { @@ -375,10 +386,9 @@ inline Replica::Descriptor Replica::get_descriptor() const { desc.descriptor_variant = std::move(disk_desc); } else if (is_local_disk_replica()) { const auto& disk_data = std::get(data_); - LocalDiskDescriptor local_disk_desc; - local_disk_desc.client_id = disk_data.client_id; - local_disk_desc.object_size = disk_data.object_size; - local_disk_desc.transport_endpoint = disk_data.transport_endpoint; + LocalDiskDescriptor local_disk_desc(disk_data.client_id, + disk_data.object_size, + disk_data.transport_endpoint); desc.descriptor_variant = std::move(local_disk_desc); } diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 1cf5d7a24f..465a6d5661 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -4,15 +4,22 @@ #include #include #include +#include #include #include #include #include "master_service.h" +#include "metadata_store.h" #include "types.h" #include "rpc_types.h" #include "master_config.h" +// Forward declaration +namespace mooncake { +// ReplicationService forward declaration removed - using etcd-based OpLog sync instead +} + namespace mooncake { extern const uint64_t kMetricReportIntervalSeconds; @@ -25,6 +32,11 @@ class WrappedMasterService { void init_http_server(); + // Restore metadata and OpLog sequence from a promoted Standby (fast failover). + void RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id); + tl::expected ExistKey(const std::string& key); tl::expected @@ -109,11 +121,15 @@ class WrappedMasterService { const UUID& client_id, const std::vector& keys, const std::vector& metadatas); + // GetReplicationService removed - using etcd-based OpLog sync instead + private: MasterService master_service_; std::thread metric_report_thread_; coro_http::coro_http_server http_server_; std::atomic metric_report_running_; + + // ReplicationService removed - using etcd-based OpLog sync instead }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/include/snapshot_provider.h b/mooncake-store/include/snapshot_provider.h new file mode 100644 index 0000000000..ae8041511c --- /dev/null +++ b/mooncake-store/include/snapshot_provider.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "metadata_store.h" + +namespace mooncake { + +/** + * @brief SnapshotProvider is an abstraction for loading metadata snapshots. + * + * Assumption: snapshot functionality exists (implemented by another team), but + * may not be synced into this repo yet. We keep Mooncake-store code progressing + * by depending on this narrow interface. + * + * Snapshot semantics for hot-standby: + * - A snapshot represents a consistent metadata baseline at `snapshot_sequence_id`. + * - Standby should: load snapshot -> recover applier to snapshot_sequence_id -> + * replay OpLog entries with sequence_id > snapshot_sequence_id. + */ +class SnapshotProvider { + public: + virtual ~SnapshotProvider() = default; + + // Load the latest available snapshot for `cluster_id`. + // Returns true on success and fills: + // - snapshot_id: opaque identifier (e.g. timestamp/version) + // - snapshot_sequence_id: global OpLog sequence_id at snapshot boundary + // - snapshot: full metadata baseline as key -> StandbyObjectMetadata + virtual bool LoadLatestSnapshot( + const std::string& cluster_id, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) = 0; +}; + +// Default no-op provider: behaves as if "no snapshot available". +class NoopSnapshotProvider final : public SnapshotProvider { + public: + bool LoadLatestSnapshot( + const std::string& /*cluster_id*/, std::string& snapshot_id, + uint64_t& snapshot_sequence_id, + std::vector>& snapshot) override { + snapshot_id.clear(); + snapshot_sequence_id = 0; + snapshot.clear(); + return false; + } +}; + +} // namespace mooncake + + diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 05968b56df..8f726fd780 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -16,8 +16,6 @@ set(MOONCAKE_STORE_SOURCES ha_helper.cpp segment.cpp transfer_task.cpp - etcd_helper.cpp - ha_helper.cpp rpc_service.cpp offset_allocator.cpp posix_file.cpp @@ -26,10 +24,26 @@ set(MOONCAKE_STORE_SOURCES dummy_client.cpp http_metadata_server.cpp file_storage.cpp + oplog_manager.cpp + etcd_oplog_store.cpp + oplog_watcher.cpp + oplog_applier.cpp + hot_standby_service.cpp + # replication_service.cpp removed - using etcd-based OpLog sync instead ) set(EXTRA_LIBS "") +# Find xxHash (required for ComputeChecksum) +find_path(XXHASH_INCLUDE_DIR NAMES xxhash.h PATHS /usr/include /usr/local/include) +find_library(XXHASH_LIBRARY NAMES xxhash libxxhash PATHS /usr/lib /usr/local/lib /usr/lib64) +if (XXHASH_INCLUDE_DIR AND XXHASH_LIBRARY) + message(STATUS "Found xxHash: include=${XXHASH_INCLUDE_DIR} lib=${XXHASH_LIBRARY}") + list(APPEND MASTER_EXTRA_INCS ${XXHASH_INCLUDE_DIR}) +else() + message(FATAL_ERROR "xxHash library/header not found. Please install xxhash (development headers) and try again.") +endif() + if(USE_3FS) add_subdirectory(hf3fs) list(APPEND MOONCAKE_STORE_SOURCES ${HF3FS_SOURCES}) @@ -43,6 +57,8 @@ endif() # The cache_allocator library include_directories(${Python3_INCLUDE_DIRS}) add_library(mooncake_store ${MOONCAKE_STORE_SOURCES}) +target_include_directories(mooncake_store PUBLIC ${XXHASH_INCLUDE_DIR}) +target_link_libraries(mooncake_store PUBLIC ${XXHASH_LIBRARY}) target_link_libraries(mooncake_store PUBLIC transfer_engine cachelib_memory_allocator ${ETCD_WRAPPER_LIB} glog::glog gflags::gflags ${EXTRA_LIBS} ) diff --git a/mooncake-store/src/etcd_helper.cpp b/mooncake-store/src/etcd_helper.cpp index 5417de5afc..df34f93945 100644 --- a/mooncake-store/src/etcd_helper.cpp +++ b/mooncake-store/src/etcd_helper.cpp @@ -152,6 +152,171 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { } return ErrorCode::OK; } + +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + char* err_msg = nullptr; + int ret = EtcdStorePutWrapper((char*)key, (int)key_size, (char*)value, + (int)value_size, &err_msg); + if (ret != 0) { + LOG(ERROR) << "key=" << std::string(key, key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + // TODO: Implement GetWithPrefix - need to simplify Go wrapper interface first + // For now, return error as this requires complex memory management + LOG(ERROR) << "GetWithPrefix not yet implemented - requires Go wrapper interface simplification"; + return ErrorCode::INTERNAL_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + char* err_msg = nullptr; + char* json_ptr = nullptr; + int json_size = 0; + // Go wrapper takes int limit. + int ret = EtcdStoreGetRangeAsJsonWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + (int)limit, &json_ptr, &json_size, + (GoInt64*)&revision_id, &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + json = std::string(json_ptr, json_size); + free(json_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + char* err_msg = nullptr; + char* first_key_ptr = nullptr; + int first_key_size = 0; + int ret = EtcdStoreGetFirstKeyWithPrefixWrapper((char*)prefix, (int)prefix_size, + &first_key_ptr, &first_key_size, + &err_msg); + if (ret == -2) { + free(err_msg); + return ErrorCode::ETCD_KEY_NOT_EXIST; + } + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + first_key = std::string(first_key_ptr, first_key_size); + free(first_key_ptr); + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + char* err_msg = nullptr; + int ret = EtcdStoreDeleteRangeWrapper((char*)start_key, (int)start_key_size, + (char*)end_key, (int)end_key_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "start_key=" << std::string(start_key, start_key_size) + << ", end_key=" << std::string(end_key, end_key_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + char* err_msg = nullptr; + // Convert function pointer to void* for passing to Go function + // Note: This is safe because we're just passing the pointer, not calling it + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + callback_context, callback_func_ptr, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionWrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + char* err_msg = nullptr; + void* callback_func_ptr = reinterpret_cast(callback_func); + int ret = EtcdStoreWatchWithPrefixFromRevisionV2Wrapper( + (char*)prefix, (int)prefix_size, (GoInt64)start_revision, callback_context, + callback_func_ptr, &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", start_revision=" << (int64_t)start_revision + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + char* err_msg = nullptr; + int ret = EtcdStoreCancelWatchWithPrefixWrapper((char*)prefix, (int)prefix_size, + &err_msg); + if (ret != 0) { + LOG(ERROR) << "prefix=" << std::string(prefix, prefix_size) + << ", error=" << err_msg; + free(err_msg); + return ErrorCode::ETCD_OPERATION_ERROR; + } + return ErrorCode::OK; +} #else ErrorCode EtcdHelper::ConnectToEtcdStoreClient( const std::string& etcd_endpoints) { @@ -200,6 +365,92 @@ ErrorCode EtcdHelper::CancelKeepAlive(EtcdLeaseId lease_id) { return ErrorCode::ETCD_OPERATION_ERROR; } +ErrorCode EtcdHelper::Put(const char* key, const size_t key_size, + const char* value, const size_t value_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetWithPrefix(const char* prefix, const size_t prefix_size, + std::vector& keys, + std::vector& values) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::GetRangeAsJson(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size, + size_t limit, + std::string& json, + EtcdRevisionId& revision_id) { + (void)start_key; + (void)start_key_size; + (void)end_key; + (void)end_key_size; + (void)limit; + (void)json; + (void)revision_id; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} +ErrorCode EtcdHelper::GetFirstKeyWithPrefix(const char* prefix, + const size_t prefix_size, + std::string& first_key) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::DeleteRange(const char* start_key, + const size_t start_key_size, + const char* end_key, + const size_t end_key_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefix(const char* prefix, const size_t prefix_size, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, + const char*, size_t, int)) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevision( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::WatchWithPrefixFromRevisionV2( + const char* prefix, const size_t prefix_size, EtcdRevisionId start_revision, + void* callback_context, + void (*callback_func)(void*, const char*, size_t, const char*, size_t, int, + int64_t)) { + (void)prefix; + (void)prefix_size; + (void)start_revision; + (void)callback_context; + (void)callback_func; + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + +ErrorCode EtcdHelper::CancelWatchWithPrefix(const char* prefix, + const size_t prefix_size) { + LOG(FATAL) << "Etcd is not enabled in compilation"; + return ErrorCode::ETCD_OPERATION_ERROR; +} + #endif } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/etcd_oplog_store.cpp b/mooncake-store/src/etcd_oplog_store.cpp new file mode 100644 index 0000000000..cc0775200c --- /dev/null +++ b/mooncake-store/src/etcd_oplog_store.cpp @@ -0,0 +1,450 @@ +#include "etcd_oplog_store.h" + +#include +#include +#include + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +#include "etcd_helper.h" + +namespace mooncake { + +EtcdOpLogStore::EtcdOpLogStore(const std::string& cluster_id, + bool enable_latest_seq_batch_update) + : cluster_id_(cluster_id), + enable_latest_seq_batch_update_(enable_latest_seq_batch_update), + last_update_time_(std::chrono::steady_clock::now()) { + // Normalize cluster_id to avoid accidental double slashes in etcd keys when + // caller passes a trailing '/' (master_view_key uses trailing '/', OpLog keys don't). + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } + // Start batch update thread only for writers. + if (enable_latest_seq_batch_update_) { + batch_update_running_.store(true); + batch_update_thread_ = + std::thread(&EtcdOpLogStore::BatchUpdateThread, this); + } +} + +EtcdOpLogStore::~EtcdOpLogStore() { + if (!enable_latest_seq_batch_update_) { + return; + } + + // Stop batch update thread + batch_update_running_.store(false); + if (batch_update_thread_.joinable()) { + batch_update_thread_.join(); + } + + // Perform final update if there are pending updates + if (pending_count_.load() > 0) { + DoBatchUpdate(); + } +} + +ErrorCode EtcdOpLogStore::WriteOpLog(const OpLogEntry& entry) { + std::string key = BuildOpLogKey(entry.sequence_id); + std::string value = SerializeOpLogEntry(entry); + + ErrorCode err = EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), + value.size()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to write OpLog entry, sequence_id=" + << entry.sequence_id; + return err; + } + + // Update `/latest`. + // - Writers: batch update to reduce etcd write pressure. + // - Readers / tests: update immediately for simplicity. + if (!enable_latest_seq_batch_update_) { + return UpdateLatestSequenceId(entry.sequence_id); + } + + pending_latest_seq_id_.store(entry.sequence_id); + size_t count = pending_count_.fetch_add(1) + 1; + if (count >= kBatchSize) { + DoBatchUpdate(); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLog(uint64_t sequence_id, + OpLogEntry& entry) { + std::string key = BuildOpLogKey(sequence_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry, sequence_id=" + << sequence_id; + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::ReadOpLogSince(uint64_t start_sequence_id, + size_t limit, + std::vector& entries) { + EtcdRevisionId rev = 0; + return ReadOpLogSinceWithRevision(start_sequence_id, limit, entries, rev); +} + +ErrorCode EtcdOpLogStore::ReadOpLogSinceWithRevision(uint64_t start_sequence_id, + size_t limit, + std::vector& entries, + EtcdRevisionId& revision_id) { + entries.clear(); + entries.reserve(limit); + + // Range is limited to OpLog entry keys only. + const std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string current_start_key = BuildOpLogKey(start_sequence_id + 1); + + // Compute prefix range end (etcd prefix end). + auto prefix_end = [](std::string p) -> std::string { + for (int i = static_cast(p.size()) - 1; i >= 0; --i) { + unsigned char c = static_cast(p[i]); + if (c < 0xFF) { + p[i] = static_cast(c + 1); + p.resize(i + 1); + return p; + } + } + return std::string(1, '\0'); + }; + const std::string end_key = prefix_end(prefix); + + // Pagination: + // - Use range-get with limit + // - Start next page from lastKey + '\0' (lexicographically just after lastKey) + // This avoids repeating the last key without adding new Go/C++ APIs. + revision_id = 0; + while (entries.size() < limit) { + const size_t page_limit = limit - entries.size(); + std::string json; + EtcdRevisionId page_rev = 0; + ErrorCode err = + EtcdHelper::GetRangeAsJson(current_start_key.c_str(), + current_start_key.size(), end_key.c_str(), + end_key.size(), page_limit, json, page_rev); + if (err != ErrorCode::OK) { + return err; + } + if (page_rev > revision_id) { + revision_id = page_rev; + } + + // Parse kv list: [{"key":"...","value":"..."}] + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json); + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse range JSON: " << errs; + return ErrorCode::INTERNAL_ERROR; + } + if (!root.isArray()) { + return ErrorCode::INTERNAL_ERROR; + } + if (root.empty()) { + break; // no more data + } + + std::string last_key_in_page; + for (const auto& kv : root) { + const std::string key = kv.get("key", "").asString(); + last_key_in_page = key; + if (key.empty() || key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + continue; + } + + // Parse seq from key suffix and filter (handles legacy keys too). + size_t pos = key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= key.size()) { + continue; + } + uint64_t seq = 0; + try { + seq = static_cast(std::stoull(key.substr(pos + 1))); + } catch (...) { + continue; + } + if (seq <= start_sequence_id) { + continue; + } + + OpLogEntry entry; + const std::string value = kv.get("value", "").asString(); + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key=" << key; + return ErrorCode::INTERNAL_ERROR; + } + entries.push_back(std::move(entry)); + if (entries.size() >= limit) { + break; + } + } + + // Advance start key for next page. + if (last_key_in_page.empty()) { + break; + } + current_start_key = last_key_in_page; + current_start_key.push_back('\0'); + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::GetLatestSequenceId(uint64_t& sequence_id) { + std::string key = BuildLatestKey(); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse latest sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::UpdateLatestSequenceId(uint64_t sequence_id) { + std::string key = BuildLatestKey(); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::RecordSnapshotSequenceId( + const std::string& snapshot_id, uint64_t sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value = std::to_string(sequence_id); + return EtcdHelper::Put(key.c_str(), key.size(), value.c_str(), value.size()); +} + +ErrorCode EtcdOpLogStore::GetSnapshotSequenceId( + const std::string& snapshot_id, uint64_t& sequence_id) { + std::string key = BuildSnapshotKey(snapshot_id); + std::string value; + EtcdRevisionId revision_id; + ErrorCode err = EtcdHelper::Get(key.c_str(), key.size(), value, revision_id); + if (err != ErrorCode::OK) { + return err; + } + + try { + sequence_id = std::stoull(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to parse snapshot sequence_id: " << e.what(); + return ErrorCode::INTERNAL_ERROR; + } + + return ErrorCode::OK; +} + +ErrorCode EtcdOpLogStore::CleanupOpLogBefore(uint64_t before_sequence_id) { + // Robust cleanup (Scheme 3): + // - Determine current minimum sequence_id in etcd + // - DeleteRange [min_key, before_key) + // + // IMPORTANT: This relies on lexicographical ordering of keys, so the + // sequence_id portion MUST be fixed-width (zero-padded). + auto min_seq_opt = GetMinSequenceId(); + if (!min_seq_opt.has_value()) { + return ErrorCode::OK; // nothing to cleanup + } + + uint64_t min_seq = min_seq_opt.value(); + if (before_sequence_id <= min_seq) { + return ErrorCode::OK; + } + + std::string start_key = BuildOpLogKey(min_seq); + std::string end_key = BuildOpLogKey(before_sequence_id); // delete < before_sequence_id + + return EtcdHelper::DeleteRange(start_key.c_str(), start_key.size(), + end_key.c_str(), end_key.size()); +} + +std::string EtcdOpLogStore::BuildOpLogKey(uint64_t sequence_id) const { + std::ostringstream oss; + // Fixed-width encoding for correct etcd lexicographical range operations. + // 20 digits is enough for uint64_t max (18446744073709551615). + oss << kOpLogPrefix << cluster_id_ << "/" + << std::setw(20) << std::setfill('0') << sequence_id; + return oss.str(); +} + +std::optional EtcdOpLogStore::GetMinSequenceId() const { + std::string prefix = std::string(kOpLogPrefix) + cluster_id_ + "/"; + std::string first_key; + ErrorCode err = + EtcdHelper::GetFirstKeyWithPrefix(prefix.c_str(), prefix.size(), first_key); + if (err != ErrorCode::OK) { + return std::nullopt; + } + + // Skip non-entry keys if any (e.g. "/latest" or "/snapshot/..."). + // Entries are expected to be ".../<20-digit-seq>". + // If the first key isn't an entry key, fall back to nullopt (safe no-op). + if (first_key.find("/latest") != std::string::npos || + first_key.find("/snapshot/") != std::string::npos) { + return std::nullopt; + } + + size_t pos = first_key.rfind('/'); + if (pos == std::string::npos || pos + 1 >= first_key.size()) { + return std::nullopt; + } + std::string seq_str = first_key.substr(pos + 1); + try { + return static_cast(std::stoull(seq_str)); + } catch (...) { + return std::nullopt; + } +} + +std::string EtcdOpLogStore::BuildLatestKey() const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kLatestSuffix; + return oss.str(); +} + +std::string EtcdOpLogStore::BuildSnapshotKey( + const std::string& snapshot_id) const { + std::ostringstream oss; + oss << kOpLogPrefix << cluster_id_ << kSnapshotSuffix << snapshot_id + << "/sequence_id"; + return oss.str(); +} + +std::string EtcdOpLogStore::SerializeOpLogEntry( + const OpLogEntry& entry) const { + Json::Value root; + root["sequence_id"] = static_cast(entry.sequence_id); + root["timestamp_ms"] = static_cast(entry.timestamp_ms); + root["op_type"] = static_cast(entry.op_type); + root["object_key"] = entry.object_key; + root["payload"] = entry.payload; + root["checksum"] = static_cast(entry.checksum); + root["prefix_hash"] = static_cast(entry.prefix_hash); + root["key_sequence_id"] = static_cast(entry.key_sequence_id); + + Json::StreamWriterBuilder builder; + builder["indentation"] = ""; // Compact format + std::unique_ptr writer(builder.newStreamWriter()); + std::ostringstream oss; + writer->write(root, &oss); + return oss.str(); +} + +bool EtcdOpLogStore::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) const { + Json::Value root; + Json::CharReaderBuilder builder; + std::unique_ptr reader(builder.newCharReader()); + std::string errors; + + if (!reader->parse(json_str.data(), json_str.data() + json_str.size(), + &root, &errors)) { + LOG(ERROR) << "Failed to parse JSON: " << errors; + return false; + } + + try { + entry.sequence_id = root["sequence_id"].asUInt64(); + entry.timestamp_ms = root["timestamp_ms"].asUInt64(); + entry.op_type = static_cast(root["op_type"].asInt()); + entry.object_key = root["object_key"].asString(); + entry.payload = root["payload"].asString(); + entry.checksum = root["checksum"].asUInt(); + entry.prefix_hash = root["prefix_hash"].asUInt(); + entry.key_sequence_id = root["key_sequence_id"].asUInt64(); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to deserialize OpLogEntry: " << e.what(); + return false; + } + + return true; +} + +void EtcdOpLogStore::BatchUpdateThread() { + if (!enable_latest_seq_batch_update_) { + return; + } + while (batch_update_running_.load()) { + std::this_thread::sleep_for( + std::chrono::milliseconds(kBatchIntervalMs)); + + // Check if we need to update based on time interval + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + now - last_update_time_).count(); + + if (pending_count_.load() > 0 && elapsed >= kBatchIntervalMs) { + DoBatchUpdate(); + } + } +} + +void EtcdOpLogStore::TriggerBatchUpdateIfNeeded() { + // This method is kept for potential future use (e.g., manual trigger) + // Currently, DoBatchUpdate() is called directly from WriteOpLog + // when batch size threshold is reached + if (pending_count_.load() >= kBatchSize) { + DoBatchUpdate(); + } +} + +void EtcdOpLogStore::DoBatchUpdate() { + if (!enable_latest_seq_batch_update_) { + return; + } + std::lock_guard lock(batch_update_mutex_); + + // Get the pending sequence_id and reset counters + uint64_t seq_id_to_update = pending_latest_seq_id_.load(); + size_t count = pending_count_.exchange(0); + + if (count == 0) { + return; // Nothing to update + } + + // Update latest_sequence_id in etcd + ErrorCode err = UpdateLatestSequenceId(seq_id_to_update); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to batch update latest_sequence_id=" + << seq_id_to_update << ", error=" << err + << ". Will retry in next batch."; + // Restore the count so it will be retried + pending_count_.fetch_add(count); + } else { + last_update_time_ = std::chrono::steady_clock::now(); + VLOG(2) << "Batch updated latest_sequence_id=" << seq_id_to_update + << " (count=" << count << " entries)"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/ha_helper.cpp b/mooncake-store/src/ha_helper.cpp index 07906772af..5a50cb4013 100644 --- a/mooncake-store/src/ha_helper.cpp +++ b/mooncake-store/src/ha_helper.cpp @@ -1,5 +1,12 @@ #include "ha_helper.h" + +#include + +#include +#include + #include "etcd_helper.h" +#include "hot_standby_service.h" #include "rpc_service.h" namespace mooncake { @@ -131,13 +138,78 @@ int MasterServiceSupervisor::Start() { << config_.etcd_endpoints; return -1; } - LOG(INFO) << "Trying to elect self as leader..."; + +#ifdef STORE_USE_ETCD + // Connect to etcd for OpLog sync + if (EtcdHelper::ConnectToEtcdStoreClient(config_.etcd_endpoints.c_str()) != + ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd store client: " + << config_.etcd_endpoints; + return -1; + } +#endif + + LOG(INFO) << "Checking for existing leader..."; EtcdLeaseId lease_id = 0; - // view_version will be updated by ElectLeader and then used in - // WrappedMasterService ViewVersionId view_version = 0; + + // Check if there is already a leader + std::string current_leader; + ViewVersionId current_version = 0; + auto ret = mv_helper.GetMasterView(current_leader, current_version); + bool had_standby = false; + + if (ret == ErrorCode::OK) { + // There is an existing leader, start Standby service + LOG(INFO) << "Found existing leader: " << current_leader + << ", starting Standby service..."; + StartStandbyService(mv_helper, current_leader); + had_standby = true; + + // Build master_view_key (same logic as MasterViewHelper) + std::string cluster_id = config_.cluster_id; + if (!cluster_id.empty() && cluster_id.back() != '/') { + cluster_id += '/'; + } + std::string master_view_key = "mooncake-store/" + cluster_id + "master_view"; + + // Watch until leader is deleted + LOG(INFO) << "Watching for leadership change..."; + auto watch_ret = EtcdHelper::WatchUntilDeleted( + master_view_key.c_str(), master_view_key.size()); + + if (watch_ret != ErrorCode::OK) { + LOG(ERROR) << "Error watching for leadership change: " << watch_ret; + // Stop Standby service on watch error and retry. + StopStandbyService(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + LOG(INFO) << "Leader disappeared, trying to elect self as leader..."; + } else { + LOG(INFO) << "No existing leader found, trying to elect self as leader..."; + } + + // Try to elect self as leader mv_helper.ElectLeader(config_.local_hostname, view_version, lease_id); + // If we were running as Standby, finalize catch-up and snapshot metadata now. + std::vector> standby_snapshot; + uint64_t standby_last_seq_id = 0; +#ifdef STORE_USE_ETCD + if (had_standby && standby_service_ && standby_running_.load()) { + LOG(INFO) << "Finalizing standby state for promotion..."; + standby_service_->Promote(); // does final catch-up sync + stops watcher + standby_last_seq_id = standby_service_->GetLatestAppliedSequenceId(); + standby_service_->ExportMetadataSnapshot(standby_snapshot); + // We are now leader; standby service is no longer needed. + StopStandbyService(); + LOG(INFO) << "Standby snapshot ready: keys=" << standby_snapshot.size() + << ", last_seq_id=" << standby_last_seq_id; + } +#endif + // Start a thread to keep the leader alive auto keep_leader_thread = std::thread([&server, &mv_helper, lease_id]() { @@ -154,6 +226,14 @@ int MasterServiceSupervisor::Start() { LOG(INFO) << "Starting master service..."; mooncake::WrappedMasterService wrapped_master_service( mooncake::WrappedMasterServiceConfig(config_, view_version)); + + // Restore from promoted standby snapshot if available. +#ifdef STORE_USE_ETCD + if (standby_last_seq_id > 0 || !standby_snapshot.empty()) { + wrapped_master_service.RestoreFromStandby(standby_snapshot, standby_last_seq_id); + } +#endif + mooncake::RegisterRpcService(server, wrapped_master_service); // Metric reporting is now handled by WrappedMasterService. @@ -186,7 +266,56 @@ int MasterServiceSupervisor::Start() { return 0; } +void MasterServiceSupervisor::StartStandbyService(MasterViewHelper& mv_helper, + const std::string& current_leader) { +#ifdef STORE_USE_ETCD + if (standby_running_.load()) { + LOG(WARNING) << "Standby service is already running"; + return; + } + + HotStandbyConfig standby_config; + standby_config.standby_id = config_.local_hostname; + standby_config.primary_address = current_leader; + standby_config.verification_interval_sec = 30; + standby_config.max_replication_lag_entries = 1000; + standby_config.enable_verification = false; // Disable verification for now + + standby_service_ = std::make_unique(standby_config); + + ErrorCode err = standby_service_->Start( + current_leader, config_.etcd_endpoints, config_.cluster_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start Standby service: " << err; + standby_service_.reset(); + return; + } + + standby_running_.store(true); + LOG(INFO) << "Standby service started successfully"; +#else + LOG(WARNING) << "STORE_USE_ETCD is not enabled, cannot start Standby service"; +#endif +} + +void MasterServiceSupervisor::StopStandbyService() { +#ifdef STORE_USE_ETCD + if (!standby_running_.load()) { + return; + } + + if (standby_service_) { + standby_service_->Stop(); + standby_service_.reset(); + } + + standby_running_.store(false); + LOG(INFO) << "Standby service stopped"; +#endif +} + MasterServiceSupervisor::~MasterServiceSupervisor() { + StopStandbyService(); if (server_thread_.joinable()) { server_thread_.join(); } diff --git a/mooncake-store/src/hot_standby_service.cpp b/mooncake-store/src/hot_standby_service.cpp new file mode 100644 index 0000000000..c187b92733 --- /dev/null +++ b/mooncake-store/src/hot_standby_service.cpp @@ -0,0 +1,488 @@ +#include "hot_standby_service.h" + +#include + +#include +#include + +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "master_service.h" +#include "oplog_applier.h" +#include "oplog_manager.h" +#include "oplog_watcher.h" + +namespace mooncake { + +HotStandbyService::HotStandbyService(const HotStandbyConfig& config) + : config_(config) { + metadata_store_ = std::make_unique(); + // OpLogApplier will be created in Start() with cluster_id + // For now, create without cluster_id (will be updated in Start) + oplog_applier_ = std::make_unique(metadata_store_.get()); +} + +// StandbyMetadataStore implementation +bool HotStandbyService::StandbyMetadataStore::PutMetadata( + const std::string& key, const StandbyObjectMetadata& metadata) { + std::lock_guard lock(mutex_); + store_[key] = metadata; + VLOG(2) << "StandbyMetadataStore: stored metadata for key=" << key + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + return true; +} + +bool HotStandbyService::StandbyMetadataStore::Put(const std::string& key, + const std::string& payload) { + // Legacy interface - create empty metadata + StandbyObjectMetadata metadata; + std::lock_guard lock(mutex_); + store_[key] = metadata; + return true; +} + +const StandbyObjectMetadata* HotStandbyService::StandbyMetadataStore::GetMetadata( + const std::string& key) const { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + return &it->second; + } + return nullptr; +} + +bool HotStandbyService::StandbyMetadataStore::Remove(const std::string& key) { + std::lock_guard lock(mutex_); + auto it = store_.find(key); + if (it != store_.end()) { + store_.erase(it); + return true; + } + return false; +} + +bool HotStandbyService::StandbyMetadataStore::Exists( + const std::string& key) const { + std::lock_guard lock(mutex_); + return store_.find(key) != store_.end(); +} + +size_t HotStandbyService::StandbyMetadataStore::GetKeyCount() const { + std::lock_guard lock(mutex_); + return store_.size(); +} + +void HotStandbyService::StandbyMetadataStore::Snapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + out.clear(); + out.reserve(store_.size()); + for (const auto& kv : store_) { + out.emplace_back(kv.first, kv.second); + } +} + +HotStandbyService::~HotStandbyService() { + Stop(); +} + +ErrorCode HotStandbyService::Start(const std::string& primary_address, + const std::string& etcd_endpoints, + const std::string& cluster_id) { + std::lock_guard lock(mutex_); + + if (running_.load()) { + LOG(WARNING) << "HotStandbyService is already running"; + return ErrorCode::OK; + } + + config_.primary_address = primary_address; + etcd_endpoints_ = etcd_endpoints; + cluster_id_ = cluster_id; + +#ifdef STORE_USE_ETCD + // Connect to etcd + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(etcd_endpoints.c_str()); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to connect to etcd: " << etcd_endpoints; + return err; + } + + // Preserve existing local state if HotStandbyService is restarted in-process: + // - metadata_store_ may already contain real-time metadata + // - oplog_applier_ may already have expected_sequence_id_ + uint64_t local_last_seq_id = 0; + if (oplog_applier_) { + uint64_t expected = oplog_applier_->GetExpectedSequenceId(); + local_last_seq_id = expected > 0 ? expected - 1 : 0; + } + const bool has_local_metadata = + metadata_store_ && metadata_store_->GetKeyCount() > 0; + const bool has_local_state = has_local_metadata && local_last_seq_id > 0; + + // Recreate OpLogApplier with cluster_id (for requesting missing OpLog). + // If we had local state, recover to keep sequence continuity. + oplog_applier_ = std::make_unique(metadata_store_.get(), cluster_id); + if (has_local_state) { + LOG(INFO) << "Standby warm start: reuse local metadata (keys=" + << metadata_store_->GetKeyCount() + << "), recover last_seq_id=" << local_last_seq_id; + oplog_applier_->Recover(local_last_seq_id); + } + + // Create OpLogWatcher + oplog_watcher_ = std::make_unique( + etcd_endpoints, cluster_id, oplog_applier_.get()); + + running_.store(true); + is_connected_.store(true); + + // Bootstrap: + // - If we already have local state (warm start), do NOT reload snapshot. + // - Otherwise (cold start/new standby), try snapshot (if enabled) then replay OpLog. + uint64_t baseline_seq_id = has_local_state ? local_last_seq_id : 0; + if (!has_local_state && config_.enable_snapshot_bootstrap && snapshot_provider_) { + std::string snapshot_id; + uint64_t snapshot_seq_id = 0; + std::vector> snapshot; + if (snapshot_provider_->LoadLatestSnapshot(cluster_id_, snapshot_id, snapshot_seq_id, + snapshot)) { + LOG(INFO) << "Loaded snapshot: snapshot_id=" << snapshot_id + << ", snapshot_seq_id=" << snapshot_seq_id + << ", keys=" << snapshot.size(); + // Apply snapshot into local standby store. + for (const auto& kv : snapshot) { + metadata_store_->PutMetadata(kv.first, kv.second); + } + // Align applier to snapshot boundary. + oplog_applier_->Recover(snapshot_seq_id); + baseline_seq_id = snapshot_seq_id; + } else { + LOG(INFO) << "No snapshot available (or provider not ready), falling back to OpLog-only bootstrap"; + } + } + + // Read historical OpLog entries since baseline_seq_id. + uint64_t last_applied_seq_id = baseline_seq_id; + + // Start OpLogWatcher with a consistent "read then watch(from revision+1)" sequence. + if (!oplog_watcher_->StartFromSequenceId(last_applied_seq_id)) { + LOG(WARNING) << "Failed to start OpLogWatcher from sequence_id=" + << last_applied_seq_id << ", continuing anyway"; + } + + // Start background threads + replication_thread_ = std::thread(&HotStandbyService::ReplicationLoop, this); + if (config_.enable_verification) { + verification_thread_ = + std::thread(&HotStandbyService::VerificationLoop, this); + } + + LOG(INFO) << "HotStandbyService started, watching etcd OpLog for cluster: " + << cluster_id; + return ErrorCode::OK; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot start HotStandbyService"; + return ErrorCode::INTERNAL_ERROR; +#endif +} + +void HotStandbyService::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + is_connected_.store(false); + + // Stop OpLogWatcher + if (oplog_watcher_) { + oplog_watcher_->Stop(); + oplog_watcher_.reset(); + } + + // Wait for threads to finish + if (replication_thread_.joinable()) { + replication_thread_.join(); + } + if (verification_thread_.joinable()) { + verification_thread_.join(); + } + + LOG(INFO) << "HotStandbyService stopped"; +} + +StandbySyncStatus HotStandbyService::GetSyncStatus() const { + StandbySyncStatus status; + + // Get applied sequence ID from OpLogApplier + if (oplog_applier_) { + status.applied_seq_id = oplog_applier_->GetExpectedSequenceId() - 1; + if (status.applied_seq_id == 0) { + status.applied_seq_id = applied_seq_id_.load(); // Fallback + } + } else { + status.applied_seq_id = applied_seq_id_.load(); + } + + // Primary sequence ID (best-effort): updated by ReplicationLoop via etcd `/latest`. + status.primary_seq_id = primary_seq_id_.load(); + status.is_connected = is_connected_.load(); + + if (status.primary_seq_id > status.applied_seq_id) { + status.lag_entries = status.primary_seq_id - status.applied_seq_id; + } else { + status.lag_entries = 0; + } + + // Calculate lag time (placeholder - in full implementation this would + // track actual time differences) + status.lag_time = std::chrono::milliseconds(0); + status.is_syncing = running_.load() && is_connected_.load(); + + return status; +} + +bool HotStandbyService::IsReadyForPromotion() const { + StandbySyncStatus status = GetSyncStatus(); + if (!status.is_connected) { + return false; + } + + // Allow promotion even with large lag - the new Primary can continue + // syncing remaining OpLog entries from etcd after promotion. + // Log a warning if lag is large, but don't block promotion. + if (status.lag_entries > config_.max_replication_lag_entries) { + LOG(WARNING) << "Standby has large replication lag: " << status.lag_entries + << " entries (threshold: " << config_.max_replication_lag_entries + << "). Promotion will proceed, but remaining OpLog entries " + << "will be synced after promotion."; + } + + return true; +} + +std::unique_ptr HotStandbyService::Promote() { + std::lock_guard lock(mutex_); + + if (!IsReadyForPromotion()) { + LOG(ERROR) << "Standby is not ready for promotion (not connected)"; + return nullptr; + } + + StandbySyncStatus status = GetSyncStatus(); + uint64_t current_applied_seq_id = status.applied_seq_id; + + LOG(INFO) << "Promoting Standby to Primary. Applied seq_id: " + << current_applied_seq_id + << ", lag: " << status.lag_entries << " entries"; + + // Final catch-up sync before promotion. + // IMPORTANT: + // - Do NOT rely on `lag_entries` here because primary_seq_id_ is best-effort. + // - Stop OpLogWatcher first to avoid concurrent Apply from watch callbacks. + if (oplog_watcher_) { + oplog_watcher_->Stop(); + } + + // Best-effort: resolve any outstanding gaps ONCE before promotion. + // Do NOT block promotion if gaps cannot be fetched. + if (oplog_applier_) { + auto res = oplog_applier_->TryResolveGapsOnceForPromotion(/*max_ids=*/1024); + if (res.attempted > 0) { + LOG(INFO) << "Promotion gap resolve (best-effort): attempted=" << res.attempted + << ", fetched=" << res.fetched + << ", applied_deletes=" << res.applied_deletes; + } + } + + LOG(INFO) << "Final catch-up sync from etcd before promotion..."; + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + const size_t batch_size = 1000; + uint64_t start_seq = current_applied_seq_id + 1; + size_t total_applied = 0; + for (;;) { + std::vector batch; + ErrorCode read_err = oplog_store.ReadOpLogSince(start_seq - 1, batch_size, batch); + if (read_err != ErrorCode::OK) { + LOG(WARNING) << "Final catch-up: failed to read OpLog since seq=" + << (start_seq - 1) << ", err=" << read_err + << ". Proceeding with promotion."; + break; + } + if (batch.empty()) { + break; + } + size_t applied = oplog_applier_->ApplyOpLogEntries(batch); + total_applied += applied; + start_seq = batch.back().sequence_id + 1; + } + LOG(INFO) << "Final catch-up sync done. total_applied=" << total_applied; + + // Stop replication (OpLogWatcher will stop watching) + Stop(); + + // In full implementation, we would: + // 1. Create a new MasterService instance with appropriate config + // 2. Initialize it with the replicated metadata from metadata_store_ + // 3. Set the OpLogManager's initial sequence_id to latest_seq_id + // 4. Return the MasterService instance + + // For now, this is a placeholder - the actual MasterService creation + // happens in MasterServiceSupervisor::Start() after leader election. + // This method ensures all remaining OpLog entries are synced before + // the new Primary starts serving requests. + + LOG(INFO) << "Standby promoted to Primary successfully. " + << "All remaining OpLog entries have been synced."; + + // Return nullptr - actual MasterService creation happens externally + // The caller (MasterServiceSupervisor) will create the MasterService + // with the appropriate configuration. + return nullptr; +} + +size_t HotStandbyService::GetMetadataCount() const { + std::lock_guard lock(mutex_); + return metadata_store_ ? metadata_store_->GetKeyCount() : 0; +} + +uint64_t HotStandbyService::GetLatestAppliedSequenceId() const { + std::lock_guard lock(mutex_); + if (oplog_applier_) { + uint64_t expected_seq = oplog_applier_->GetExpectedSequenceId(); + // GetExpectedSequenceId returns the next expected sequence_id, + // so the latest applied is expected_seq - 1 + return expected_seq > 0 ? expected_seq - 1 : 0; + } + return applied_seq_id_.load(); +} + +bool HotStandbyService::ExportMetadataSnapshot( + std::vector>& out) const { + std::lock_guard lock(mutex_); + if (!metadata_store_) { + out.clear(); + return false; + } + metadata_store_->Snapshot(out); + return true; +} + +void HotStandbyService::SetSnapshotProvider(std::unique_ptr provider) { + std::lock_guard lock(mutex_); + if (provider) { + snapshot_provider_ = std::move(provider); + } else { + snapshot_provider_ = std::make_unique(); + } +} + +void HotStandbyService::ReplicationLoop() { + LOG(INFO) << "Replication loop started (etcd-based OpLog sync)"; + + // With etcd-based OpLog sync, OpLogWatcher handles the actual watching + // in its own thread. This loop now just monitors the status and updates + // metrics. + + while (running_.load()) { + if (!is_connected_.load()) { + // Not connected - wait a bit before checking again + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + // Update applied_seq_id from OpLogApplier + if (oplog_applier_) { + uint64_t current_applied = oplog_applier_->GetExpectedSequenceId() - 1; + if (current_applied > 0) { + applied_seq_id_.store(current_applied); + } + } + + // Update primary_seq_id by querying etcd `/latest` (best-effort). + // Note: `/latest` is batch-updated on Primary, so this is for monitoring only. +#ifdef STORE_USE_ETCD + if (!cluster_id_.empty()) { + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + uint64_t latest_seq = 0; + ErrorCode err = oplog_store.GetLatestSequenceId(latest_seq); + if (err == ErrorCode::OK) { + primary_seq_id_.store(latest_seq); + } + } +#endif + + // Sleep and check again + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + LOG(INFO) << "Replication loop stopped"; +} + +void HotStandbyService::VerificationLoop() { + LOG(INFO) << "Verification loop started"; + + while (running_.load()) { + std::this_thread::sleep_for( + std::chrono::seconds(config_.verification_interval_sec)); + + if (!is_connected_.load()) { + continue; + } + + // In full implementation, this would: + // 1. Sample keys from local metadata store + // 2. Calculate checksums + // 3. Send verification request to Primary + // 4. Handle mismatches if any + + // Placeholder: Log that verification would happen + VLOG(1) << "Verification check (placeholder)"; + } + + LOG(INFO) << "Verification loop stopped"; +} + +void HotStandbyService::ApplyOpLogEntry(const OpLogEntry& entry) { + // NOTE: This method is deprecated. OpLog entries are now applied via + // OpLogApplier, which is called by OpLogWatcher. This method is kept + // for backward compatibility but should not be used in the new etcd-based + // implementation. + + // Update applied_seq_id for status tracking + applied_seq_id_.store(entry.sequence_id); + + // The actual application is handled by OpLogApplier via OpLogWatcher + VLOG(2) << "ApplyOpLogEntry called (deprecated), sequence_id=" + << entry.sequence_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; +} + +void HotStandbyService::ProcessOpLogBatch( + const std::vector& entries) { + for (const auto& entry : entries) { + ApplyOpLogEntry(entry); + } +} + +bool HotStandbyService::ConnectToPrimary() { + // With etcd-based OpLog sync, connection is handled by OpLogWatcher + // This method is kept for compatibility but is no longer used + LOG(INFO) << "ConnectToPrimary called (no-op with etcd-based sync)"; + return true; +} + +void HotStandbyService::DisconnectFromPrimary() { + // With etcd-based OpLog sync, disconnection is handled by OpLogWatcher + // This method is kept for compatibility + if (is_connected_.load()) { + is_connected_.store(false); + replication_stream_.reset(); + LOG(INFO) << "Disconnected from Primary (etcd-based sync)"; + } +} + +} // namespace mooncake + diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e049cb10eb..937d275768 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -2,19 +2,137 @@ #include #include -#include #include #include +#include #include +#include +#include "allocator.h" +#include "etcd_helper.h" +#include "etcd_oplog_store.h" #include "master_metric_manager.h" +#include "metadata_store.h" // For MetadataPayload #include "segment.h" #include "types.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { +namespace { + +// A minimal allocator implementation used only to keep AllocatedBuffer handles +// "valid" after standby promotion. It does NOT own memory. +class DummyBufferAllocator final : public BufferAllocatorBase { + public: + DummyBufferAllocator(std::string segment_name, std::string transport_endpoint) + : segment_name_(std::move(segment_name)), + transport_endpoint_(std::move(transport_endpoint)) {} + + std::unique_ptr allocate(size_t /*size*/) override { + return nullptr; + } + void deallocate(AllocatedBuffer* /*handle*/) override { + // no-op: we don't own memory + } + size_t capacity() const override { return kAllocatorUnknownFreeSpace; } + size_t size() const override { return 0; } + std::string getSegmentName() const override { return segment_name_; } + std::string getTransportEndpoint() const override { return transport_endpoint_; } + size_t getLargestFreeRegion() const override { return kAllocatorUnknownFreeSpace; } + + private: + std::string segment_name_; + std::string transport_endpoint_; +}; + +static Replica ReplicaFromDescriptor( + const Replica::Descriptor& desc, + const std::shared_ptr& allocator_keepalive) { + if (desc.is_memory_replica()) { + const auto& mem = desc.get_memory_descriptor(); + const auto& bd = mem.buffer_descriptor; + if (!allocator_keepalive) { + // This would make the buffer handle invalid immediately (allocator stored + // as weak_ptr in AllocatedBuffer). Callers restoring from standby should + // always provide a keepalive allocator. + LOG(ERROR) << "ReplicaFromDescriptor(memory) missing keepalive allocator, " + << "transport_endpoint=" << bd.transport_endpoint_; + } + + auto buf = std::make_unique( + allocator_keepalive, reinterpret_cast(bd.buffer_address_), + static_cast(bd.size_)); + return Replica(std::move(buf), desc.status); + } + if (desc.is_disk_replica()) { + const auto& disk = desc.get_disk_descriptor(); + return Replica(disk.file_path, disk.object_size, desc.status); + } + const auto& ld = desc.get_local_disk_descriptor(); + UUID client_id{ld.client_id_first, ld.client_id_second}; + return Replica(client_id, ld.object_size, ld.transport_endpoint, desc.status); +} + +} // namespace + MasterService::MasterService() : MasterService(MasterServiceConfig()) {} +std::string MasterService::SerializeMetadataForOpLog(const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + // Extract replica descriptors + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + payload.replicas.push_back(replica.get_descriptor()); + } + + // NOTE: Lease information is NOT serialized because: + // 1. Standby does not perform eviction, so lease info is not used + // 2. After promotion, new Primary should grant fresh leases, not restore old ones + + // Serialize to JSON + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + +std::string MasterService::SerializeMetadataForOpLogWithoutMemReplicas( + const ObjectMetadata& metadata) const { + MetadataPayload payload; + payload.client_id_first = metadata.client_id.first; + payload.client_id_second = metadata.client_id.second; + payload.size = metadata.size; + + payload.replicas.reserve(metadata.replicas.size()); + for (const auto& replica : metadata.replicas) { + if (replica.type() == ReplicaType::MEMORY) { + continue; + } + payload.replicas.push_back(replica.get_descriptor()); + } + + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + +std::string MasterService::SerializeMetadataForOpLogFromReplicaDescriptors( + const UUID& client_id, uint64_t size, + const std::vector& replicas) const { + MetadataPayload payload; + payload.client_id_first = client_id.first; + payload.client_id_second = client_id.second; + payload.size = size; + payload.replicas = replicas; + std::string json_str; + struct_json::to_json(payload, json_str); + return json_str; +} + MasterService::MasterService(const MasterServiceConfig& config) : default_kv_lease_ttl_(config.default_kv_lease_ttl), default_kv_soft_pin_ttl_(config.default_kv_soft_pin_ttl), @@ -72,6 +190,150 @@ MasterService::MasterService(const MasterServiceConfig& config) MasterMetricManager::instance().inc_total_file_capacity( global_file_segment_size_); } + + // Initialize EtcdOpLogStore if HA is enabled + // Note: This requires STORE_USE_ETCD to be enabled at compile time + // Note: etcd connection should be established before MasterService construction + // (e.g., in MasterServiceSupervisor), so we can use the existing connection +#ifdef STORE_USE_ETCD + if (enable_ha_ && !cluster_id_.empty()) { + // Try to create EtcdOpLogStore - if etcd is not connected, operations will fail + // but we can still use memory buffer as fallback + // Writer: enable batch update for `/latest` to reduce etcd write pressure. + auto etcd_oplog_store = + std::make_shared(cluster_id_, /*enable_latest_seq_batch_update=*/true); + oplog_manager_.SetEtcdOpLogStore(etcd_oplog_store); + LOG(INFO) << "EtcdOpLogStore initialized for cluster_id=" + << cluster_id_ << " (etcd connection should be established " + << "before MasterService construction)"; + } else if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but cluster_id is empty, " + "OpLog will only be stored in memory buffer"; + } +#else + if (enable_ha_) { + LOG(WARNING) << "HA mode enabled but STORE_USE_ETCD is not enabled at " + "compile time, OpLog will only be stored in memory buffer. " + "Recompile with -DSTORE_USE_ETCD=ON to enable etcd support."; + } +#endif + + // Start pending durable mutation retry thread (HA only). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + pending_mutations_running_.store(true); + pending_mutations_thread_ = + std::thread(&MasterService::PendingMutationWorker, this); + } +#endif +} + +// Helper function to append an OpLog entry. +// In the current etcd-based design: +// - OpLogManager always appends to its in-memory buffer +// - If EtcdOpLogStore is configured (HA mode), OpLogManager also writes to etcd +// synchronously (best-effort; see OpLogManager::Append). +void MasterService::AppendOpLogAndNotify(OpType type, const std::string& key, + const std::string& payload) { + oplog_manager_.Append(type, key, payload); +} + +auto MasterService::AppendOpLogAndNotifyDurable(OpType type, const std::string& key, + const std::string& payload) + -> tl::expected { +#ifdef STORE_USE_ETCD + // In HA mode, EtcdOpLogStore should have been configured into OpLogManager. + // For safety, treat missing store as an error for durable ops. + // Best-effort synchronous retries to absorb transient etcd blips. + // + // IMPORTANT: + // sequence_id must be allocated ONCE (pre-allocation) and retried with the same + // OpLogEntry, otherwise multiple attempts would allocate multiple sequence_ids + // for a single logical operation. + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode err = PersistOpLogEntryWithSyncRetries(entry); + if (err == ErrorCode::OK) { + return entry.sequence_id; + } + return tl::make_unexpected(err); +#else + (void)type; + (void)key; + (void)payload; + return tl::make_unexpected(ErrorCode::ETCD_OPERATION_ERROR); +#endif +} + +void MasterService::RestoreFromStandbySnapshot( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + // 1) Ensure OpLog sequence continues without regression after failover. + oplog_manager_.SetInitialSequenceId(initial_oplog_sequence_id); + + // 2) Restore metadata entries. + // Keep dummy allocators alive for restored memory replicas. AllocatedBuffer + // only holds a weak_ptr to allocator, so without this keepalive map the + // allocator would expire immediately and transport_endpoint_ would be lost. + standby_allocator_keepalive_.clear(); + auto get_keepalive_allocator = + [this](const std::string& transport_endpoint) + -> std::shared_ptr { + auto it = standby_allocator_keepalive_.find(transport_endpoint); + if (it != standby_allocator_keepalive_.end()) { + return it->second; + } + auto alloc = std::make_shared( + /*segment_name=*/std::string(), transport_endpoint); + standby_allocator_keepalive_.emplace(transport_endpoint, alloc); + return alloc; + }; + + const auto now = std::chrono::steady_clock::now(); + size_t restored = 0; + for (const auto& kv : snapshot) { + const std::string& key = kv.first; + const StandbyObjectMetadata& sm = kv.second; + + std::vector replicas; + replicas.reserve(sm.replicas.size()); + for (const auto& rd : sm.replicas) { + if (rd.is_memory_replica()) { + const auto& bd = rd.get_memory_descriptor().buffer_descriptor; + replicas.emplace_back( + ReplicaFromDescriptor(rd, get_keepalive_allocator(bd.transport_endpoint_))); + } else { + replicas.emplace_back(ReplicaFromDescriptor(rd, nullptr)); + } + } + + // NOTE: Lease information is NOT restored because: + // 1. Standby does not use lease info (no eviction) + // 2. New Primary should grant fresh leases after promotion + // 3. Restoring old lease TTLs could cause immediate eviction if they're expired + const bool enable_soft_pin = false; // Will be set by new Primary if needed + + const size_t shard_idx = getShardIndex(key); + MutexLocker lock(&metadata_shards_[shard_idx].mutex); + + // Overwrite existing key if any. + metadata_shards_[shard_idx].metadata.erase(key); + auto [it, inserted] = metadata_shards_[shard_idx].metadata.emplace( + std::piecewise_construct, std::forward_as_tuple(key), + std::forward_as_tuple(sm.client_id, now, static_cast(sm.size), + std::move(replicas), enable_soft_pin)); + (void)inserted; + + // Lease will be granted by new Primary when objects are accessed + // (via GetReplicaList, ExistKey, etc.) + + // Objects restored from PUT_END are expected to be completed. + metadata_shards_[shard_idx].processing_keys.erase(key); + + restored++; + } + + LOG(INFO) << "Restored metadata from standby snapshot: restored_keys=" + << restored << ", initial_oplog_sequence_id=" << initial_oplog_sequence_id; } MasterService::~MasterService() { @@ -84,6 +346,207 @@ MasterService::~MasterService() { if (client_monitor_thread_.joinable()) { client_monitor_thread_.join(); } + +#ifdef STORE_USE_ETCD + if (pending_mutations_running_.load()) { + pending_mutations_running_.store(false); + pending_mutations_cv_.notify_all(); + if (pending_mutations_thread_.joinable()) { + pending_mutations_thread_.join(); + } + } +#endif +} + +void MasterService::EnqueuePendingMutation(PendingMutation m) { + m.attempt = 0; + m.next_retry_at = std::chrono::steady_clock::now(); + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); +} + +ErrorCode MasterService::PersistOpLogEntryWithSyncRetries( + const OpLogEntry& entry) const { +#ifdef STORE_USE_ETCD + static constexpr int kSyncRetries = 3; + static constexpr int kBaseBackoffMs = 20; + ErrorCode persist_err = ErrorCode::ETCD_OPERATION_ERROR; + for (int attempt = 0; attempt < kSyncRetries; ++attempt) { + persist_err = oplog_manager_.PersistEntryToEtcd(entry); + if (persist_err == ErrorCode::OK) { + break; + } + std::this_thread::sleep_for( + std::chrono::milliseconds(kBaseBackoffMs * (1 << attempt))); + } + return persist_err; +#else + (void)entry; + return ErrorCode::ETCD_OPERATION_ERROR; +#endif +} + +void MasterService::EnqueueRetryOnPersistFailure( + const char* ctx, const OpLogEntry& entry, ErrorCode persist_err, + PendingMutationKind kind, const std::string& segment_name) { +#ifdef STORE_USE_ETCD + LOG(ERROR) << ctx << ": failed to persist OpLog to etcd, key=" + << entry.object_key << ", seq=" << entry.sequence_id + << ", err=" << persist_err << ". Enqueue retry."; + EnqueuePendingMutation(PendingMutation{ + kind, + entry.object_key, + segment_name, + /*oplog_entry=*/entry}); +#else + (void)ctx; + (void)entry; + (void)persist_err; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueue( + const char* ctx, OpType type, const std::string& key, + const std::string& payload, PendingMutationKind kind, + const std::string& segment_name) { +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, payload); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, payload); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, payload); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +void MasterService::AppendOrPersistOrEnqueueLazy( + const char* ctx, OpType type, const std::string& key, + const std::function& payload_factory, + PendingMutationKind kind, const std::string& segment_name) { + std::string payload; + bool payload_ready = false; + auto get_payload = [&]() -> const std::string& { + if (!payload_ready) { + payload = payload_factory ? payload_factory() : std::string(); + payload_ready = true; + } + return payload; + }; + +#ifdef STORE_USE_ETCD + if (enable_ha_) { + const OpLogEntry entry = oplog_manager_.AllocateEntry(type, key, get_payload()); + ErrorCode persist_err = PersistOpLogEntryWithSyncRetries(entry); + if (persist_err != ErrorCode::OK) { + EnqueueRetryOnPersistFailure(ctx, entry, persist_err, kind, segment_name); + } + } else { + AppendOpLogAndNotify(type, key, get_payload()); + } +#else + // No etcd support at compile time: + // - non-HA: keep best-effort in-memory OpLog for debugging/consistency + // - HA: no-op (constructor already warns) + if (!enable_ha_) { + AppendOpLogAndNotify(type, key, get_payload()); + } + (void)ctx; + (void)kind; + (void)segment_name; +#endif +} + +// Return true if processed successfully (done), false if should retry later. +bool MasterService::ProcessPendingMutationOnce(PendingMutation& m) { +#ifndef STORE_USE_ETCD + (void)m; + return true; +#else + const auto now = std::chrono::steady_clock::now(); + if (m.next_retry_at > now) { + return false; + } + + // Retrier responsibility: + // only persist the original pre-allocated OpLogEntry (fixed sequence_id) to etcd. + // Do NOT mutate local metadata here because the caller may have already moved on. + if (m.oplog_entry.sequence_id == 0) { + LOG(WARNING) << "PendingMutation has no pre-allocated OpLogEntry, drop. key=" + << m.key << ", kind=" << static_cast(m.kind); + return true; + } + + ErrorCode err = oplog_manager_.PersistEntryToEtcd(m.oplog_entry); + if (err != ErrorCode::OK) { + return false; + } + return true; +#endif +} + +void MasterService::PendingMutationWorker() { +#ifndef STORE_USE_ETCD + return; +#else + while (pending_mutations_running_.load()) { + PendingMutation m; + bool has_item = false; + { + std::unique_lock lk(pending_mutations_mutex_); + pending_mutations_cv_.wait_for(lk, std::chrono::milliseconds(200), [&] { + return !pending_mutations_running_.load() || !pending_mutations_.empty(); + }); + if (!pending_mutations_running_.load()) { + break; + } + if (pending_mutations_.empty()) { + continue; + } + m = std::move(pending_mutations_.front()); + pending_mutations_.pop_front(); + has_item = true; + } + if (!has_item) { + continue; + } + + const bool done = ProcessPendingMutationOnce(m); + if (done) { + continue; + } + + // Retry with exponential backoff (cap at 30s). + m.attempt++; + const uint32_t exp = std::min(m.attempt, 8); + const auto delay = std::chrono::milliseconds(200u * (1u << exp)); + const auto capped = std::min(delay, std::chrono::milliseconds(30000)); + m.next_retry_at = std::chrono::steady_clock::now() + capped; + + { + std::lock_guard lg(pending_mutations_mutex_); + pending_mutations_.push_back(std::move(m)); + } + pending_mutations_cv_.notify_one(); + } +#endif } auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) @@ -172,10 +635,40 @@ void MasterService::ClearInvalidHandles() { MutexLocker lock(&shard.mutex); auto it = shard.metadata.begin(); while (it != shard.metadata.end()) { + // CleanupStaleHandles may remove MEMORY replicas whose allocator has + // become invalid (segment unmounted). If key remains valid (has disk + // replicas), Standby must receive an updated metadata payload that + // excludes those MEMORY replicas (Scheme A). if (CleanupStaleHandles(it->second)) { - // If the object is empty, we need to erase the iterator + // No replicas remain after cleanup -> key should be deleted. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("ClearInvalidHandles(REMOVE)", + OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, it->first); + } +#endif it = shard.metadata.erase(it); } else { + // Still has some replicas. If HA is enabled, publish updated + // metadata WITHOUT MEMORY replicas (safe superset update). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueueLazy( + "ClearInvalidHandles(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } +#endif ++it; } } @@ -231,6 +724,9 @@ auto MasterService::ExistKey(const std::string& key) // client. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return true; } } @@ -387,6 +883,23 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // This operation may free/reuse MEMORY replicas. Persist REMOVE to etcd + // BEFORE actually erasing local metadata. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("BatchReplicaClear(all)", OpType::REMOVE, + key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + // Before erasing, decrement cache metrics for each COMPLETE replica for (const auto& replica : metadata.replicas) { if (replica.status() == ReplicaStatus::COMPLETE) { @@ -433,6 +946,45 @@ auto MasterService::BatchReplicaClear( continue; } + // HA safety (Scheme A): + // Removing replicas may free/reuse MEMORY replicas. Persist updated metadata + // BEFORE mutating metadata.replicas (which may free memory). +#ifdef STORE_USE_ETCD + if (enable_ha_) { + // Build the remaining replica descriptor list after removal. + std::vector remove_mask(metadata.replicas.size(), false); + for (size_t idx : replicas_to_remove) { + if (idx < remove_mask.size()) { + remove_mask[idx] = true; + } + } + std::vector remaining; + remaining.reserve(metadata.replicas.size()); + for (size_t i = 0; i < metadata.replicas.size(); ++i) { + if (remove_mask[i]) { + continue; + } + remaining.emplace_back(metadata.replicas[i].get_descriptor()); + } + + if (remaining.empty()) { + AppendOrPersistOrEnqueue("BatchReplicaClear(partial REMOVE)", + OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } else { + const std::string payload = + SerializeMetadataForOpLogFromReplicaDescriptors( + metadata.client_id, static_cast(metadata.size), + remaining); + AppendOrPersistOrEnqueue("BatchReplicaClear(partial PUT_END)", + OpType::PUT_END, key, payload, + PendingMutationKind::CLEAR_REPLICAS_ON_SEGMENT, + segment_name); + } + } +#endif + // Remove replicas on the specified segment (in reverse order to // maintain indices) for (auto it = replicas_to_remove.rbegin(); @@ -450,7 +1002,21 @@ auto MasterService::BatchReplicaClear( // If no valid replicas remain, erase the entire metadata if (metadata.replicas.empty() || !metadata.IsValid()) { +#ifndef STORE_USE_ETCD + // Non-HA: keep old behavior; HA already persisted REMOVE above. + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif accessor.Erase(); + } else { +#ifndef STORE_USE_ETCD + // Non-HA: best-effort update to keep future behavior consistent. + if (!enable_ha_) { + const std::string payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, payload); + } +#endif } cleared_keys.emplace_back(key); @@ -501,6 +1067,9 @@ auto MasterService::GetReplicaListByRegex(const std::string& regex_pattern) results.emplace(key, std::move(replica_list)); metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. } } } @@ -542,6 +1111,9 @@ auto MasterService::GetReplicaList(std::string_view key) // Grant a lease to the object so it will not be removed // when the client is reading it. metadata.GrantLease(default_kv_lease_ttl_, default_kv_soft_pin_ttl_); + // Note: LEASE_RENEW is not recorded in OpLog since Standby does not + // perform eviction. Standby will receive DELETE events from Primary + // when objects are evicted. return GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_); @@ -696,6 +1268,13 @@ auto MasterService::PutEnd(const UUID& client_id, const std::string& key, // at beginning. 2. If this object has soft pin enabled, set it to be soft // pinned. metadata.GrantLease(0, default_kv_soft_pin_ttl_); + + // Record OpLog entry for PUT_END so that standbys can replay this change. + // Serialize metadata (replicas, size, lease) to payload so Standby can restore + // complete metadata when promoted to Primary. + std::string metadata_payload = SerializeMetadataForOpLog(metadata); + AppendOpLogAndNotify(OpType::PUT_END, key, metadata_payload); + return {}; } @@ -719,7 +1298,7 @@ auto MasterService::AddReplica(const UUID& client_id, const std::string& key, auto& descriptor = metadata.replicas[i] .get_descriptor() .get_local_disk_descriptor(); - if (descriptor.client_id == client_id) { + if (descriptor.GetClientId() == client_id) { update = true; descriptor.transport_endpoint = replica.get_descriptor() .get_local_disk_descriptor() @@ -760,6 +1339,23 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::INVALID_WRITE); } + // HA behavior: + // Do NOT block subsequent ops for the same key if etcd write fails. + // We allocate sequence_id once and retry persisting this OpLogEntry + // asynchronously if needed. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("PutRevoke", OpType::PUT_REVOKE, key, std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::PUT_REVOKE, key); + } +#endif + if (replica_type == ReplicaType::MEMORY) { MasterMetricManager::instance().dec_mem_cache_nums(); } else if (replica_type == ReplicaType::DISK) { @@ -776,6 +1372,7 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, if (metadata.IsValid() == false) { accessor.Erase(); } + return {}; } @@ -819,8 +1416,25 @@ auto MasterService::Remove(const std::string& key) return tl::make_unexpected(ErrorCode::REPLICA_IS_NOT_READY); } - // Remove object metadata + // HA behavior: + // If etcd write fails, enqueue retry but still proceed with local remove. + // Standby will handle gaps via timeout + late-arrival policy. +#ifdef STORE_USE_ETCD + if (enable_ha_) { + AppendOrPersistOrEnqueue("Remove", OpType::REMOVE, key, std::string(), + PendingMutationKind::CLEAR_ALL_REPLICAS); + } else { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#else + if (!enable_ha_) { + AppendOpLogAndNotify(OpType::REMOVE, key); + } +#endif + + // Remove object metadata (may deallocate memory replicas) accessor.Erase(); + return {}; } @@ -1268,9 +1882,39 @@ void MasterService::BatchEvict(double evict_ratio_target, continue; } if (it->second.lease_timeout <= target_timeout) { - // Evict this object + // Evict this object (MEMORY replicas only). + // + // Scheme A: + // - If key remains valid after removing MEMORY replicas, + // durably persist a PUT_END carrying the updated metadata + // (without MEMORY replicas) before freeing memory. + // - If key becomes invalid (only had MEMORY replicas), + // durably persist REMOVE before freeing memory. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1332,9 +1976,32 @@ void MasterService::BatchEvict(double evict_ratio_target, !it->second.HasDiffRepStatus(ReplicaStatus::COMPLETE, ReplicaType::MEMORY) && it->second.HasMemReplica()) { - // Evict this object + // Evict this object (MEMORY replicas only). See Scheme A above. total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1388,6 +2055,29 @@ void MasterService::BatchEvict(double evict_ratio_target, it->second.lease_timeout <= soft_target_timeout) { total_freed_size += it->second.size * it->second.GetMemReplicaCount(); + + if (enable_ha_) { + const bool has_non_mem_replica = + std::any_of(it->second.replicas.begin(), + it->second.replicas.end(), + [](const Replica& r) { + return r.type() != ReplicaType::MEMORY; + }); + if (has_non_mem_replica) { + AppendOrPersistOrEnqueueLazy( + "BatchEvict(PUT_END)", OpType::PUT_END, it->first, + [&]() { + return SerializeMetadataForOpLogWithoutMemReplicas(it->second); + }, + PendingMutationKind::EVICT_MEM_REPLICAS); + } else { + AppendOrPersistOrEnqueue( + "BatchEvict(REMOVE)", OpType::REMOVE, it->first, + std::string(), + PendingMutationKind::EVICT_MEM_REPLICAS); + } + } + it->second.EraseReplica( ReplicaType::MEMORY); // Erase memory replicas if (it->second.IsValid() == false) { @@ -1559,4 +2249,10 @@ std::string MasterService::ResolvePath(const std::string& key) const { return full_path.lexically_normal().string(); } +OpLogManager& MasterService::GetOpLogManager() { + return oplog_manager_; +} + +// SetReplicationService removed - using etcd-based OpLog sync instead + } // namespace mooncake \ No newline at end of file diff --git a/mooncake-store/src/oplog_applier.cpp b/mooncake-store/src/oplog_applier.cpp new file mode 100644 index 0000000000..2a7e83f690 --- /dev/null +++ b/mooncake-store/src/oplog_applier.cpp @@ -0,0 +1,505 @@ +#include "oplog_applier.h" + +#include +#include + +#include +#include + +#include "etcd_oplog_store.h" +#include "metadata_store.h" + +namespace mooncake { + +OpLogApplier::OpLogApplier(MetadataStore* metadata_store, + const std::string& cluster_id) + : metadata_store_(metadata_store), + cluster_id_(cluster_id), + expected_sequence_id_(1) { + if (metadata_store_ == nullptr) { + LOG(FATAL) << "OpLogApplier: metadata_store cannot be null"; + } +} + +EtcdOpLogStore* OpLogApplier::GetEtcdOpLogStore() const { +#ifdef STORE_USE_ETCD + if (cluster_id_.empty()) { + return nullptr; + } + + std::lock_guard lock(etcd_oplog_store_mutex_); + if (!etcd_oplog_store_) { + // Reader: do not start `/latest` batch update thread. + etcd_oplog_store_ = + std::make_unique(cluster_id_, /*enable_latest_seq_batch_update=*/false); + } + return etcd_oplog_store_.get(); +#else + return nullptr; +#endif +} + +bool OpLogApplier::ApplyOpLogEntry(const OpLogEntry& entry) { + // Global ordering only (key_sequence_id is deprecated and ignored). + // + // IMPORTANT: + // - Watch callbacks / retries may deliver duplicate or already-applied entries. + // - Those must be treated as no-op, not as "out-of-order pending", otherwise + // pending_entries_ can grow and the applier may appear stuck. + const uint64_t expected = expected_sequence_id_.load(); + if (entry.sequence_id < expected) { + // Late arrival of a previously-skipped gap entry: apply only if it's a delete/revoke. + bool was_skipped = false; + { + std::lock_guard lock(pending_mutex_); + auto it = skipped_sequence_ids_.find(entry.sequence_id); + if (it != skipped_sequence_ids_.end()) { + was_skipped = true; + skipped_sequence_ids_.erase(it); + } + } + if (was_skipped) { + if (entry.op_type == OpType::REMOVE || entry.op_type == OpType::PUT_REVOKE) { + // Safe: ensure we don't keep stale metadata. + if (entry.op_type == OpType::REMOVE) { + ApplyRemove(entry); + } else { + ApplyPutRevoke(entry); + } + return true; + } + // PUT_END (or others): discard to avoid resurrecting stale state. + VLOG(1) << "OpLogApplier: discard late skipped entry, op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return true; + } + + VLOG(2) << "OpLogApplier: skip already-applied entry, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key; + return true; // consumed (no-op) + } + if (entry.sequence_id > expected) { + // Future entry - store into pending, wait for the gap to be filled. + std::lock_guard lock(pending_mutex_); + + if (pending_entries_.size() >= static_cast(kMaxPendingEntries)) { + LOG(ERROR) << "OpLogApplier: too many pending entries (" + << pending_entries_.size() << "), discarding entry sequence_id=" + << entry.sequence_id << ", key=" << entry.object_key; + return false; + } + + pending_entries_[entry.sequence_id] = entry; + VLOG(1) << "OpLogApplier: future entry buffered, sequence_id=" + << entry.sequence_id << ", expected=" << expected + << ", key=" << entry.object_key + << ", pending_entries=" << pending_entries_.size(); + return false; + } + + // Apply the operation based on type + switch (entry.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry); + break; + case OpType::REMOVE: + ApplyRemove(entry); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type=" + << static_cast(entry.op_type) + << ", sequence_id=" << entry.sequence_id + << ", key=" << entry.object_key; + return false; + } + + // Update expected sequence ID + expected_sequence_id_.store(entry.sequence_id + 1); + + // Try to process pending entries + ProcessPendingEntries(); + + return true; +} + +size_t OpLogApplier::ApplyOpLogEntries(const std::vector& entries) { + size_t applied_count = 0; + for (const auto& entry : entries) { + if (ApplyOpLogEntry(entry)) { + applied_count++; + } + } + return applied_count; +} + +uint64_t OpLogApplier::GetKeySequenceId(const std::string& key) const { + // Deprecated: key_sequence_id is no longer tracked. + // Global sequence_id is used for ordering. + (void)key; // Suppress unused parameter warning + return 0; +} + +uint64_t OpLogApplier::GetExpectedSequenceId() const { + return expected_sequence_id_.load(); +} + +void OpLogApplier::Recover(uint64_t last_applied_sequence_id) { + expected_sequence_id_.store(last_applied_sequence_id + 1); + LOG(INFO) << "OpLogApplier: recovered from sequence_id=" + << last_applied_sequence_id + << ", expected_sequence_id set to=" << expected_sequence_id_.load(); +} + +size_t OpLogApplier::ProcessPendingEntries() { + // Check for missing sequence IDs, possibly skip after timeout, and/or request them. + uint64_t missing_seq_to_request = 0; + uint64_t skipped_count = 0; + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (;;) { + if (pending_entries_.empty()) { + break; + } + const uint64_t first_pending_seq = pending_entries_.begin()->first; + const uint64_t expected = expected_sequence_id_.load(); + if (first_pending_seq <= expected) { + break; + } + + // There's a gap: expected is missing. + const uint64_t missing_seq = expected; + auto it = missing_sequence_ids_.find(missing_seq); + if (it == missing_sequence_ids_.end()) { + missing_sequence_ids_[missing_seq] = now; + ScheduleWaitForMissingEntries(missing_seq); + break; + } + + const auto waited = std::chrono::duration_cast(now - it->second); + + // Skip after 3s to avoid global stall (user requested behavior). + if (waited.count() >= kMissingEntrySkipSeconds) { + skipped_sequence_ids_[missing_seq] = now; + missing_sequence_ids_.erase(missing_seq); + expected_sequence_id_.store(missing_seq + 1); + skipped_count++; + continue; // may skip multiple consecutive gaps + } + + // Optionally request from etcd after a longer wait (best-effort). + if (waited.count() >= kMissingEntryWaitSeconds) { + missing_seq_to_request = missing_seq; + } + break; + } + } + + // Request missing OpLog if needed (outside the lock to avoid deadlock) + bool retrieved_missing = false; + if (missing_seq_to_request > 0) { + retrieved_missing = RequestMissingOpLog(missing_seq_to_request); + if (retrieved_missing) { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(missing_seq_to_request); + } + } + + size_t processed_count = 0; + for (;;) { + OpLogEntry entry_copy; + bool has_entry = false; + + { + std::lock_guard lock(pending_mutex_); + if (pending_entries_.empty()) { + break; + } + + auto it = pending_entries_.begin(); + const uint64_t expected = expected_sequence_id_.load(); + if (it->first != expected) { + break; // still waiting for earlier sequence_id + } + + entry_copy = it->second; + pending_entries_.erase(it); + has_entry = true; + } + + if (!has_entry) { + break; + } + + // Apply outside lock. + switch (entry_copy.op_type) { + case OpType::PUT_END: + ApplyPutEnd(entry_copy); + break; + case OpType::PUT_REVOKE: + ApplyPutRevoke(entry_copy); + break; + case OpType::REMOVE: + ApplyRemove(entry_copy); + break; + default: + LOG(ERROR) << "OpLogApplier: unsupported op_type in pending entry"; + break; + } + + expected_sequence_id_.store(entry_copy.sequence_id + 1); + + { + std::lock_guard lock(pending_mutex_); + missing_sequence_ids_.erase(entry_copy.sequence_id); + } + + processed_count++; + } + + // Clean up old missing sequence IDs (older than 1 minute) + { + std::lock_guard lock(pending_mutex_); + auto now = std::chrono::steady_clock::now(); + for (auto it = missing_sequence_ids_.begin(); it != missing_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + LOG(WARNING) << "OpLogApplier: giving up on missing sequence_id=" + << it->first << " after " << age.count() << " seconds"; + it = missing_sequence_ids_.erase(it); + } else { + ++it; + } + } + + // Clean up old skipped sequence IDs too (avoid unbounded growth). + for (auto it = skipped_sequence_ids_.begin(); it != skipped_sequence_ids_.end();) { + auto age = std::chrono::duration_cast(now - it->second); + if (age.count() > 60) { + it = skipped_sequence_ids_.erase(it); + } else { + ++it; + } + } + } + + if (skipped_count > 0) { + LOG(WARNING) << "OpLogApplier: skipped " << skipped_count + << " missing sequence_id(s) after timeout, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + if (processed_count > 0) { + LOG(INFO) << "OpLogApplier: processed " << processed_count + << " pending entries, expected_sequence_id now=" + << expected_sequence_id_.load(); + } + + return processed_count; +} + +OpLogApplier::GapResolveResult OpLogApplier::TryResolveGapsOnceForPromotion( + size_t max_ids) { + GapResolveResult r; +#ifdef STORE_USE_ETCD + EtcdOpLogStore* store = GetEtcdOpLogStore(); + if (store == nullptr) { + return r; + } + + std::vector gap_ids; + gap_ids.reserve(max_ids); + { + std::lock_guard lock(pending_mutex_); + for (const auto& kv : missing_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + for (const auto& kv : skipped_sequence_ids_) { + if (gap_ids.size() >= max_ids) break; + gap_ids.push_back(kv.first); + } + } + + if (gap_ids.empty()) { + return r; + } + + std::sort(gap_ids.begin(), gap_ids.end()); + gap_ids.erase(std::unique(gap_ids.begin(), gap_ids.end()), gap_ids.end()); + + r.attempted = gap_ids.size(); + for (uint64_t seq : gap_ids) { + OpLogEntry e; + ErrorCode err = store->ReadOpLog(seq, e); + if (err != ErrorCode::OK) { + continue; + } + r.fetched++; + + // Apply policy: only delete/revoke; drop PUT_END. + if (e.op_type == OpType::REMOVE) { + ApplyRemove(e); + r.applied_deletes++; + } else if (e.op_type == OpType::PUT_REVOKE) { + ApplyPutRevoke(e); + r.applied_deletes++; + } + } + + // Clear gaps we attempted so promotion won't keep retrying them. + { + std::lock_guard lock(pending_mutex_); + for (uint64_t seq : gap_ids) { + missing_sequence_ids_.erase(seq); + skipped_sequence_ids_.erase(seq); + } + } + return r; +#else + (void)max_ids; + return r; +#endif +} + +bool OpLogApplier::CheckSequenceOrder(const OpLogEntry& entry) { + // Only check global sequence order. + // key_sequence_id is no longer used for ordering. + return entry.sequence_id == expected_sequence_id_.load(); +} + +void OpLogApplier::ApplyPutEnd(const OpLogEntry& entry) { + // Payload contains serialized metadata (replicas, size, etc.) in JSON format. + // Deserialize the payload immediately and store structured metadata. + // This allows Standby to serve requests immediately after promotion. + + if (entry.payload.empty()) { + // No payload - create empty metadata (legacy compatibility) + LOG(WARNING) << "OpLogApplier: PUT_END without payload, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + if (!metadata_store_->PutMetadata(entry.object_key, empty_metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } + return; + } + + // Deserialize payload to MetadataPayload + MetadataPayload payload; + bool parse_success = false; + try { + struct_json::from_json(payload, entry.payload); + parse_success = true; + } catch (const std::exception& e) { + LOG(ERROR) << "OpLogApplier: failed to parse payload for key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", error=" << e.what(); + } + + if (!parse_success) { + // Fallback to empty metadata if parsing fails + StandbyObjectMetadata empty_metadata; + empty_metadata.last_sequence_id = entry.sequence_id; + metadata_store_->PutMetadata(entry.object_key, empty_metadata); + return; + } + + // Convert to StandbyObjectMetadata and store + StandbyObjectMetadata metadata = payload.ToStandbyMetadata(entry.sequence_id); + + if (!metadata_store_->PutMetadata(entry.object_key, metadata)) { + LOG(ERROR) << "OpLogApplier: failed to PutMetadata key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } else { + VLOG(1) << "OpLogApplier: applied PUT_END, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << ", replicas=" << metadata.replicas.size() + << ", size=" << metadata.size; + } +} + +void OpLogApplier::ApplyPutRevoke(const OpLogEntry& entry) { + // PUT_REVOKE means the object should be removed from metadata store + // (but the key itself may still exist if there are other replicas) + // For now, we treat it as a remove operation + // In the future, we may need to handle partial replica removal + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << " in PUT_REVOKE, sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied PUT_REVOKE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +void OpLogApplier::ApplyRemove(const OpLogEntry& entry) { + if (!metadata_store_->Remove(entry.object_key)) { + LOG(WARNING) << "OpLogApplier: failed to Remove key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id + << " (key may not exist)"; + } else { + VLOG(1) << "OpLogApplier: applied REMOVE, key=" << entry.object_key + << ", sequence_id=" << entry.sequence_id; + } +} + +bool OpLogApplier::RequestMissingOpLog(uint64_t missing_seq_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore* oplog_store = GetEtcdOpLogStore(); + if (oplog_store == nullptr) { + LOG(WARNING) << "OpLogApplier: cannot request missing OpLog, cluster_id not set"; + return false; + } + + OpLogEntry entry; + ErrorCode err = oplog_store->ReadOpLog(missing_seq_id, entry); + if (err == ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(INFO) << "OpLogApplier: missing OpLog entry not found in etcd, sequence_id=" + << missing_seq_id; + return false; + } + if (err != ErrorCode::OK) { + LOG(ERROR) << "OpLogApplier: failed to read missing OpLog from etcd, sequence_id=" + << missing_seq_id << ", error=" << static_cast(err); + return false; + } + + // Successfully retrieved the missing OpLog entry + LOG(INFO) << "OpLogApplier: retrieved missing OpLog entry, sequence_id=" + << missing_seq_id << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + + // Add to pending entries + // Note: We don't call ProcessPendingEntries() here to avoid potential recursion. + // The caller (ProcessPendingEntries itself) will process the entry in the next loop. + { + std::lock_guard lock(pending_mutex_); + pending_entries_[entry.sequence_id] = entry; + } + + return true; +#else + LOG(WARNING) << "OpLogApplier: STORE_USE_ETCD not enabled, cannot request missing OpLog"; + return false; +#endif +} + +void OpLogApplier::ScheduleWaitForMissingEntries(uint64_t missing_seq_id) { + // This method is called when we first detect a missing sequence_id. + // The actual waiting and requesting is handled in ProcessPendingEntries(). + // We just log it here for tracking. + VLOG(1) << "OpLogApplier: scheduling wait for missing sequence_id=" << missing_seq_id + << ", will request after " << kMissingEntryWaitSeconds << " seconds"; +} + +} // namespace mooncake + diff --git a/mooncake-store/src/oplog_manager.cpp b/mooncake-store/src/oplog_manager.cpp new file mode 100644 index 0000000000..a9f5faa5bf --- /dev/null +++ b/mooncake-store/src/oplog_manager.cpp @@ -0,0 +1,153 @@ +#include "oplog_manager.h" + +#include +#include +#include +#include + +#include "etcd_oplog_store.h" + +namespace mooncake { + +OpLogManager::OpLogManager() = default; + +void OpLogManager::SetEtcdOpLogStore( + std::shared_ptr etcd_oplog_store) { + std::unique_lock lock(mutex_); + etcd_oplog_store_ = etcd_oplog_store; +} + +uint64_t OpLogManager::Append(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + + // Note: We use global sequence_id for ordering guarantee. + // key_sequence_id is set to sequence_id for backward compatibility, + // but the actual ordering is based on global sequence_id. + entry.key_sequence_id = entry.sequence_id; + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + + buffer_.emplace_back(entry); // Copy entry to buffer + + // Write to etcd if EtcdOpLogStore is set + if (etcd_oplog_store_) { + // Release lock before writing to etcd to avoid blocking + // We use the original entry (before it was copied to buffer) + lock.unlock(); + ErrorCode err = etcd_oplog_store_->WriteOpLog(entry); + if (err != ErrorCode::OK) { + // Log error but don't fail the operation + // The entry is already in the memory buffer + LOG(WARNING) << "Failed to write OpLog to etcd, sequence_id=" + << entry.sequence_id + << ", but entry is in memory buffer"; + } + } + + return last_seq_id_; +} + +OpLogEntry OpLogManager::AllocateEntry(OpType type, const std::string& key, + const std::string& payload) { + OpLogEntry entry; + entry.op_type = type; + entry.object_key = key; + entry.payload = payload; + entry.timestamp_ms = NowMs(); + entry.checksum = ComputeChecksum(entry.payload); + entry.prefix_hash = ComputePrefixHash(entry.object_key); + + std::unique_lock lock(mutex_); + entry.sequence_id = ++last_seq_id_; + entry.key_sequence_id = entry.sequence_id; // deprecated + + if (buffer_.size() >= kMaxBufferEntries_) { + buffer_.pop_front(); + ++first_seq_id_; + } + buffer_.emplace_back(entry); + return entry; +} + +ErrorCode OpLogManager::PersistEntryToEtcd(const OpLogEntry& entry) const { + std::shared_lock lock(mutex_); + auto store = etcd_oplog_store_; + lock.unlock(); + if (!store) { + return ErrorCode::ETCD_OPERATION_ERROR; + } + return store->WriteOpLog(entry); +} + +tl::expected OpLogManager::AppendAndPersist( + OpType type, const std::string& key, const std::string& payload) { + // Seq pre-allocation semantics: allocate first, then persist. + OpLogEntry entry = AllocateEntry(type, key, payload); + ErrorCode err = PersistEntryToEtcd(entry); + if (err != ErrorCode::OK) { + return tl::make_unexpected(err); + } + return entry.sequence_id; +} + +uint64_t OpLogManager::GetLastSequenceId() const { + std::shared_lock lock(mutex_); + return last_seq_id_; +} + +void OpLogManager::SetInitialSequenceId(uint64_t sequence_id) { + std::unique_lock lock(mutex_); + if (last_seq_id_ == 0 && buffer_.empty()) { + // Only allow setting initial sequence_id if OpLogManager is empty + last_seq_id_ = sequence_id; + first_seq_id_ = sequence_id + 1; // first_seq_id_ should be > last_seq_id_ when empty + LOG(INFO) << "OpLogManager initial sequence_id set to " << sequence_id; + } else { + LOG(WARNING) << "Cannot set initial sequence_id: OpLogManager is not empty " + << "(last_seq_id_=" << last_seq_id_ << ", buffer_size=" << buffer_.size() << ")"; + } +} + +size_t OpLogManager::GetEntryCount() const { + std::shared_lock lock(mutex_); + return buffer_.size(); +} + +uint64_t OpLogManager::NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()) + .count(); +} + +uint32_t OpLogManager::ComputeChecksum(const std::string& data) { + // Use xxHash XXH32 for a fast, deterministic 32-bit checksum. + // Requires linking against xxHash (e.g., libxxhash) and including . + return static_cast(XXH32(data.data(), data.size(), 0)); +} + +uint32_t OpLogManager::ComputePrefixHash(const std::string& key) { + if (key.empty()) { + return 0; + } + // Use XXH32 for consistency with ComputeChecksum and better performance. + // XXH32 provides faster hashing and lower collision rate than std::hash. + // Computing hash for the entire key ensures better distribution and fewer collisions. + return static_cast(XXH32(key.data(), key.size(), 0)); +} + +} // namespace mooncake + + diff --git a/mooncake-store/src/oplog_watcher.cpp b/mooncake-store/src/oplog_watcher.cpp new file mode 100644 index 0000000000..3fa15fdcb4 --- /dev/null +++ b/mooncake-store/src/oplog_watcher.cpp @@ -0,0 +1,506 @@ +#include "oplog_watcher.h" + +#include +#include +#include +#include +#include + +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#include "etcd_oplog_store.h" +#include "oplog_applier.h" + +#if __has_include() +#include // Ubuntu +#else +#include // CentOS +#endif + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + if (applier_ == nullptr) { + LOG(FATAL) << "OpLogApplier cannot be null"; + } + // Normalize cluster_id to avoid double slashes in watch prefix. + while (!cluster_id_.empty() && cluster_id_.back() == '/') { + cluster_id_.pop_back(); + } +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + // Backward-compatible: start from the last processed sequence id. + (void)StartFromSequenceId(last_processed_sequence_id_.load()); +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t start_seq_id) { + if (running_.load()) { + LOG(WARNING) << "OpLogWatcher is already running"; + return true; + } + +#ifdef STORE_USE_ETCD + uint64_t cursor_seq = start_seq_id; + EtcdRevisionId last_read_rev = 0; + size_t total_applied = 0; + + for (;;) { + std::vector batch; + EtcdRevisionId rev = 0; + if (!ReadOpLogSinceWithRevision(cursor_seq, batch, rev)) { + last_read_rev = 0; + break; + } + last_read_rev = rev; + if (!batch.empty()) { + for (const auto& e : batch) { + if (applier_->ApplyOpLogEntry(e)) { + last_processed_sequence_id_.store(e.sequence_id); + cursor_seq = e.sequence_id; + total_applied++; + } + } + } + if (batch.size() < kSyncBatchSize) { + break; + } + } + + if (last_read_rev > 0) { + next_watch_revision_.store(static_cast(last_read_rev + 1)); + } else { + next_watch_revision_.store(0); + } + + LOG(INFO) << "OpLogWatcher initial sync done: applied=" << total_applied + << ", last_seq=" << last_processed_sequence_id_.load() + << ", next_watch_revision=" << next_watch_revision_.load(); +#endif + + running_.store(true); + watch_thread_ = std::thread(&OpLogWatcher::WatchOpLog, this); + LOG(INFO) << "OpLogWatcher started for cluster_id=" << cluster_id_; + return true; +} + +void OpLogWatcher::Stop() { + if (!running_.load()) { + return; + } + + running_.store(false); + +#ifdef STORE_USE_ETCD + // Cancel the watch + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + ErrorCode err = EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + if (err != ErrorCode::OK) { + LOG(WARNING) << "Failed to cancel watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + } +#endif + + // Wait for watch thread to finish + if (watch_thread_.joinable()) { + watch_thread_.join(); + } + + LOG(INFO) << "OpLogWatcher stopped"; +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSince(start_seq_id, 1000, entries); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + LOG(INFO) << "Read " << entries.size() << " OpLog entries since sequence_id=" + << start_seq_id; + return true; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot read OpLog from etcd"; + return false; +#endif +} + +bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t start_seq_id, + std::vector& entries, + EtcdRevisionId& revision_id) { +#ifdef STORE_USE_ETCD + EtcdOpLogStore oplog_store(cluster_id_, /*enable_latest_seq_batch_update=*/false); + ErrorCode err = oplog_store.ReadOpLogSinceWithRevision( + start_seq_id, kSyncBatchSize, entries, revision_id); + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to read OpLog since sequence_id=" << start_seq_id + << ", error=" << static_cast(err); + return false; + } + return true; +#else + (void)start_seq_id; + (void)entries; + (void)revision_id; + return false; +#endif +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +// Static callback function for etcd Watch (defined before WatchOpLog uses it) +void OpLogWatcher::WatchCallback(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, int event_type) { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + LOG(ERROR) << "OpLogWatcher context is null"; + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + + watcher->HandleWatchEvent(key_str, value_str, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::WatchCallbackV2(void* context, const char* key, size_t key_size, + const char* value, size_t value_size, + int event_type, int64_t mod_revision) { + OpLogWatcher* watcher = static_cast(context); + if (watcher == nullptr) { + LOG(ERROR) << "OpLogWatcher context is null"; + return; + } + + std::string key_str; + if (key != nullptr && key_size > 0) { + key_str.assign(key, key_size); + } + std::string value_str; + if (value != nullptr && value_size > 0) { + value_str = std::string(value, value_size); + } + watcher->HandleWatchEvent(key_str, value_str, event_type, mod_revision); +} + +void OpLogWatcher::WatchOpLog() { +#ifdef STORE_USE_ETCD + LOG(INFO) << "OpLog watch thread started for cluster_id=" << cluster_id_; + + std::string watch_prefix = "/oplog/" + cluster_id_ + "/"; + + while (running_.load()) { + // Start watching - pass static callback function and this pointer as context + EtcdRevisionId start_rev = + static_cast(next_watch_revision_.load()); + // Always use V2 watcher so we can update next_watch_revision_ precisely. + ErrorCode err = EtcdHelper::WatchWithPrefixFromRevisionV2( + watch_prefix.c_str(), watch_prefix.size(), start_rev, this, WatchCallbackV2); + + if (err != ErrorCode::OK) { + LOG(ERROR) << "Failed to start watch for prefix " << watch_prefix + << ", error=" << static_cast(err); + watch_healthy_.store(false); + + // Try to reconnect + TryReconnect(); + continue; + } + + LOG(INFO) << "Watch started for prefix " << watch_prefix; + watch_healthy_.store(true); + consecutive_errors_.store(0); + + // The watch is now running in the background (via Go goroutine) + // We just need to keep the thread alive until Stop() is called or watch fails + while (running_.load() && watch_healthy_.load()) { + // Drive pending/missing handling even when no new watch events arrive. + // Without this, a single out-of-order arrival could park entries in + // pending_entries_ forever if the missing entry isn't delivered via watch + // (but exists in etcd and could be fetched). + (void)applier_->ProcessPendingEntries(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Periodically check watch health + if (consecutive_errors_.load() >= kMaxConsecutiveErrors) { + LOG(WARNING) << "Too many consecutive errors (" << consecutive_errors_.load() + << "), reconnecting watch..."; + watch_healthy_.store(false); + break; + } + } + + if (running_.load() && !watch_healthy_.load()) { + // Cancel current watch before reconnecting + EtcdHelper::CancelWatchWithPrefix(watch_prefix.c_str(), watch_prefix.size()); + TryReconnect(); + } + } + + LOG(INFO) << "OpLog watch thread stopped"; +#else + LOG(ERROR) << "STORE_USE_ETCD is not enabled, cannot watch OpLog from etcd"; + running_.store(false); +#endif +} + +void OpLogWatcher::TryReconnect() { + if (!running_.load()) { + return; + } + + int reconnect_attempt = reconnect_count_.fetch_add(1) + 1; + + // Calculate delay with exponential backoff + int delay_ms = std::min(kReconnectDelayMs * reconnect_attempt, kMaxReconnectDelayMs); + + LOG(INFO) << "Attempting to reconnect watch (attempt #" << reconnect_attempt + << "), waiting " << delay_ms << "ms..."; + + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + + // Sync any missed entries before resuming watch + if (SyncMissedEntries()) { + LOG(INFO) << "Successfully synced missed OpLog entries"; + } else { + LOG(WARNING) << "Failed to sync missed OpLog entries, continuing anyway"; + } +} + +bool OpLogWatcher::SyncMissedEntries() { +#ifdef STORE_USE_ETCD + uint64_t last_seq = last_processed_sequence_id_.load(); + if (last_seq == 0) { + // No entries processed yet, nothing to sync + return true; + } + + LOG(INFO) << "Syncing missed OpLog entries since sequence_id=" << last_seq; + + std::vector entries; + EtcdRevisionId rev = 0; + if (!ReadOpLogSinceWithRevision(last_seq, entries, rev)) { + LOG(ERROR) << "Failed to read missed OpLog entries"; + return false; + } + if (rev > 0) { + next_watch_revision_.store(static_cast(rev + 1)); + } + + if (entries.empty()) { + LOG(INFO) << "No missed OpLog entries to sync"; + return true; + } + + LOG(INFO) << "Syncing " << entries.size() << " missed OpLog entries"; + + for (const auto& entry : entries) { + if (applier_->ApplyOpLogEntry(entry)) { + last_processed_sequence_id_.store(entry.sequence_id); + } else { + LOG(WARNING) << "Failed to apply missed OpLog entry, sequence_id=" + << entry.sequence_id; + } + } + + return true; +#else + return false; +#endif +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + HandleWatchEvent(key, value, event_type, /*mod_revision=*/0); +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + // event_type: + // 0 = PUT, 1 = DELETE, 2 = WATCH_BROKEN (Go watcher terminated; should reconnect) + if (event_type == 2) { + LOG(WARNING) << "OpLog watch broken, will reconnect. cluster_id=" << cluster_id_ + << ", next_watch_revision=" << next_watch_revision_.load() + << ", last_seq=" << last_processed_sequence_id_.load(); + watch_healthy_.store(false); + consecutive_errors_.fetch_add(1); + return; + } + + if (mod_revision > 0) { + // Keep next_watch_revision_ monotonic: next = max(next, modRev+1) + int64_t candidate = mod_revision + 1; + int64_t cur = next_watch_revision_.load(); + while (candidate > cur && + !next_watch_revision_.compare_exchange_weak(cur, candidate)) { + // retry + } + } + // event_type: 0 = PUT, 1 = DELETE + if (event_type == 1) { + // DELETE event - OpLog entry was cleaned up + VLOG(1) << "OpLog entry deleted: " << key; + consecutive_errors_.store(0); // Watch is working + return; + } + + if (event_type != 0) { + LOG(WARNING) << "Unknown event type: " << event_type << " for key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Skip the "latest" key and snapshot keys + if (key.find("/latest") != std::string::npos || + key.find("/snapshot/") != std::string::npos) { + return; + } + + // Parse the OpLog entry from JSON + OpLogEntry entry; + if (!DeserializeOpLogEntry(value, entry)) { + LOG(ERROR) << "Failed to deserialize OpLog entry from key: " << key; + consecutive_errors_.fetch_add(1); + return; + } + + // Apply the OpLog entry + if (applier_->ApplyOpLogEntry(entry)) { + // last_processed_sequence_id_ must be monotonic. We may "consume" duplicate + // / already-applied entries (entry.sequence_id < expected) as no-ops, so + // never regress this counter. + uint64_t cur = last_processed_sequence_id_.load(); + while (entry.sequence_id > cur && + !last_processed_sequence_id_.compare_exchange_weak(cur, entry.sequence_id)) { + // retry + } + consecutive_errors_.store(0); // Reset error counter on success + reconnect_count_.store(0); // Reset reconnect counter on success + VLOG(2) << "Applied OpLog entry: sequence_id=" << entry.sequence_id + << ", op_type=" << static_cast(entry.op_type) + << ", key=" << entry.object_key; + } else { + // ApplyOpLogEntry returns false for out-of-order entries, + // which is expected behavior, not an error + VLOG(1) << "OpLog entry not applied (may be out of order): sequence_id=" + << entry.sequence_id; + } +} + +bool OpLogWatcher::DeserializeOpLogEntry(const std::string& json_str, + OpLogEntry& entry) { + Json::Value root; + Json::CharReaderBuilder reader; + std::string errs; + std::istringstream s(json_str); + + if (!Json::parseFromStream(reader, s, &root, &errs)) { + LOG(ERROR) << "Failed to parse OpLogEntry JSON: " << errs; + return false; + } + + entry.sequence_id = root.get("sequence_id", 0).asUInt64(); + entry.timestamp_ms = root.get("timestamp_ms", 0).asUInt64(); + entry.op_type = static_cast(root.get("op_type", 0).asInt()); + entry.object_key = root.get("object_key", "").asString(); + entry.payload = root.get("payload", "").asString(); + entry.checksum = root.get("checksum", 0).asUInt(); + entry.prefix_hash = root.get("prefix_hash", 0).asUInt(); + entry.key_sequence_id = root.get("key_sequence_id", 0).asUInt64(); + return true; +} + +} // namespace mooncake + +#else // STORE_USE_ETCD not defined + +namespace mooncake { + +OpLogWatcher::OpLogWatcher(const std::string& etcd_endpoints, + const std::string& cluster_id, OpLogApplier* applier) + : etcd_endpoints_(etcd_endpoints), cluster_id_(cluster_id), applier_(applier) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +OpLogWatcher::~OpLogWatcher() { + Stop(); +} + +void OpLogWatcher::Start() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::StartFromSequenceId(uint64_t /*start_seq_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +void OpLogWatcher::Stop() { + // No-op when STORE_USE_ETCD is not enabled +} + +bool OpLogWatcher::ReadOpLogSince(uint64_t start_seq_id, + std::vector& entries) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +bool OpLogWatcher::ReadOpLogSinceWithRevision(uint64_t /*start_seq_id*/, + std::vector& /*entries*/, + EtcdRevisionId& /*revision_id*/) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +uint64_t OpLogWatcher::GetLastProcessedSequenceId() const { + return last_processed_sequence_id_.load(); +} + +void OpLogWatcher::WatchOpLog() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type) { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::HandleWatchEvent(const std::string& key, const std::string& value, + int event_type, int64_t mod_revision) { + (void)key; + (void)value; + (void)event_type; + (void)mod_revision; + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +void OpLogWatcher::TryReconnect() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; +} + +bool OpLogWatcher::SyncMissedEntries() { + LOG(FATAL) << "OpLogWatcher requires STORE_USE_ETCD to be enabled"; + return false; +} + +} // namespace mooncake + +#endif // STORE_USE_ETCD + diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index db1cec0cad..df9fdb9938 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -19,6 +19,7 @@ #include "types.h" #include "utils/scoped_vlog_timer.h" #include "version.h" +// replication_service.h removed - using etcd-based OpLog sync instead namespace mooncake { @@ -31,6 +32,14 @@ WrappedMasterService::WrappedMasterService( metric_report_running_(config.enable_metric_reporting) { init_http_server(); + // ReplicationService removed - using etcd-based OpLog sync instead + // TODO: In Phase 1, initialize EtcdOpLogStore and integrate with + // OpLogManager + if (config.enable_ha) { + LOG(INFO) << "HA mode enabled - etcd-based OpLog sync will be " + "implemented in Phase 1"; + } + if (config.enable_metric_reporting) { metric_report_thread_ = std::thread([this]() { while (metric_report_running_) { @@ -49,9 +58,19 @@ WrappedMasterService::~WrappedMasterService() { if (metric_report_thread_.joinable()) { metric_report_thread_.join(); } + + // ReplicationService removed - using etcd-based OpLog sync instead + http_server_.stop(); } +void WrappedMasterService::RestoreFromStandby( + const std::vector>& snapshot, + uint64_t initial_oplog_sequence_id) { + master_service_.RestoreFromStandbySnapshot(snapshot, + initial_oplog_sequence_id); +} + void WrappedMasterService::init_http_server() { using namespace coro_http;