Review/fastsync migration

.gitignore

@@ -46,4 +46,9 @@ __debug_bin
 vendor/
 
 # Internal team references
-docs/SONARQUBE_SETUP_GUIDE.md
+docs/SONARQUBE_SETUP_GUIDE.md
+jmdn.yaml
+internal/WAL/.tmp/*
+.claude/*
+.code-review-graph/*
+.cursor/*

AVC/BLS/bls-sign/bls-sgin.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -97,7 +96,7 @@ func GenerateBLSKeyPair() ([]byte, []byte, error) {
 		PubKey  string `json:"bls_pub"`
 	}
 
-	if data, err := ioutil.ReadFile(config.BLSFile); err == nil {
+	if data, err := os.ReadFile(config.BLSFile); err == nil {
 		var bf blsFile
 		if err := json.Unmarshal(data, &bf); err == nil && bf.PrivKey != "" && bf.PubKey != "" {
 			if priv, err := base64.StdEncoding.DecodeString(bf.PrivKey); err == nil {
@@ -128,7 +127,7 @@ func GenerateBLSKeyPair() ([]byte, []byte, error) {
 		PeerID string `json:"peer_id"`
 	}
 	var pf peerFile
-	if pdata, err := ioutil.ReadFile(config.PeerFile); err == nil {
+	if pdata, err := os.ReadFile(config.PeerFile); err == nil {
 		_ = json.Unmarshal(pdata, &pf)
 	}
 
@@ -138,7 +137,7 @@ func GenerateBLSKeyPair() ([]byte, []byte, error) {
 		PubKey:  base64.StdEncoding.EncodeToString(pubBytes),
 	}
 	if out, err := json.MarshalIndent(bf, "", "  "); err == nil {
-		_ = ioutil.WriteFile(config.BLSFile, out, 0o600)
+		_ = os.WriteFile(config.BLSFile, out, 0o600)
 	}
 
 	return privBytes, pubBytes, nil

CLI/CLI.go

@@ -12,6 +12,7 @@ import (
 	"gossipnode/Block"
 	CLICommon "gossipnode/CLI/common"
 	"gossipnode/DB_OPs"
+	"gossipnode/FastsyncV2"
 	"gossipnode/config"
 	"gossipnode/config/GRO"
 	"gossipnode/config/version"
@@ -52,13 +53,15 @@ type CommandHandler struct {
 	Node            *config.Node
 	NodeManager     *node.NodeManager
 	FastSyncer      *fastsync.FastSync
+	FastSyncerV2    *FastsyncV2.FastsyncV2
 	MainClient      *config.PooledConnection
 	DIDClient       *config.PooledConnection
 	SeedNode        string
 	EnableYggdrasil bool
 	ChainID         int
 	FacadePort      int
 	WSPort          int
+	PullAllowed     bool
 }
 
 // Simple helper to print the CLI prompt in color
@@ -104,8 +107,8 @@ func PrintFuncs() {
 	fmt.Println("  mempoolStats                      - Show mempool statistics")
 	fmt.Println("  stats                             - Show messaging statistics")
 	fmt.Println("  broadcast <message>              - Broadcast a message to all connected peers")
-	fmt.Println("  fastsync <peer_multiaddr>        - Fast sync blockchain data with a peer")
-	fmt.Println("  firstsync <peer_multiaddr> <server|client> - First sync: get all data from peer (server) or receive all data (client)")
+	fmt.Println("  fastsync <peer_multiaddr>        - Fast sync blockchain data with a peer (V2 Engine)")
+	fmt.Println("  accountsync <peer_multiaddr>     - Sync missing accounts only (skip block sync)")
 	fmt.Println("  dbstate                           - Show current ImmuDB database state")
 	fmt.Println("  propagateDID <did> <public_key>  - Propagate a DID to the network")
 	fmt.Println("  getDID <did>                      - Get a DID document from the network")
@@ -263,10 +266,10 @@ func (h *CommandHandler) handleCommand(parts []string) {
 		h.handleShowStats()
 	case "broadcast":
 		h.handleBroadcast(parts)
-	case "fastsync":
+	case "fastsync", "fastsyncv2", "firstsync":
 		h.handleFastSync(parts)
-	case "firstsync":
-		h.handleFirstSync(parts)
+	case "accountsync":
+		h.handleAccountSync(parts)
 	case "propagateDID":
 		h.handlePropagateDID(parts)
 	case "syncinfo":
@@ -577,15 +580,8 @@ func (h *CommandHandler) handleFastSync(parts []string) {
 		return
 	}
 
-	err := h.checkDBClient()
-	if err != nil {
-		fmt.Printf("Database client not initialized: %v\n", err)
-		return
-	}
-
-	err = h.checkDIDClient()
-	if err != nil {
-		fmt.Printf("DID database client not initialized: %v\n", err)
+	if h.FastSyncerV2 == nil {
+		fmt.Println("Error: FastsyncV2 engine is not initialized")
 		return
 	}
 
@@ -603,140 +599,59 @@ func (h *CommandHandler) handleFastSync(parts []string) {
 		return
 	}
 
-	// Get both database states before sync
-	mainState, err := DB_OPs.GetDatabaseState(h.MainClient.Client)
-	if err != nil {
-		fmt.Printf("Failed to get main database state: %v\n", err)
-		return
-	}
-
-	accountsState, err := DB_OPs.GetDatabaseState(h.DIDClient.Client)
-	if err != nil {
-		fmt.Printf("Failed to get accounts database state: %v\n", err)
-		return
+	// Show pre-sync DB state if clients are available
+	if h.MainClient != nil && h.DIDClient != nil {
+		mainState, err := DB_OPs.GetDatabaseState(h.MainClient.Client)
+		if err == nil {
+			fmt.Printf("Pre-sync main DB state: TxID=%d, Root=%x\n", mainState.TxId, mainState.TxHash)
+		}
 	}
 
-	fmt.Printf("Starting blockchain sync with peer %s\n", addrInfo.ID.String())
-	fmt.Printf("Our current main DB state: TxID=%d, Root=%x\n", mainState.TxId, mainState.TxHash)
-	fmt.Printf("Our current accounts DB state: TxID=%d, Root=%x\n", accountsState.TxId, accountsState.TxHash)
+	fmt.Printf("Starting blockchain fastsync (V2 Engine) with peer %s\n", addrInfo.ID.String())
 
-	// Start the sync process
 	startTime := time.Now().UTC()
-
-	maxRetries := 3
-	var syncErr error
-
-	for retry := 0; retry < maxRetries; retry++ {
-		if retry > 0 {
-			fmt.Printf("Retry %d/%d after error: %v\n", retry+1, maxRetries, syncErr)
-			time.Sleep(2 * time.Second)
-		}
-
-		_, syncErr = h.FastSyncer.HandleSync(addrInfo.ID)
-		if syncErr == nil {
-			break
-		}
-	}
-
+	syncErr := h.FastSyncerV2.HandleSync(parts[1])
 	if syncErr != nil {
-		fmt.Printf("Sync failed after %d attempts: %v\n", maxRetries, syncErr)
-		return
-	}
-
-	// Get post-sync states
-	newMainState, err := DB_OPs.GetDatabaseState(h.MainClient.Client)
-	if err != nil {
-		fmt.Printf("Failed to get main database state after sync: %v\n", err)
+		fmt.Printf("Fastsync failed: %v\n", syncErr)
 		return
 	}
 
-	newAccountsState, err := DB_OPs.GetDatabaseState(h.DIDClient.Client)
-	if err != nil {
-		fmt.Printf("Failed to get accounts database state after sync: %v\n", err)
-		return
+	// Show post-sync DB state if clients are available
+	if h.MainClient != nil && h.DIDClient != nil {
+		newMainState, err := DB_OPs.GetDatabaseState(h.MainClient.Client)
+		if err == nil {
+			fmt.Printf("Post-sync main DB state: TxID=%d, Root=%x\n", newMainState.TxId, newMainState.TxHash)
+		}
+		newAccountsState, err := DB_OPs.GetDatabaseState(h.DIDClient.Client)
+		if err == nil {
+			fmt.Printf("Post-sync accounts DB state: TxID=%d, Root=%x\n", newAccountsState.TxId, newAccountsState.TxHash)
+		}
 	}
 
-	fmt.Printf("Sync completed in %v\n", time.Since(startTime))
-	fmt.Printf("New main DB state: TxID=%d, Root=%x\n", newMainState.TxId, newMainState.TxHash)
-	fmt.Printf("New accounts DB state: TxID=%d, Root=%x\n", newAccountsState.TxId, newAccountsState.TxHash)
+	fmt.Printf("Fastsync completed in %v\n", time.Since(startTime))
 	printDashes()
 }
 
-func (h *CommandHandler) handleFirstSync(parts []string) {
-	if len(parts) != 3 {
-		fmt.Println("Usage: firstsync <peer_multiaddr> <server|client>")
-		fmt.Println("  server - Export and send all data from this node")
-		fmt.Println("  client - Receive and load all data from peer")
-		return
-	}
-
-	err := h.checkDBClient()
-	if err != nil {
-		fmt.Printf("Database client not initialized: %v\n", err)
-		return
-	}
-
-	err = h.checkDIDClient()
-	if err != nil {
-		fmt.Printf("DID database client not initialized: %v\n", err)
-		return
-	}
-
-	// Parse the multiaddr
-	addr, err := ma.NewMultiaddr(parts[1])
-	if err != nil {
-		fmt.Printf("Invalid multiaddress: %v\n", err)
-		return
-	}
-
-	// Extract peer ID from multiaddr
-	addrInfo, err := peer.AddrInfoFromP2pAddr(addr)
-	if err != nil {
-		fmt.Printf("Failed to extract peer info: %v\n", err)
+func (h *CommandHandler) handleAccountSync(parts []string) {
+	if len(parts) != 2 {
+		fmt.Println("Usage: accountsync <peer_multiaddr>")
 		return
 	}
-
-	mode := strings.ToLower(parts[2])
-	if mode != "server" && mode != "client" {
-		fmt.Printf("Invalid mode: %s. Must be 'server' or 'client'\n", parts[2])
+	if h.FastSyncerV2 == nil {
+		fmt.Println("Error: FastsyncV2 engine is not initialized")
 		return
 	}
 
-	fmt.Printf("Starting first sync with peer %s (mode: %s)\n", addrInfo.ID.String(), mode)
+	fmt.Printf("Starting account-only sync with peer %s\n", parts[1])
 	startTime := time.Now().UTC()
 
-	var syncErr error
-	if mode == "server" {
-		// Server mode: export and send all data
-		fmt.Println(">>> Running in SERVER mode - exporting all data...")
-		syncErr = h.FastSyncer.FirstSyncServer(addrInfo.ID)
-	} else {
-		// Client mode: receive and load all data
-		fmt.Println(">>> Running in CLIENT mode - receiving all data...")
-		syncErr = h.FastSyncer.FirstSyncClient(addrInfo.ID)
-	}
-
-	if syncErr != nil {
-		fmt.Printf("First sync failed: %v\n", syncErr)
-		return
-	}
-
-	// Get post-sync states
-	newMainState, err := DB_OPs.GetDatabaseState(h.MainClient.Client)
-	if err != nil {
-		fmt.Printf("Failed to get main database state after sync: %v\n", err)
-		return
-	}
-
-	newAccountsState, err := DB_OPs.GetDatabaseState(h.DIDClient.Client)
+	synced, err := h.FastSyncerV2.AccountSyncOnly(parts[1])
 	if err != nil {
-		fmt.Printf("Failed to get accounts database state after sync: %v\n", err)
+		fmt.Printf("AccountSync failed: %v\n", err)
 		return
 	}
 
-	fmt.Printf("First sync completed in %v\n", time.Since(startTime))
-	fmt.Printf("New main DB state: TxID=%d, Root=%x\n", newMainState.TxId, newMainState.TxHash)
-	fmt.Printf("New accounts DB state: TxID=%d, Root=%x\n", newAccountsState.TxId, newAccountsState.TxHash)
+	fmt.Printf("AccountSync complete: %d missing accounts synced in %v\n", synced, time.Since(startTime))
 	printDashes()
 }
 

CLI/CLI_GRPC.go

@@ -226,6 +226,9 @@ func (h *CommandHandler) HandleFastSync(peeraddr string) (SyncStats, error) {
 	if peeraddr == "" {
 		return SyncStats{}, fmt.Errorf("usage: fastsync <peer_multiaddr>")
 	}
+	if !h.PullAllowed {
+		return SyncStats{}, fmt.Errorf("node is configured as a serve-only participant (pulling disabled). cannot pull data")
+	}
 
 	err := h.checkDBClient()
 	if err != nil {
@@ -291,6 +294,70 @@ func (h *CommandHandler) HandleFastSync(peeraddr string) (SyncStats, error) {
 	}, nil
 }
 
+func (h *CommandHandler) HandleFastSyncV2(peeraddr string) (SyncStats, error) {
+	if peeraddr == "" {
+		return SyncStats{}, fmt.Errorf("usage: fastsyncv2 <peer_multiaddr>")
+	}
+	if !h.PullAllowed {
+		return SyncStats{}, fmt.Errorf("node is configured as a serve-only participant (pulling disabled). cannot pull data")
+	}
+
+	// Make sure engine exists
+	if h.FastSyncerV2 == nil {
+		return SyncStats{}, fmt.Errorf("FastsyncV2 engine is inactive")
+	}
+
+	startTime := time.Now().UTC()
+	err := h.FastSyncerV2.HandleSync(peeraddr)
+	if err != nil {
+		return SyncStats{}, fmt.Errorf("FastsyncV2 failed: %w", err)
+	}
+
+	// Re-fetch DB states to report. FastsyncV2 doesn't require MainClient/DIDClient
+	// for the sync itself, so guard against nil before querying.
+	var newMainState, newAccountsState *schema.ImmutableState
+	if h.MainClient != nil {
+		newMainState, _ = DB_OPs.GetDatabaseState(h.MainClient.Client)
+	}
+	if h.DIDClient != nil {
+		newAccountsState, _ = DB_OPs.GetDatabaseState(h.DIDClient.Client)
+	}
+
+	return SyncStats{
+		TimeTaken:     time.Since(startTime),
+		MainState:     newMainState,
+		AccountsState: newAccountsState,
+	}, nil
+}
+
+func (h *CommandHandler) HandleAccountSync(peeraddr string) (SyncStats, error) {
+	if peeraddr == "" {
+		return SyncStats{}, fmt.Errorf("usage: accountsync <peer_multiaddr>")
+	}
+	if !h.PullAllowed {
+		return SyncStats{}, fmt.Errorf("node is configured as a serve-only participant (pulling disabled). cannot pull data")
+	}
+	if h.FastSyncerV2 == nil {
+		return SyncStats{}, fmt.Errorf("FastsyncV2 engine is inactive")
+	}
+
+	startTime := time.Now().UTC()
+	_, err := h.FastSyncerV2.AccountSyncOnly(peeraddr)
+	if err != nil {
+		return SyncStats{}, fmt.Errorf("AccountSync failed: %w", err)
+	}
+
+	var newAccountsState *schema.ImmutableState
+	if h.DIDClient != nil {
+		newAccountsState, _ = DB_OPs.GetDatabaseState(h.DIDClient.Client)
+	}
+
+	return SyncStats{
+		TimeTaken:     time.Since(startTime),
+		AccountsState: newAccountsState,
+	}, nil
+}
+
 func (h *CommandHandler) HandleFirstSync(peeraddr string, mode string) (SyncStats, error) {
 	if peeraddr == "" {
 		return SyncStats{}, fmt.Errorf("usage: firstsync <peer_multiaddr> <server|client>")
@@ -300,6 +367,11 @@ func (h *CommandHandler) HandleFirstSync(peeraddr string, mode string) (SyncStat
 		return SyncStats{}, fmt.Errorf("usage: firstsync <peer_multiaddr> <server|client>")
 	}
 
+	modeLower := strings.ToLower(mode)
+	if modeLower == "client" && !h.PullAllowed {
+		return SyncStats{}, fmt.Errorf("node is configured as a serve-only participant (pulling disabled). cannot pull data")
+	}
+
 	err := h.checkDBClient()
 	if err != nil {
 		return SyncStats{}, fmt.Errorf("database client not initialized: %v", err)
@@ -322,7 +394,6 @@ func (h *CommandHandler) HandleFirstSync(peeraddr string, mode string) (SyncStat
 		return SyncStats{}, fmt.Errorf("failed to extract peer info: %v", err)
 	}
 
-	modeLower := strings.ToLower(mode)
 	if modeLower != "server" && modeLower != "client" {
 		return SyncStats{}, fmt.Errorf("invalid mode: %s. Must be 'server' or 'client'", mode)
 	}