diff --git a/README.md b/README.md index 53dff43..10efeb0 100644 --- a/README.md +++ b/README.md @@ -237,9 +237,11 @@ Available Commands: forceclose Force-close the last state that is in the channel.db provided scbforceclose Force-close the last state that is in the SCB provided genimportscript Generate a script containing the on-chain keys of an lnd wallet that can be imported into other software like bitcoind + inspectwaitingproofs Inspect lnd waiting proofs for startup-fatal records without modifying the database migratedb Apply all recent lnd channel database migrations pullanchor Attempt to CPFP an anchor output of a channel recoverloopin Recover a loop in swap that the loop daemon is not able to sweep + repairwaitingproofs Repair legacy lnd waiting proofs that can prevent startup removechannel Remove a single channel from the given channel DB rescueclosed Try finding the private keys for funds that are in outputs of remotely force-closed channels rescuefunding Rescue funds locked in a funding multisig output that never resulted in a proper channel; this is the command the initiator of the channel needs to run @@ -306,9 +308,11 @@ Legend: | [fixoldbackup](doc/chantools_fixoldbackup.md) | ✏️ ( 📌 ) Fixes an issue with old `channel.backup` files | | [forceclose](doc/chantools_forceclose.md) | ✏️ ( ☠️ ⚠️ ) Publish an old channel state from a `channel.db` file | | [genimportscript](doc/chantools_genimportscript.md) | ✏️ Create a script/text file that can be used to import `lnd` keys into other software | +| [inspectwaitingproofs](doc/chantools_inspectwaitingproofs.md) | Inspect the waiting proof store for records that can prevent `lnd` from starting | | [migratedb](doc/chantools_migratedb.md) | Upgrade the `channel.db` file to the latest version | | [pullanchor](doc/chantools_pullanchor.md) | ✏️ Attempt to CPFP an anchor output of a channel | | [recoverloopin](doc/chantools_recoverloopin.md) | ✏️ Recover funds from a failed Lightning Loop inbound swap | +| [repairwaitingproofs](doc/chantools_repairwaitingproofs.md) | Repair legacy waiting proofs that can prevent `lnd` from starting | | [removechannel](doc/chantools_removechannel.md) | (☠️ ⚠️) Remove a single channel from a `channel.db` file | | [rescueclosed](doc/chantools_rescueclosed.md) | ✏️ ( 📌 ) Rescue funds in a legacy (pre `STATIC_REMOTE_KEY`) channel output | | [rescuefunding](doc/chantools_rescuefunding.md) | ✏️ ( 📌 ) Rescue funds from a funding transaction. Deprecated, use [zombierecovery](doc/chantools_zombierecovery.md) instead | diff --git a/cmd/chantools/inspectwaitingproofs.go b/cmd/chantools/inspectwaitingproofs.go new file mode 100644 index 0000000..ee4dfa9 --- /dev/null +++ b/cmd/chantools/inspectwaitingproofs.go @@ -0,0 +1,400 @@ +package main + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/spf13/cobra" + bbolt "go.etcd.io/bbolt" +) + +var ( + waitingProofsBucket = []byte("waitingproofs") + metadataBucket = []byte("metadata") + dbVersionKey = []byte("dbp") +) + +const ( + waitingProofV1Type = byte(0) + waitingProofV2Type = byte(1) + + legacyWaitingProofKeyLen = 9 + typedWaitingProofKeyLen = 10 + announceSignatures1Len = 168 + + waitingProofClassV1OK = "v1_ok" + waitingProofClassV2Candidate = "v2_candidate" + waitingProofClassDecodeError = "decode_error" +) + +type inspectWaitingProofsCommand struct { + ChannelDB string + JSON bool + + cmd *cobra.Command +} + +func newInspectWaitingProofsCommand() *cobra.Command { + cc := &inspectWaitingProofsCommand{} + cc.cmd = &cobra.Command{ + Use: "inspectwaitingproofs", + Short: "Inspect lnd waiting proofs for startup-fatal records " + + "without modifying the database", + Long: `This command opens an offline copy of an lnd channel.db in +strictly read-only mode and inspects the waitingproofs bucket. It identifies +records that the lnd v0.21 typed waiting-proof decoder would reject, including +legacy remote proofs that are misread as V2 MuSig2 nonces. + +The report also prints the raw channel DB version key status. A +db_version_status=db_version_key_missing result means the metadata bucket is +present but metadata/dbp is absent. Affected lnd versions can interpret that +missing key as the latest schema version, which can explain why legacy waiting +proofs were left unmigrated. + +Always stop lnd and create a copy of channel.db first. This command does not +repair or delete anything. Do not share channel.db because it contains +sensitive channel state.`, + Example: `chantools inspectwaitingproofs \ + --channeldb /tmp/channel.db.waitingproof-check`, + RunE: cc.Execute, + } + cc.cmd.Flags().StringVar( + &cc.ChannelDB, "channeldb", "", "offline copy of the lnd "+ + "channel.db file to inspect", + ) + cc.cmd.Flags().BoolVar( + &cc.JSON, "json", false, "print the inspection report as JSON", + ) + + return cc.cmd +} + +func (c *inspectWaitingProofsCommand) Execute(cmd *cobra.Command, + _ []string) error { + + if c.ChannelDB == "" { + return errors.New("channel DB is required") + } + + report, err := inspectWaitingProofDB(c.ChannelDB) + if err != nil { + return err + } + + if c.JSON { + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(report) + } + + writeWaitingProofReport(cmd.OutOrStdout(), report) + + return nil +} + +type waitingProofReport struct { + DBVersion *uint32 `json:"db_version,omitempty"` + DBVersionStatus string `json:"db_version_status"` + BucketState string `json:"bucket_state"` + Verdict string `json:"verdict"` + Records []waitingProofRecord `json:"records,omitempty"` + Total int `json:"total"` + V1OK int `json:"v1_ok"` + V2Candidates int `json:"v2_candidates"` + Fatal int `json:"fatal"` + ExactMatch int `json:"exact_unsupported_format_3d"` +} + +type waitingProofRecord struct { + Key string `json:"key"` + KeyLength int `json:"key_length"` + ValueLength int `json:"value_length"` + ValuePrefix string `json:"value_prefix"` + Classification string `json:"classification"` + Fatal bool `json:"fatal"` + DecodeError string `json:"decode_error,omitempty"` + Legacy string `json:"legacy,omitempty"` + KeyStatus string `json:"key_status,omitempty"` +} + +func inspectWaitingProofDB(path string) (*waitingProofReport, error) { + db, err := bbolt.Open(path, dbFilePermission, &bbolt.Options{ + ReadOnly: true, + Timeout: 5 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("error opening channel DB read-only: %w", err) + } + defer func() { _ = db.Close() }() + + report := &waitingProofReport{ + DBVersionStatus: "metadata_bucket_missing", + BucketState: "absent", + } + err = db.View(func(tx *bbolt.Tx) error { + meta := tx.Bucket(metadataBucket) + if meta != nil { + versionBytes := meta.Get(dbVersionKey) + switch { + case versionBytes == nil: + report.DBVersionStatus = "db_version_key_missing" + + case len(versionBytes) == 4: + version := binary.BigEndian.Uint32(versionBytes) + report.DBVersion = &version + report.DBVersionStatus = "present" + + default: + report.DBVersionStatus = fmt.Sprintf( + "db_version_key_malformed_len_%d", + len(versionBytes), + ) + } + } + + bucket := tx.Bucket(waitingProofsBucket) + if bucket == nil { + return nil + } + + report.BucketState = "empty" + return bucket.ForEach(func(k, v []byte) error { + if v == nil { + return nil + } + + report.BucketState = "present" + record := classifyWaitingProof(k, v) + report.Records = append(report.Records, record) + report.Total++ + + switch record.Classification { + case waitingProofClassV1OK: + report.V1OK++ + case waitingProofClassV2Candidate: + report.V2Candidates++ + } + if record.Fatal { + report.Fatal++ + } + if record.DecodeError == + "invalid public key: unsupported format: 3d" { + + report.ExactMatch++ + } + + return nil + }) + }) + if err != nil { + return nil, fmt.Errorf("error reading channel DB: %w", err) + } + report.Verdict = waitingProofVerdict(report) + + return report, nil +} + +func classifyWaitingProof(k, v []byte) waitingProofRecord { + record := waitingProofRecord{ + Key: hex.EncodeToString(k), + KeyLength: len(k), + ValueLength: len(v), + ValuePrefix: hex.EncodeToString(v[:min(4, len(v))]), + Legacy: classifyLegacyWaitingProof(k, v), + } + + if len(v) < 2 { + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = "unexpected EOF reading proof type/side" + return record + } + + switch v[0] { + case waitingProofV1Type: + return classifyV1WaitingProof(record, k, v) + + case waitingProofV2Type: + return classifyV2WaitingProof(record, k, v) + + default: + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = fmt.Sprintf( + "unknown waiting proof type: %d", v[0], + ) + return record + } +} + +func classifyV1WaitingProof(record waitingProofRecord, k, + v []byte) waitingProofRecord { + + if len(k) == legacyWaitingProofKeyLen { + record.KeyStatus = "legacy_length_key" + } + + if len(v) < 2+announceSignatures1Len { + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = "unexpected EOF decoding AnnounceSignatures1" + return record + } + + record.Classification = waitingProofClassV1OK + if len(k) != typedWaitingProofKeyLen { + record.KeyStatus = fmt.Sprintf( + "unexpected typed V1 key length %d", len(k), + ) + return record + } + + // Typed V1 key: [type(1), scid(8), isRemote(1)]. The SCID in the + // value follows [type(1), isRemote(1), channelID(32)]. + if k[0] != waitingProofV1Type || k[9] != v[1] || + !bytes.Equal(k[1:9], v[34:42]) { + + record.KeyStatus = "typed V1 key does not match value" + } else { + record.KeyStatus = "typed V1 key matches value" + } + + return record +} + +func classifyV2WaitingProof(record waitingProofRecord, k, + v []byte) waitingProofRecord { + + record.KeyStatus = classifyV2WaitingProofKey(k) + + if len(v) < 3 { + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = "unexpected EOF reading V2 nonce presence" + return record + } + + if v[2] == 0 { + record.Classification = waitingProofClassV2Candidate + return record + } + if len(v) < 3+btcec.PubKeyBytesLenCompressed { + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = "unexpected EOF reading V2 combined nonce" + return record + } + + _, err := btcec.ParsePubKey(v[3 : 3+btcec.PubKeyBytesLenCompressed]) + if err != nil { + record.Classification = waitingProofClassDecodeError + record.Fatal = true + record.DecodeError = err.Error() + return record + } + + record.Classification = waitingProofClassV2Candidate + return record +} + +func classifyV2WaitingProofKey(k []byte) string { + switch len(k) { + case legacyWaitingProofKeyLen: + return "legacy_length_key" + + case typedWaitingProofKeyLen: + if k[0] != waitingProofV2Type { + return "typed key/value proof types disagree" + } + + return "typed V2 key" + + default: + return fmt.Sprintf("unexpected key length %d", len(k)) + } +} + +func classifyLegacyWaitingProof(k, v []byte) string { + if len(k) != legacyWaitingProofKeyLen || + len(v) < 1+announceSignatures1Len || + (v[0] != 0 && v[0] != 1) { + + return "not_legacy" + } + + // Legacy key: [scid(8), isRemote(1)]. The SCID in the legacy value + // follows [isRemote(1), channelID(32)]. + if k[8] != v[0] || !bytes.Equal(k[:8], v[33:41]) { + return "legacy_shape_key_mismatch" + } + + return "clean_legacy_v1" +} + +func writeWaitingProofReport(w io.Writer, report *waitingProofReport) { + if report.DBVersion == nil { + _, _ = fmt.Fprintf( + w, "db_version=unknown db_version_status=%s\n", + report.DBVersionStatus, + ) + } else { + _, _ = fmt.Fprintf( + w, "db_version=%d db_version_status=%s\n", + *report.DBVersion, report.DBVersionStatus, + ) + } + _, _ = fmt.Fprintf(w, "waitingproofs_bucket=%s\n", report.BucketState) + + for _, record := range report.Records { + _, _ = fmt.Fprintf( + w, "[%s] key=%s key_len=%d value_len=%d prefix=%s", + record.Classification, record.Key, record.KeyLength, + record.ValueLength, record.ValuePrefix, + ) + if record.DecodeError != "" { + _, _ = fmt.Fprintf(w, " error=%q", record.DecodeError) + } + if record.Legacy != "" { + _, _ = fmt.Fprintf(w, " legacy=%s", record.Legacy) + } + if record.KeyStatus != "" { + _, _ = fmt.Fprintf(w, " key_status=%q", record.KeyStatus) + } + _, _ = fmt.Fprintln(w) + } + + _, _ = fmt.Fprintf( + w, "summary total=%d v1_ok=%d v2_candidates=%d fatal=%d "+ + "exact_unsupported_format_3d=%d\n", report.Total, + report.V1OK, report.V2Candidates, report.Fatal, + report.ExactMatch, + ) + + _, _ = fmt.Fprintf(w, "verdict=%s\n", report.Verdict) +} + +func waitingProofVerdict(report *waitingProofReport) string { + switch { + case report.BucketState == "absent" || report.BucketState == "empty": + return "waiting_proof_store_ruled_out" + + case report.ExactMatch > 0: + return "exact_reported_crash_reproduced" + + case report.Fatal > 0: + return "other_startup_fatal_records_found" + + case report.V2Candidates > 0: + return "needs_full_v2_decode" + + default: + return "reported_crash_not_found" + } +} diff --git a/cmd/chantools/inspectwaitingproofs_test.go b/cmd/chantools/inspectwaitingproofs_test.go new file mode 100644 index 0000000..5fb1669 --- /dev/null +++ b/cmd/chantools/inspectwaitingproofs_test.go @@ -0,0 +1,256 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + bbolt "go.etcd.io/bbolt" +) + +func TestInspectWaitingProofsExactLegacyCrash(t *testing.T) { + path := createWaitingProofTestDB(t, true) + before := fileSHA256(t, path) + + report, err := inspectWaitingProofDB(path) + require.NoError(t, err) + require.NotNil(t, report.DBVersion) + require.Equal(t, uint32(35), *report.DBVersion) + require.Equal(t, "present", report.DBVersionStatus) + require.Equal(t, "present", report.BucketState) + require.Equal(t, 2, report.Total) + require.Equal(t, 1, report.V1OK) + require.Equal(t, 1, report.Fatal) + require.Equal(t, 1, report.ExactMatch) + require.Equal(t, "exact_reported_crash_reproduced", report.Verdict) + require.Equal(t, "clean_legacy_v1", report.Records[1].Legacy) + require.Equal(t, before, fileSHA256(t, path)) +} + +func TestInspectWaitingProofsMissingVersionAndTwoLegacyProofs(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + before := fileSHA256(t, path) + + report, err := inspectWaitingProofDB(path) + require.NoError(t, err) + require.Nil(t, report.DBVersion) + require.Equal(t, "db_version_key_missing", report.DBVersionStatus) + require.Equal(t, "present", report.BucketState) + require.Equal(t, 2, report.Total) + require.Zero(t, report.V1OK) + require.Equal(t, 2, report.Fatal) + require.Equal(t, 1, report.ExactMatch) + require.Equal(t, "exact_reported_crash_reproduced", report.Verdict) + + for _, record := range report.Records { + require.Equal(t, "clean_legacy_v1", record.Legacy) + require.Equal(t, "legacy_length_key", record.KeyStatus) + } + + require.Equal(t, before, fileSHA256(t, path)) +} + +func TestInspectWaitingProofsCommandOutput(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + + cmd := newInspectWaitingProofsCommand() + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetErr(&output) + cmd.SetArgs([]string{"--channeldb", path}) + + require.NoError(t, cmd.Execute()) + + lines := output.String() + require.Contains( + t, lines, + "db_version=unknown db_version_status=db_version_key_missing", + ) + require.Contains(t, lines, "waitingproofs_bucket=present") + require.Contains(t, lines, "legacy=clean_legacy_v1") + require.Contains(t, lines, "key_status=\"legacy_length_key\"") + require.Contains(t, lines, "error=\"invalid public key: unsupported format: 3d\"") + require.Contains( + t, lines, + "summary total=2 v1_ok=0 v2_candidates=0 fatal=2 "+ + "exact_unsupported_format_3d=1", + ) + require.Contains(t, lines, "verdict=exact_reported_crash_reproduced") + require.Equal(t, 2, strings.Count(lines, "legacy=clean_legacy_v1")) +} + +func TestInspectWaitingProofsAbsentAndEmpty(t *testing.T) { + absentPath := createWaitingProofTestDB(t, false) + report, err := inspectWaitingProofDB(absentPath) + require.NoError(t, err) + require.Equal(t, "present", report.DBVersionStatus) + require.Equal(t, "absent", report.BucketState) + require.Equal(t, "waiting_proof_store_ruled_out", report.Verdict) + require.Zero(t, report.Total) + + emptyPath := t.TempDir() + "/channel.db" + db, err := bbolt.Open(emptyPath, dbFilePermission, nil) + require.NoError(t, err) + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucket(waitingProofsBucket) + return err + })) + require.NoError(t, db.Close()) + + report, err = inspectWaitingProofDB(emptyPath) + require.NoError(t, err) + require.Equal(t, "metadata_bucket_missing", report.DBVersionStatus) + require.Equal(t, "empty", report.BucketState) + require.Equal(t, "waiting_proof_store_ruled_out", report.Verdict) + require.Zero(t, report.Total) +} + +func TestInspectWaitingProofsTypeFlipIsNotLegacy(t *testing.T) { + key := make([]byte, typedWaitingProofKeyLen) + value := make([]byte, 2+announceSignatures1Len) + value[0], value[1], value[2], value[3] = 1, 1, 1, 0x3d + + record := classifyWaitingProof(key, value) + require.True(t, record.Fatal) + require.Equal( + t, "invalid public key: unsupported format: 3d", + record.DecodeError, + ) + require.Equal(t, "not_legacy", record.Legacy) + require.Equal(t, "typed key/value proof types disagree", record.KeyStatus) +} + +func TestInspectWaitingProofsUnknownAndTruncated(t *testing.T) { + tests := []struct { + name string + value []byte + err string + }{ + { + name: "truncated header", + value: []byte{0}, + err: "unexpected EOF reading proof type/side", + }, + { + name: "unknown type", + value: []byte{9, 0}, + err: "unknown waiting proof type: 9", + }, + { + name: "truncated v1", + value: []byte{0, 0}, + err: "unexpected EOF decoding AnnounceSignatures1", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + record := classifyWaitingProof(nil, test.value) + require.True(t, record.Fatal) + require.Equal(t, test.err, record.DecodeError) + }) + } +} + +func createWaitingProofTestDB(t *testing.T, withBucket bool) string { + t.Helper() + + path := t.TempDir() + "/channel.db" + db, err := bbolt.Open(path, dbFilePermission, nil) + require.NoError(t, err) + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { + meta, err := tx.CreateBucket(metadataBucket) + if err != nil { + return err + } + version := make([]byte, 4) + binary.BigEndian.PutUint32(version, 35) + if err := meta.Put(dbVersionKey, version); err != nil { + return err + } + if !withBucket { + return nil + } + + bucket, err := tx.CreateBucket(waitingProofsBucket) + if err != nil { + return err + } + + goodKey := make([]byte, typedWaitingProofKeyLen) + binary.BigEndian.PutUint64(goodKey[1:9], 111) + goodKey[9] = 1 + goodValue := make([]byte, 2+announceSignatures1Len) + goodValue[1] = 1 + binary.BigEndian.PutUint64(goodValue[34:42], 111) + if err := bucket.Put(goodKey, goodValue); err != nil { + return err + } + + const scid = uint64(222) + badKey := make([]byte, legacyWaitingProofKeyLen) + binary.BigEndian.PutUint64(badKey[:8], scid) + badKey[8] = 1 + badValue := make([]byte, 1+announceSignatures1Len) + badValue[0], badValue[1], badValue[2], badValue[3] = 1, 1, 1, 0x3d + binary.BigEndian.PutUint64(badValue[33:41], scid) + + return bucket.Put(badKey, badValue) + })) + require.NoError(t, db.Close()) + + return path +} + +func createWaitingProofMissingVersionTestDB(t *testing.T) string { + t.Helper() + + path := t.TempDir() + "/channel.db" + db, err := bbolt.Open(path, dbFilePermission, nil) + require.NoError(t, err) + require.NoError(t, db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucket(metadataBucket) + if err != nil { + return err + } + + bucket, err := tx.CreateBucket(waitingProofsBucket) + if err != nil { + return err + } + + localKey := make([]byte, legacyWaitingProofKeyLen) + binary.BigEndian.PutUint64(localKey[:8], 0x0e9a2f0005eb0001) + localValue := make([]byte, 1+announceSignatures1Len) + localValue[1], localValue[2], localValue[3] = 0xd9, 0xd0, 0x75 + binary.BigEndian.PutUint64(localValue[33:41], 0x0e9a2f0005eb0001) + if err := bucket.Put(localKey, localValue); err != nil { + return err + } + + remoteKey := make([]byte, legacyWaitingProofKeyLen) + binary.BigEndian.PutUint64(remoteKey[:8], 0x0e730200077f0001) + remoteKey[8] = 1 + remoteValue := make([]byte, 1+announceSignatures1Len) + remoteValue[0] = 1 + remoteValue[1], remoteValue[2], remoteValue[3] = 0x5e, 0x72, 0x3d + binary.BigEndian.PutUint64(remoteValue[33:41], 0x0e730200077f0001) + + return bucket.Put(remoteKey, remoteValue) + })) + require.NoError(t, db.Close()) + + return path +} + +func fileSHA256(t *testing.T, path string) [sha256.Size]byte { + t.Helper() + + data, err := os.ReadFile(path) + require.NoError(t, err) + return sha256.Sum256(data) +} diff --git a/cmd/chantools/repairwaitingproofs.go b/cmd/chantools/repairwaitingproofs.go new file mode 100644 index 0000000..a614a2f --- /dev/null +++ b/cmd/chantools/repairwaitingproofs.go @@ -0,0 +1,356 @@ +package main + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/spf13/cobra" + bbolt "go.etcd.io/bbolt" +) + +type repairWaitingProofsCommand struct { + ChannelDB string + Commit bool + + cmd *cobra.Command +} + +func newRepairWaitingProofsCommand() *cobra.Command { + cc := &repairWaitingProofsCommand{} + cc.cmd = &cobra.Command{ + Use: "repairwaitingproofs", + Short: "Repair legacy lnd waiting proofs that can prevent " + + "startup", + Long: `This command repairs the specific waitingproofs bucket state that +can prevent affected lnd v0.21 nodes from starting with errors such as: + + invalid public key: unsupported format: 3d + +The command only migrates clean legacy V1 waiting proof records from the old +format to the typed V1 format expected by lnd v0.21. It does not write or +invent metadata/dbp, and it refuses to overwrite conflicting typed records. + +By default this command performs a dry run. To modify the database, stop lnd, +make sure you are operating on the intended channel.db, and pass --commit. When +--commit is used, the command creates a timestamped backup next to channel.db +before writing any changes.`, + Example: `chantools repairwaitingproofs \ + --channeldb ~/.lnd/data/graph/mainnet/channel.db + +chantools repairwaitingproofs \ + --channeldb ~/.lnd/data/graph/mainnet/channel.db \ + --commit`, + RunE: cc.Execute, + } + cc.cmd.Flags().StringVar( + &cc.ChannelDB, "channeldb", "", "offline lnd channel.db "+ + "file to repair", + ) + cc.cmd.Flags().BoolVar( + &cc.Commit, "commit", false, "modify channel.db after "+ + "creating a backup; without this flag only a dry run is "+ + "performed", + ) + + return cc.cmd +} + +func (c *repairWaitingProofsCommand) Execute(cmd *cobra.Command, + _ []string) error { + + if c.ChannelDB == "" { + return errors.New("channel DB is required") + } + + report, err := repairWaitingProofDB(c.ChannelDB, c.Commit) + if err != nil { + return err + } + + writeRepairWaitingProofReport(cmd.OutOrStdout(), report) + + return nil +} + +type repairWaitingProofReport struct { + DryRun bool + BackupPath string + DBVersionStatus string + BucketState string + TotalRecords int + LegacyClean int + TypedRecords int + OtherRecords int + Migrated int + WouldMigrate int + DeletedLegacy int + ConflictingTypedKey string +} + +type waitingProofRepairAction struct { + oldKey []byte + newKey []byte + newValue []byte +} + +func repairWaitingProofDB(path string, commit bool) (*repairWaitingProofReport, + error) { + + if !commit { + return dryRunRepairWaitingProofDB(path) + } + + db, err := bbolt.Open(path, dbFilePermission, &bbolt.Options{ + Timeout: 5 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("error opening channel DB: %w", err) + } + defer func() { _ = db.Close() }() + + backupPath, err := backupFile(path) + if err != nil { + return nil, err + } + + report := &repairWaitingProofReport{ + DryRun: false, + BackupPath: backupPath, + } + + err = db.Update(func(tx *bbolt.Tx) error { + return collectAndApplyWaitingProofRepair(tx, report, true) + }) + if err != nil { + return nil, fmt.Errorf("error repairing waiting proofs: %w", err) + } + + return report, nil +} + +func dryRunRepairWaitingProofDB(path string) (*repairWaitingProofReport, error) { + db, err := bbolt.Open(path, dbFilePermission, &bbolt.Options{ + ReadOnly: true, + Timeout: 5 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("error opening channel DB read-only: %w", + err) + } + defer func() { _ = db.Close() }() + + report := &repairWaitingProofReport{ + DryRun: true, + } + + err = db.View(func(tx *bbolt.Tx) error { + return collectAndApplyWaitingProofRepair(tx, report, false) + }) + if err != nil { + return nil, fmt.Errorf("error inspecting waiting proofs: %w", err) + } + + return report, nil +} + +type waitingProofTx interface { + Bucket(name []byte) *bbolt.Bucket +} + +func collectAndApplyWaitingProofRepair(tx waitingProofTx, + report *repairWaitingProofReport, apply bool) error { + + report.DBVersionStatus = dbVersionStatus(tx) + report.BucketState = "absent" + + bucket := tx.Bucket(waitingProofsBucket) + if bucket == nil { + return nil + } + + report.BucketState = "empty" + + var actions []waitingProofRepairAction + err := bucket.ForEach(func(k, v []byte) error { + if v == nil { + return nil + } + + report.BucketState = "present" + report.TotalRecords++ + + record := classifyWaitingProof(k, v) + switch { + case record.Legacy == "clean_legacy_v1": + report.LegacyClean++ + action := legacyWaitingProofRepairAction(k, v) + existing := bucket.Get(action.newKey) + switch { + case existing == nil: + + case bytes.Equal(existing, action.newValue): + // The typed record is already present. We can safely + // remove the duplicate legacy key during commit. + + default: + report.ConflictingTypedKey = hex.EncodeToString( + action.newKey, + ) + return fmt.Errorf("typed waiting proof key %x "+ + "already exists with different value", + action.newKey) + } + + actions = append(actions, action) + + case len(k) == typedWaitingProofKeyLen: + report.TypedRecords++ + + default: + report.OtherRecords++ + } + + return nil + }) + if err != nil { + return err + } + + report.WouldMigrate = len(actions) + if !apply { + return nil + } + + for _, action := range actions { + if existing := bucket.Get(action.newKey); existing == nil { + err := bucket.Put(action.newKey, action.newValue) + if err != nil { + return err + } + report.Migrated++ + } + + if err := bucket.Delete(action.oldKey); err != nil { + return err + } + report.DeletedLegacy++ + } + + return nil +} + +func legacyWaitingProofRepairAction(k, v []byte) waitingProofRepairAction { + newKey := make([]byte, typedWaitingProofKeyLen) + newKey[0] = waitingProofV1Type + copy(newKey[1:9], k[:8]) + newKey[9] = k[8] + + newValue := make([]byte, len(v)+1) + newValue[0] = waitingProofV1Type + copy(newValue[1:], v) + + return waitingProofRepairAction{ + oldKey: append([]byte(nil), k...), + newKey: newKey, + newValue: newValue, + } +} + +func dbVersionStatus(tx waitingProofTx) string { + meta := tx.Bucket(metadataBucket) + if meta == nil { + return "metadata_bucket_missing" + } + + versionBytes := meta.Get(dbVersionKey) + switch { + case versionBytes == nil: + return "db_version_key_missing" + + case len(versionBytes) == 4: + return "present" + + default: + return fmt.Sprintf( + "db_version_key_malformed_len_%d", len(versionBytes), + ) + } +} + +func backupFile(path string) (string, error) { + backupPath := fmt.Sprintf( + "%s.repairwaitingproofs.%s.bak", path, + time.Now().UTC().Format("20060102-150405.000000000"), + ) + + source, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("error opening channel DB for backup: %w", + err) + } + defer func() { _ = source.Close() }() + + dest, err := os.OpenFile( + backupPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, dbFilePermission, + ) + if err != nil { + return "", fmt.Errorf("error creating backup %s: %w", + backupPath, err) + } + defer func() { _ = dest.Close() }() + + if _, err := io.Copy(dest, source); err != nil { + return "", fmt.Errorf("error writing backup %s: %w", backupPath, + err) + } + if err := dest.Sync(); err != nil { + return "", fmt.Errorf("error syncing backup %s: %w", backupPath, + err) + } + + return backupPath, nil +} + +func writeRepairWaitingProofReport(w io.Writer, + report *repairWaitingProofReport) { + + _, _ = fmt.Fprintf(w, "dry_run=%v\n", report.DryRun) + if report.BackupPath != "" { + _, _ = fmt.Fprintf(w, "backup=%s\n", report.BackupPath) + } + + _, _ = fmt.Fprintf(w, "db_version_status=%s\n", report.DBVersionStatus) + _, _ = fmt.Fprintf(w, "waitingproofs_bucket=%s\n", report.BucketState) + _, _ = fmt.Fprintf( + w, "summary total=%d clean_legacy=%d typed=%d other=%d "+ + "would_migrate=%d migrated=%d deleted_legacy=%d\n", + report.TotalRecords, report.LegacyClean, report.TypedRecords, + report.OtherRecords, report.WouldMigrate, report.Migrated, + report.DeletedLegacy, + ) + + switch { + case report.ConflictingTypedKey != "": + _, _ = fmt.Fprintf( + w, "verdict=conflict typed_key=%s\n", + report.ConflictingTypedKey, + ) + + case report.DryRun && report.WouldMigrate > 0: + _, _ = fmt.Fprintln(w, "verdict=would_repair") + + case report.DryRun: + _, _ = fmt.Fprintln(w, "verdict=nothing_to_repair") + + case report.Migrated > 0 || report.DeletedLegacy > 0: + _, _ = fmt.Fprintln(w, "verdict=repaired") + + default: + _, _ = fmt.Fprintln(w, "verdict=nothing_to_repair") + } +} diff --git a/cmd/chantools/repairwaitingproofs_test.go b/cmd/chantools/repairwaitingproofs_test.go new file mode 100644 index 0000000..ea98e4a --- /dev/null +++ b/cmd/chantools/repairwaitingproofs_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "encoding/binary" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" + bbolt "go.etcd.io/bbolt" +) + +func TestRepairWaitingProofsDryRun(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + before := fileSHA256(t, path) + + report, err := repairWaitingProofDB(path, false) + require.NoError(t, err) + require.True(t, report.DryRun) + require.Equal(t, "db_version_key_missing", report.DBVersionStatus) + require.Equal(t, "present", report.BucketState) + require.Equal(t, 2, report.TotalRecords) + require.Equal(t, 2, report.LegacyClean) + require.Equal(t, 2, report.WouldMigrate) + require.Zero(t, report.Migrated) + require.Zero(t, report.DeletedLegacy) + require.Empty(t, report.BackupPath) + + require.Equal(t, before, fileSHA256(t, path)) +} + +func TestRepairWaitingProofsCommit(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + before := fileSHA256(t, path) + + report, err := repairWaitingProofDB(path, true) + require.NoError(t, err) + require.False(t, report.DryRun) + require.Equal(t, "db_version_key_missing", report.DBVersionStatus) + require.Equal(t, "present", report.BucketState) + require.Equal(t, 2, report.TotalRecords) + require.Equal(t, 2, report.LegacyClean) + require.Equal(t, 2, report.WouldMigrate) + require.Equal(t, 2, report.Migrated) + require.Equal(t, 2, report.DeletedLegacy) + require.FileExists(t, report.BackupPath) + require.Equal(t, before, fileSHA256(t, report.BackupPath)) + + inspect, err := inspectWaitingProofDB(path) + require.NoError(t, err) + require.Nil(t, inspect.DBVersion) + require.Equal(t, "db_version_key_missing", inspect.DBVersionStatus) + require.Equal(t, "present", inspect.BucketState) + require.Equal(t, 2, inspect.Total) + require.Equal(t, 2, inspect.V1OK) + require.Zero(t, inspect.Fatal) + require.Zero(t, inspect.ExactMatch) + require.Equal(t, "reported_crash_not_found", inspect.Verdict) + + require.NotEqual(t, before, fileSHA256(t, path)) +} + +func TestRepairWaitingProofsCommandOutput(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + + cmd := newRepairWaitingProofsCommand() + var output bytes.Buffer + cmd.SetOut(&output) + cmd.SetErr(&output) + cmd.SetArgs([]string{"--channeldb", path, "--commit"}) + + require.NoError(t, cmd.Execute()) + + lines := output.String() + require.Contains(t, lines, "dry_run=false") + require.Contains(t, lines, "backup=") + require.Contains(t, lines, "db_version_status=db_version_key_missing") + require.Contains( + t, lines, + "summary total=2 clean_legacy=2 typed=0 other=0 "+ + "would_migrate=2 migrated=2 deleted_legacy=2", + ) + require.Contains(t, lines, "verdict=repaired") +} + +func TestRepairWaitingProofsConflict(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + + err := withWaitingProofBucket(path, func(bucket *bbolt.Bucket) error { + legacyKey := make([]byte, legacyWaitingProofKeyLen) + binary.BigEndian.PutUint64(legacyKey[:8], 0x0e730200077f0001) + legacyKey[8] = 1 + + typedKey := make([]byte, typedWaitingProofKeyLen) + typedKey[0] = waitingProofV1Type + copy(typedKey[1:9], legacyKey[:8]) + typedKey[9] = legacyKey[8] + + return bucket.Put(typedKey, []byte{0, 1, 99}) + }) + require.NoError(t, err) + + _, err = repairWaitingProofDB(path, true) + require.ErrorContains(t, err, "already exists with different value") +} + +func TestRepairWaitingProofsNoop(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + + _, err := repairWaitingProofDB(path, true) + require.NoError(t, err) + + report, err := repairWaitingProofDB(path, true) + require.NoError(t, err) + require.Zero(t, report.WouldMigrate) + require.Zero(t, report.Migrated) + require.Zero(t, report.DeletedLegacy) + require.Equal(t, 2, report.TypedRecords) +} + +func TestRepairWaitingProofsRequiresChannelDB(t *testing.T) { + cmd := newRepairWaitingProofsCommand() + cmd.SetArgs(nil) + + err := cmd.Execute() + require.ErrorContains(t, err, "channel DB is required") +} + +func TestRepairWaitingProofsCreatesBackupNextToDB(t *testing.T) { + path := createWaitingProofMissingVersionTestDB(t) + + report, err := repairWaitingProofDB(path, true) + require.NoError(t, err) + require.True(t, strings.HasPrefix(report.BackupPath, path)) + require.FileExists(t, report.BackupPath) + + _, err = os.Stat(report.BackupPath) + require.NoError(t, err) +} + +func withWaitingProofBucket(path string, + fn func(bucket *bbolt.Bucket) error) error { + + db, err := bbolt.Open(path, dbFilePermission, nil) + if err != nil { + return err + } + defer func() { _ = db.Close() }() + + return db.Update(func(tx *bbolt.Tx) error { + return fn(tx.Bucket(waitingProofsBucket)) + }) +} diff --git a/cmd/chantools/root.go b/cmd/chantools/root.go index b32ba8f..fef2d00 100644 --- a/cmd/chantools/root.go +++ b/cmd/chantools/root.go @@ -140,9 +140,11 @@ func main() { newForceCloseCommand(), newScbForceCloseCommand(), newGenImportScriptCommand(), + newInspectWaitingProofsCommand(), newMigrateDBCommand(), newPullAnchorCommand(), newRecoverLoopInCommand(), + newRepairWaitingProofsCommand(), newRemoveChannelCommand(), newRescueClosedCommand(), newRescueFundingCommand(), diff --git a/doc/chantools.md b/doc/chantools.md index 56db82e..1962801 100644 --- a/doc/chantools.md +++ b/doc/chantools.md @@ -40,10 +40,12 @@ https://github.com/lightninglabs/chantools/. * [chantools fixoldbackup](chantools_fixoldbackup.md) - Fixes an old channel.backup file that is affected by the lnd issue #3881 (unable to derive shachain root key) * [chantools forceclose](chantools_forceclose.md) - Force-close the last state that is in the channel.db provided * [chantools genimportscript](chantools_genimportscript.md) - Generate a script containing the on-chain keys of an lnd wallet that can be imported into other software like bitcoind +* [chantools inspectwaitingproofs](chantools_inspectwaitingproofs.md) - Inspect lnd waiting proofs for startup-fatal records without modifying the database * [chantools migratedb](chantools_migratedb.md) - Apply all recent lnd channel database migrations * [chantools pullanchor](chantools_pullanchor.md) - Attempt to CPFP an anchor output of a channel * [chantools recoverloopin](chantools_recoverloopin.md) - Recover a loop in swap that the loop daemon is not able to sweep * [chantools removechannel](chantools_removechannel.md) - Remove a single channel from the given channel DB +* [chantools repairwaitingproofs](chantools_repairwaitingproofs.md) - Repair legacy lnd waiting proofs that can prevent startup * [chantools rescueclosed](chantools_rescueclosed.md) - Try finding the private keys for funds that are in outputs of remotely force-closed channels * [chantools rescuefunding](chantools_rescuefunding.md) - Rescue funds locked in a funding multisig output that never resulted in a proper channel; this is the command the initiator of the channel needs to run * [chantools rescuetweakedkey](chantools_rescuetweakedkey.md) - Attempt to rescue funds locked in an address with a key that was affected by a specific bug in lnd diff --git a/doc/chantools_inspectwaitingproofs.md b/doc/chantools_inspectwaitingproofs.md new file mode 100644 index 0000000..e2c1fe9 --- /dev/null +++ b/doc/chantools_inspectwaitingproofs.md @@ -0,0 +1,54 @@ +## chantools inspectwaitingproofs + +Inspect lnd waiting proofs for startup-fatal records without modifying the database + +### Synopsis + +This command opens an offline copy of an lnd channel.db in +strictly read-only mode and inspects the waitingproofs bucket. It identifies +records that the lnd v0.21 typed waiting-proof decoder would reject, including +legacy remote proofs that are misread as V2 MuSig2 nonces. + +The report also prints the raw channel DB version key status. A +`db_version_status=db_version_key_missing` result means the `metadata` bucket is +present but `metadata/dbp` is absent. Affected lnd versions can interpret that +missing key as the latest schema version, which can explain why legacy waiting +proofs were left unmigrated. + +Always stop lnd and create a copy of channel.db first. This command does not +repair or delete anything. Do not share channel.db because it contains +sensitive channel state. + +``` +chantools inspectwaitingproofs [flags] +``` + +### Examples + +``` +chantools inspectwaitingproofs \ + --channeldb /tmp/channel.db.waitingproof-check +``` + +### Options + +``` + --channeldb string offline copy of the lnd channel.db file to inspect + -h, --help help for inspectwaitingproofs + --json print the inspection report as JSON +``` + +### Options inherited from parent commands + +``` + --nologfile If set, no log file will be created. This is useful for testing purposes where we don't want to create a log file. + -r, --regtest Indicates if regtest parameters should be used + --resultsdir string Directory where results should be stored (default "./results") + -s, --signet Indicates if the public signet parameters should be used + -t, --testnet Indicates if testnet parameters should be used + --testnet4 Indicates if testnet4 parameters should be used +``` + +### SEE ALSO + +* [chantools](chantools.md) - Chantools helps recover funds from lightning channels diff --git a/doc/chantools_repairwaitingproofs.md b/doc/chantools_repairwaitingproofs.md new file mode 100644 index 0000000..d770615 --- /dev/null +++ b/doc/chantools_repairwaitingproofs.md @@ -0,0 +1,58 @@ +## chantools repairwaitingproofs + +Repair legacy lnd waiting proofs that can prevent startup + +### Synopsis + +This command repairs the specific waitingproofs bucket state that +can prevent affected lnd v0.21 nodes from starting with errors such as: + + invalid public key: unsupported format: 3d + +The command only migrates clean legacy V1 waiting proof records from the old +format to the typed V1 format expected by lnd v0.21. It does not write or +invent metadata/dbp, and it refuses to overwrite conflicting typed records. + +By default this command performs a dry run. To modify the database, stop lnd, +make sure you are operating on the intended channel.db, and pass --commit. When +--commit is used, the command creates a timestamped backup next to channel.db +before writing any changes. + +``` +chantools repairwaitingproofs [flags] +``` + +### Examples + +``` +chantools repairwaitingproofs \ + --channeldb ~/.lnd/data/graph/mainnet/channel.db + +chantools repairwaitingproofs \ + --channeldb ~/.lnd/data/graph/mainnet/channel.db \ + --commit +``` + +### Options + +``` + --channeldb string offline lnd channel.db file to repair + --commit modify channel.db after creating a backup; without this flag only a dry run is performed + -h, --help help for repairwaitingproofs +``` + +### Options inherited from parent commands + +``` + --nologfile If set, no log file will be created. This is useful for testing purposes where we don't want to create a log file. + -r, --regtest Indicates if regtest parameters should be used + --resultsdir string Directory where results should be stored (default "./results") + -s, --signet Indicates if the public signet parameters should be used + -t, --testnet Indicates if testnet parameters should be used + --testnet4 Indicates if testnet4 parameters should be used +``` + +### SEE ALSO + +* [chantools](chantools.md) - Chantools helps recover funds from lightning channels +