diff --git a/cmd/relay/coordinator_credentials.go b/cmd/relay/coordinator_credentials.go index e6a2781..7e9d940 100644 --- a/cmd/relay/coordinator_credentials.go +++ b/cmd/relay/coordinator_credentials.go @@ -20,6 +20,10 @@ func sameProfileExceptCredentials(a, b guidedProfile) bool { // Rotate only credential references, never images, release pins, identities or // ceremony bindings. Preserve the previous profile and all operation attempts. func (w *coordinatorWizard) refreshWorkflowCredentials(dir string, p guidedProfile) error { + return w.refreshWorkflowCredentialsExcept(dir, p, "") +} + +func (w *coordinatorWizard) refreshWorkflowCredentialsExcept(dir string, p guidedProfile, allowedAttemptID string) error { next := p next.Credentials, next.R2Parent, next.R2Control = w.d.Credentials, w.d.R2Parent, w.d.R2Control needsRefresh := !reflect.DeepEqual(next, p) @@ -58,6 +62,9 @@ func (w *coordinatorWizard) refreshWorkflowCredentials(dir string, p guidedProfi } for _, a := range latest { if a.Status == "running" || a.Status == "failed" { + if allowedAttemptID != "" && a.ID == allowedAttemptID && a.RecoveryClass == recoveryPublication { + continue + } return errCredentialsNeedRecovery } } diff --git a/cmd/relay/docker_roles.go b/cmd/relay/docker_roles.go index 1e8958b..6b145ba 100644 --- a/cmd/relay/docker_roles.go +++ b/cmd/relay/docker_roles.go @@ -18,6 +18,7 @@ import ( type dockerRoleOptions struct { role, image, platform, work, trust, keys, credentials, config, docker string r2Parent, r2Control string + recoveryContext string } var roleImagePattern = regexp.MustCompile(`^(sha256:[0-9a-f]{64}|[^\s@]+@sha256:[0-9a-f]{64})$`) @@ -161,7 +162,7 @@ func dockerRoleArgs(o dockerRoleOptions, command []string, uid, gid int) ([]stri return nil, errors.New("upload stations must not receive signing keys") } var sources []string - for _, source := range []string{o.work, o.trust, o.keys, o.credentials, o.r2Parent, o.r2Control} { + for _, source := range []string{o.work, o.trust, o.keys, o.credentials, o.r2Parent, o.r2Control, o.recoveryContext} { if source == "" { continue } @@ -188,6 +189,7 @@ func dockerRoleArgs(o dockerRoleOptions, command []string, uid, gid int) ([]stri }{ {o.work, "/work", false, false}, {o.trust, "/trust", true, false}, {o.keys, "/keys", true, false}, {o.credentials, "/credentials/aws", true, true}, {o.r2Parent, "/credentials/r2-parent", true, true}, {o.r2Control, "/credentials/r2-control", true, true}, + {o.recoveryContext, "/recovery", true, false}, } { if mount.source == "" && mount.target != "/work" { continue diff --git a/cmd/relay/legacy_publication_recovery.go b/cmd/relay/legacy_publication_recovery.go new file mode 100644 index 0000000..332971e --- /dev/null +++ b/cmd/relay/legacy_publication_recovery.go @@ -0,0 +1,440 @@ +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/zksecurity/relay/internal/access" +) + +const expiredAWSRecoveryRelease = "1828c720da11a3f9a83ba905a6c3352b6ca06615" + +var ( + legacyRecoveryLauncherCommit = launcherCommit + legacyRecoveryInput io.Reader = os.Stdin + legacyRecoveryOutput io.Writer = os.Stdout + legacyRecoveryRemote = executeLegacyRemoteRecovery + legacyRecoveryAfterCredentialRefresh = func() error { return nil } + legacyRecoverySave = saveJSONAtomic + legacyRecoveryContextRoot = "/recovery" + legacyRecoveryWorkRoot = "/work" + legacyRecoveryTrustRoot = "/trust" + legacyRecoveryStore = func(config access.StorageConfig) publicationStore { + return coordinatorClient(config, config.PublishedBucket) + } +) + +func validateFrozenRecoveryProfile(p guidedProfile) error { + if p.Schema != guidedSchema || p.Role != "coordinator" || p.ReleaseCommit != expiredAWSRecoveryRelease || + p.Name == "" || p.Image == "" || p.Platform == "" || p.Work == "" || p.Trust == "" || p.Keys == "" || p.Credentials == "" || + len(p.Command) != 0 || p.Config != "" || p.R2Parent != "" || p.R2Control != "" { + return errors.New("saved profile is not the exact frozen 1828 coordinator runtime") + } + return nil +} + +func retainedLegacyPublication(state roleFlowState) (*flowAttempt, error) { + var found *flowAttempt + for index := len(state.Attempts) - 1; index >= 0; index-- { + a := &state.Attempts[index] + if a.Stage == "storage" && a.Task == "publish" { + found = a + break + } + } + if found == nil { + return nil, errors.New("no retained storage-stage initial publication was found") + } + if found.OperationSchema != flowOperationSchema || found.RecoveryClass != recoveryPublication || + !strings.HasPrefix(found.ID, "flow-") || !validFlowAttemptID(strings.TrimPrefix(found.ID, "flow-")) { + return nil, errors.New("retained publication metadata is not the recoverable 1828 operation schema") + } + if len(found.Command) != 10 || found.Command[0] != "relay" || found.Command[1] != "coordinator" || found.Command[2] != "publish" || + found.Command[3] != "--verify" || found.Command[4] != "--storage" || found.Command[6] != "--chain" || found.Command[8] != "--chain-signature" || + found.Command[5] != "/work/ceremony/config/relay-storage.json" || found.Command[7] != "/work/ceremony/public/phase1/chain-0000.json" || found.Command[9] != "/work/ceremony/public/phase1/chain-0000.sig" { + return nil, errors.New("retained publication command is not the exact 1828 initial-head recipe") + } + wantBindings := map[string]bool{found.Command[5]: true, found.Command[7]: true, found.Command[9]: true} + if len(wantBindings) != 3 || len(found.InputBindings) != len(wantBindings) { + return nil, errors.New("retained publication does not bind the exact storage, chain and signature inputs") + } + for value := range wantBindings { + if found.InputBindings[value] == "" { + return nil, errors.New("retained publication is missing an input digest") + } + } + return found, nil +} + +func unresolvedLegacyPublication(state roleFlowState) (*flowAttempt, error) { + found, err := retainedLegacyPublication(state) + if err != nil { + return nil, err + } + if found.Status != "running" && found.Status != "failed" { + return nil, errors.New("the retained storage-stage publication is not failed or interrupted") + } + return found, nil +} + +func validateFrozenRecoveryState(p guidedProfile, workflow roleFlowState) (*flowAttempt, error) { + if err := validateFrozenRecoveryProfile(p); err != nil { + return nil, err + } + if workflow.Schema != roleFlowSchema || workflow.Name != p.Name || workflow.Role != "coordinator" || !reflect.DeepEqual(workflow.Profile, p) { + return nil, errors.New("retained workflow does not exactly match the frozen coordinator profile") + } + attempt, err := retainedLegacyPublication(workflow) + if err != nil { + return nil, err + } + if attempt.ImageDigest != p.Image || attempt.Platform != p.Platform { + return nil, errors.New("retained publication names a different frozen runtime") + } + if len(attempt.Mounts) != 3 || attempt.Mounts["/work"] != p.Work || attempt.Mounts["/trust"] != p.Trust || attempt.Mounts["/keys"] != p.Keys || + len(attempt.ExpectedOutputs) != 0 || len(attempt.DirectoryBindings) != 0 || attempt.ReceiptScope != nil || attempt.TurnScope != nil { + return nil, errors.New("retained publication runtime bindings are not the exact frozen coordinator profile") + } + return attempt, nil +} + +func awsStorageUnchanged(configPath string, draft coordinatorDraft, f roleFlow) error { + hostPath, err := f.publicHostPath(configPath) + if err != nil { + return err + } + config, err := loadStorageConfig(hostPath) + if err != nil { + return err + } + if config.Provider != "aws" || draft.Storage["provider"] != "aws" || draft.R2Parent != "" || draft.R2Control != "" { + return errors.New("this hotfix supports AWS storage only; R2 credentials and settings are refused") + } + actual := map[string]string{ + "provider": config.Provider, "endpoint": config.Endpoint, "region": config.Region, + "account-id": config.AccountID, "parent-access-key-id": config.ParentAccessKeyID, + "published-bucket": config.PublishedBucket, "published-base-url": config.PublishedBaseURL, + "inbox-bucket": config.InboxBucket, "profile": config.CoordinatorProfile, + "issuer-profile": config.IssuerProfile, "grant-role-arn": config.GrantRoleARN, + "grant-role-max-ttl": config.GrantRoleMaxTTL, + } + for field, value := range actual { + if draft.Storage[field] != value { + return fmt.Errorf("AWS storage field %q changed; only the credential snapshot may be refreshed", field) + } + } + return nil +} + +func validateFrozenRecoveryDraft(p guidedProfile, draft coordinatorDraft, credentialsRotated bool) error { + if draft.Release != "role-images-"+expiredAWSRecoveryRelease || draft.Name != p.Name || draft.Work != p.Work || draft.Trust != p.Trust || draft.Keys != p.Keys || + draft.Credentials == "" || draft.R2Parent != "" || draft.R2Control != "" || draft.Storage["provider"] != "aws" { + return errors.New("retained preparation differs from the frozen AWS coordinator profile") + } + if credentialsRotated && p.Credentials != draft.Credentials { + return errors.New("refreshed credential reference is not checkpointed in the frozen profile") + } + return nil +} + +func legacyRecoveryNote(attemptID, commit string) string { + return "Recovered retained attempt " + strings.TrimPrefix(attemptID, "flow-") + " using compatible release " + commit + "." +} + +func legacyRecoveryCompleted(workflow roleFlowState, attempt *flowAttempt) bool { + if attempt == nil || attempt.Status != "succeeded" { + return false + } + prefix := "Recovered retained attempt " + strings.TrimPrefix(attempt.ID, "flow-") + " using compatible release " + for index := len(workflow.Attempts) - 1; index >= 0; index-- { + event := workflow.Attempts[index] + if event.Task == "publication-recovery" && event.Stage == attempt.Stage && event.Status == "succeeded" && + event.OperationSchema == flowOperationSchema && event.RecoveryClass == recoveryPublication && strings.HasPrefix(event.Note, prefix) && strings.HasSuffix(event.Note, ".") { + commit := strings.TrimSuffix(strings.TrimPrefix(event.Note, prefix), ".") + if launcherReleaseTag.MatchString("role-images-" + commit) { + return true + } + } + } + return false +} + +func markLegacyRecoveryComplete(workflow *roleFlowState, attemptID, commit, now string) error { + var attempt *flowAttempt + for index := range workflow.Attempts { + if workflow.Attempts[index].ID == attemptID { + attempt = &workflow.Attempts[index] + attempt.Status = "succeeded" + attempt.FinishedAt = now + attempt.Note = "Reconciled by compatible release " + commit + "; all retained objects and the initial head were verified." + break + } + } + if attempt == nil { + return errors.New("retained publication attempt disappeared before completion checkpoint") + } + eventID, err := randomID() + if err != nil { + return err + } + workflow.Attempts = append(workflow.Attempts, flowAttempt{ID: eventID, Task: "publication-recovery", Stage: attempt.Stage, Status: "succeeded", OperationSchema: flowOperationSchema, RecoveryClass: recoveryPublication, StartedAt: now, FinishedAt: now, Note: legacyRecoveryNote(attempt.ID, commit)}) + return nil +} + +func persistLegacyRecoveryCompletion(path string, workflow roleFlowState, expectedProfile guidedProfile, attemptID string, save func(string, any) error) error { + if err := save(path, workflow); err == nil { + return nil + } else { + var retained roleFlowState + if readErr := setupReadJSON(path, &retained); readErr == nil { + attempt, validateErr := validateFrozenRecoveryState(expectedProfile, retained) + if validateErr == nil && attempt.ID == attemptID && legacyRecoveryCompleted(retained, attempt) { + return nil + } + } + return fmt.Errorf("remote publication reconciled but local completion is not confirmed: %w", err) + } +} + +// runRecoverInitial1828 is reachable only inside the hotfix container. The +// host launcher mounts the locked, exact legacy profile and workflow read-only +// at /recovery; ordinary publish has no recovery switch. +func runRecoverInitial1828(args []string) error { + set := flag.NewFlagSet("coordinator recover-initial-1828", flag.ContinueOnError) + var attemptID string + set.StringVar(&attemptID, "attempt-id", "", "exact retained guided-operation attempt") + if err := set.Parse(args); err != nil { + return err + } + if len(set.Args()) != 0 || !strings.HasPrefix(attemptID, "flow-") || !validFlowAttemptID(strings.TrimPrefix(attemptID, "flow-")) { + return errors.New("--attempt-id must name the exact retained 1828 publication attempt") + } + profilePath := filepath.Join(legacyRecoveryContextRoot, "profile.json") + var decoded guidedProfile + if err := setupReadJSON(profilePath, &decoded); err != nil { + return errors.New("the locked recovery profile is unavailable") + } + p, err := readGuidedProfile(profilePath, decoded.Name, "coordinator") + if err != nil { + return err + } + var workflow roleFlowState + if err := setupReadJSON(filepath.Join(legacyRecoveryContextRoot, "workflow", "state.json"), &workflow); err != nil { + return errors.New("the locked recovery workflow is unavailable") + } + attempt, err := validateFrozenRecoveryState(p, workflow) + if err != nil { + return err + } + if attempt.Status != "running" && attempt.Status != "failed" { + return errors.New("mounted recovery attempt is not failed or interrupted") + } + if attempt.ID != attemptID { + return errors.New("mounted recovery workflow does not contain the requested exact attempt") + } + for _, value := range []string{attempt.Command[5], attempt.Command[7], attempt.Command[9]} { + local := filepath.Join(legacyRecoveryWorkRoot, strings.TrimPrefix(value, "/work/")) + digest, err := setupFileHash(local) + if err != nil || digest != attempt.InputBindings[value] { + return fmt.Errorf("mounted retained input changed or is unavailable: %s", value) + } + } + storagePath := filepath.Join(legacyRecoveryWorkRoot, strings.TrimPrefix(attempt.Command[5], "/work/")) + config, err := loadStorageConfig(storagePath) + if err != nil { + return err + } + if config.Provider != "aws" || config.CeremonyPath != "/work/ceremony/public/ceremony.json" || + config.CeremonySignature != "/work/ceremony/public/ceremony.sig" || config.CoordinatorPublicKey != "/trust/coordinator-public-key.hex" { + return errors.New("mounted recovery storage configuration is not the exact frozen AWS ceremony target") + } + workPath := func(value string) string { + return filepath.Join(legacyRecoveryWorkRoot, strings.TrimPrefix(value, "/work/")) + } + trustPath := func(value string) string { + return filepath.Join(legacyRecoveryTrustRoot, strings.TrimPrefix(value, "/trust/")) + } + o := roleOpts{root: workPath("/work/ceremony/public"), definition: workPath(config.CeremonyPath), definitionSig: workPath(config.CeremonySignature), phase: "phase1", + coordinatorKey: trustPath(config.CoordinatorPublicKey), ceremonyBinary: config.CeremonyBinary, client: coordinatorClient(config, config.PublishedBucket)} + if err := checkRole(o); err != nil { + return err + } + return runWithProgress("reconciling retained authenticated publication", func() error { + return recoverInitialPublicationWithStore(o, legacyRecoveryStore(config), workPath(attempt.Command[7]), workPath(attempt.Command[9])) + }) +} + +func executeLegacyRemoteRecovery(p guidedProfile, draft coordinatorDraft, dir string, attempt *flowAttempt, commit string) error { + tag := "role-images-" + commit + image, _, err := verifiedReleaseImage(tag, "coordinator", p.Platform) + if err != nil { + return err + } + if err := prepareGuidedImage(image, p.Platform, "docker", true); err != nil { + return err + } + command := []string{"relay", "coordinator", "recover-initial-1828", "--attempt-id", attempt.ID} + options := p.options() + options.image = image + options.credentials = draft.Credentials + options.recoveryContext = dir + argv, err := dockerRoleArgs(options, command, os.Getuid(), os.Getgid()) + if err != nil { + return err + } + client := osDockerCommandClient{binary: "docker"} + _, endpoint, err := resolveDockerEndpoint(client) + if err != nil { + return err + } + if err := validateLocalDockerEndpoint(endpoint); err != nil { + return err + } + fmt.Fprintln(legacyRecoveryOutput, "Reconciling the exact retained publication with create-only writes. Existing differing bytes will stop recovery.") + if err := client.BindHost(endpoint).Attached(legacyRecoveryOutput, legacyRecoveryOutput, argv...); err != nil { + return fmt.Errorf("retained publication was not reconciled: %w", err) + } + return nil +} + +// runLegacyPublicationRecovery opens one deliberately narrow compatibility +// bridge. It never changes the frozen release, image, definition, command, or +// ceremony files in the old profile. The current image is used ephemerally to +// reconcile storage, after which the old workflow can resume. +func runLegacyPublicationRecovery(args []string) error { + if len(args) == 0 { + return errors.New("usage: relay ceremony recover-publication NAME [--settings-root DIR]") + } + root, err := guidedRoot() + if err != nil { + return err + } + set := flag.NewFlagSet("ceremony recover-publication", flag.ContinueOnError) + set.StringVar(&root, "settings-root", root, "private saved-settings directory containing the retained workflow") + if err := set.Parse(args[1:]); err != nil { + return err + } + if len(set.Args()) != 0 { + return errors.New("unexpected recovery arguments") + } + commit := legacyRecoveryLauncherCommit() + if !launcherReleaseTag.MatchString("role-images-"+commit) || commit == expiredAWSRecoveryRelease { + return errors.New("publication recovery requires an installed, attested hotfix launcher release") + } + dir, err := guidedDirectory(root, args[0], "coordinator") + if err != nil { + return err + } + profilePath := filepath.Join(dir, "profile.json") + p, err := readGuidedProfile(profilePath, args[0], "coordinator") + if err != nil { + return err + } + if p.ReleaseCommit != expiredAWSRecoveryRelease { + return fmt.Errorf("this recovery command only supports the affected release %s", expiredAWSRecoveryRelease) + } + draftPath := filepath.Join(p.Work, "coordinator-setup", "draft.json") + var draft coordinatorDraft + if err := setupReadJSON(draftPath, &draft); err != nil { + return fmt.Errorf("load retained coordinator preparation: %w", err) + } + if err := validateFrozenRecoveryDraft(p, draft, false); err != nil { + return err + } + statePath := filepath.Join(dir, "workflow", "state.json") + var workflow roleFlowState + if err := setupReadJSON(statePath, &workflow); err != nil { + return fmt.Errorf("load retained workflow: %w", err) + } + attempt, err := validateFrozenRecoveryState(p, workflow) + if err != nil { + return err + } + f := roleFlow{state: workflow} + if err := f.checkAttemptEvidence(attempt); err != nil { + return fmt.Errorf("retained publication inputs are no longer exact: %w", err) + } + if err := awsStorageUnchanged(attempt.Command[5], draft, f); err != nil { + return fmt.Errorf("validate unchanged AWS publication target: %w", err) + } + if legacyRecoveryCompleted(workflow, attempt) { + fmt.Fprintln(legacyRecoveryOutput, "Recovery was already durably checkpointed. Resume the ceremony with its original 1828 start script.") + return nil + } + if attempt.Status != "running" && attempt.Status != "failed" { + return errors.New("the retained publication is not an unresolved recoverable attempt") + } + ui := coordinatorWizard{d: draft, draftPath: draftPath, input: bufio.NewReader(legacyRecoveryInput), output: legacyRecoveryOutput} + fmt.Fprintf(legacyRecoveryOutput, "Recovering only retained attempt %s from release %s.\n", attempt.ID, expiredAWSRecoveryRelease) + if err := ui.refreshWorkflowCredentialsExcept(dir, p, attempt.ID); err != nil { + return fmt.Errorf("refresh only the credential-file reference: %w", err) + } + if err := legacyRecoveryAfterCredentialRefresh(); err != nil { + return err + } + + // Reload after the atomic credential checkpoint and then hold the workflow + // lock through remote reconciliation and the final local checkpoint. + p, err = readGuidedProfile(profilePath, args[0], "coordinator") + if err != nil { + return err + } + lock, err := acquireParticipantRunLock(statePath, filepath.Dir(statePath)) + if err != nil { + return err + } + defer lock.release() + if err := setupReadJSON(statePath, &workflow); err != nil { + return err + } + var currentDraft coordinatorDraft + if err := setupReadJSON(draftPath, ¤tDraft); err != nil { + return err + } + if !reflect.DeepEqual(currentDraft, draft) { + return errors.New("coordinator preparation changed during credential rotation; stopped before remote recovery") + } + if err := validateFrozenRecoveryDraft(p, currentDraft, true); err != nil { + return err + } + attempt, err = validateFrozenRecoveryState(p, workflow) + if err != nil { + return err + } + f = roleFlow{state: workflow} + if err := f.checkAttemptEvidence(attempt); err != nil { + return err + } + if err := awsStorageUnchanged(attempt.Command[5], currentDraft, f); err != nil { + return fmt.Errorf("revalidate unchanged AWS publication target: %w", err) + } + if legacyRecoveryCompleted(workflow, attempt) { + fmt.Fprintln(legacyRecoveryOutput, "Recovery was already durably checkpointed. Resume the ceremony with its original 1828 start script.") + return nil + } + if attempt.Status != "running" && attempt.Status != "failed" { + return errors.New("the retained publication changed state during credential rotation") + } + + if err := legacyRecoveryRemote(p, currentDraft, dir, attempt, commit); err != nil { + return err + } + + now := time.Now().UTC().Format(time.RFC3339Nano) + if err := markLegacyRecoveryComplete(&workflow, attempt.ID, commit, now); err != nil { + return err + } + if err := persistLegacyRecoveryCompletion(statePath, workflow, p, attempt.ID, legacyRecoverySave); err != nil { + return errors.Join(err, errors.New("preserve all files and rerun this command; it will verify whether the completion checkpoint landed")) + } + fmt.Fprintln(legacyRecoveryOutput, "Recovery complete. Resume the ceremony with its original 1828 start script; its signed definition and frozen runtime remain unchanged.") + return nil +} diff --git a/cmd/relay/legacy_publication_recovery_test.go b/cmd/relay/legacy_publication_recovery_test.go new file mode 100644 index 0000000..c4bbe78 --- /dev/null +++ b/cmd/relay/legacy_publication_recovery_test.go @@ -0,0 +1,408 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/zksecurity/relay/internal/access" + "github.com/zksecurity/relay/internal/state" +) + +func writeRecoveryFixtureFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestFrozen1828GoldenContextRemainsCompatible(t *testing.T) { + root := filepath.Join("testdata", "frozen-1828-recovery") + var profile guidedProfile + if err := setupReadJSON(filepath.Join(root, "profile.json"), &profile); err != nil { + t.Fatal(err) + } + var workflow roleFlowState + if err := setupReadJSON(filepath.Join(root, "state.json"), &workflow); err != nil { + t.Fatal(err) + } + var draft coordinatorDraft + if err := setupReadJSON(filepath.Join(root, "draft.json"), &draft); err != nil { + t.Fatal(err) + } + attempt, err := validateFrozenRecoveryState(profile, workflow) + if err != nil || attempt.ID != "flow-"+strings.Repeat("b", 32) { + t.Fatalf("1828 golden workflow rejected: attempt=%v err=%v", attempt, err) + } + if err := validateFrozenRecoveryDraft(profile, draft, false); err != nil { + t.Fatalf("1828 golden preparation rejected: %v", err) + } +} + +func frozen1828RecoveryFixture(t *testing.T) (string, string, guidedProfile, coordinatorDraft, string) { + t.Helper() + root := t.TempDir() + settingsRoot := filepath.Join(root, "settings") + work, trust, keys := filepath.Join(root, "work"), filepath.Join(root, "trust"), filepath.Join(root, "keys") + for _, dir := range []string{settingsRoot, work, trust, keys} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + oldCredentials, newCredentials := filepath.Join(root, "old.aws"), filepath.Join(root, "new.aws") + writeRecoveryFixtureFile(t, oldCredentials, "[default]\naws_access_key_id=old\naws_secret_access_key=old\n") + writeRecoveryFixtureFile(t, newCredentials, "[default]\naws_access_key_id=new\naws_secret_access_key=new\n") + name := "frozen-1828-fixture" + p := guidedProfile{Schema: guidedSchema, Name: name, Role: "coordinator", ReleaseCommit: expiredAWSRecoveryRelease, + Image: "sha256:" + strings.Repeat("a", 64), Platform: "linux/arm64", Work: work, Trust: trust, Keys: keys, Credentials: oldCredentials} + dir, err := guidedDirectory(settingsRoot, name, "coordinator") + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "workflow"), 0o700); err != nil { + t.Fatal(err) + } + if err := saveJSONAtomic(filepath.Join(dir, "profile.json"), p); err != nil { + t.Fatal(err) + } + settings := storageSettingsFixture() + config, err := settings.infrastructure() + if err != nil { + t.Fatal(err) + } + config.Schema = access.StorageConfigSchema + config.CeremonyID = "sha256:" + strings.Repeat("c", 64) + config.CeremonyPath = "/work/ceremony/public/ceremony.json" + config.CeremonySignature = "/work/ceremony/public/ceremony.sig" + config.CoordinatorPublicKey = "/trust/coordinator-public-key.hex" + config.CeremonyBinary = "mpc-ceremony" + storagePath := filepath.Join(work, "ceremony", "config", "relay-storage.json") + if err := os.MkdirAll(filepath.Dir(storagePath), 0o700); err != nil { + t.Fatal(err) + } + if err := writeJSONNoReplace(storagePath, config, 0o600); err != nil { + t.Fatal(err) + } + chainPath := filepath.Join(work, "ceremony", "public", "phase1", "chain-0000.json") + signaturePath := filepath.Join(work, "ceremony", "public", "phase1", "chain-0000.sig") + writeRecoveryFixtureFile(t, chainPath, "retained-chain") + writeRecoveryFixtureFile(t, signaturePath, "retained-signature") + command := []string{"relay", "coordinator", "publish", "--verify", "--storage", "/work/ceremony/config/relay-storage.json", "--chain", "/work/ceremony/public/phase1/chain-0000.json", "--chain-signature", "/work/ceremony/public/phase1/chain-0000.sig"} + bindings := map[string]string{} + for index, hostPath := range []string{storagePath, chainPath, signaturePath} { + digest, err := setupFileHash(hostPath) + if err != nil { + t.Fatal(err) + } + bindings[command[5+index*2]] = digest + } + attempt := flowAttempt{ID: "flow-" + strings.Repeat("b", 32), Task: "publish", Stage: "storage", Status: "failed", + OperationSchema: flowOperationSchema, RecoveryClass: recoveryPublication, ImageDigest: p.Image, Platform: p.Platform, + Command: command, Mounts: map[string]string{"/work": work, "/trust": trust, "/keys": keys}, InputBindings: bindings} + workflow := roleFlowState{Schema: roleFlowSchema, Name: name, Role: "coordinator", Profile: p, Attempts: []flowAttempt{attempt}, Values: map[string]string{}} + if err := saveJSONAtomic(filepath.Join(dir, "workflow", "state.json"), workflow); err != nil { + t.Fatal(err) + } + draft := coordinatorDraft{Name: name, Release: "role-images-" + expiredAWSRecoveryRelease, Work: work, Trust: trust, Keys: keys, + Credentials: newCredentials, Storage: settings.Settings} + draftPath := filepath.Join(work, "coordinator-setup", "draft.json") + if err := os.MkdirAll(filepath.Dir(draftPath), 0o700); err != nil { + t.Fatal(err) + } + if err := saveJSONAtomic(draftPath, draft); err != nil { + t.Fatal(err) + } + return settingsRoot, name, p, draft, attempt.ID +} + +func TestFrozen1828RecoveryPathAndIdempotentRerun(t *testing.T) { + settingsRoot, name, oldProfile, draft, attemptID := frozen1828RecoveryFixture(t) + oldCommit, oldInput, oldOutput, oldRemote, oldHook, oldSave := legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote, legacyRecoveryAfterCredentialRefresh, legacyRecoverySave + defer func() { + legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote = oldCommit, oldInput, oldOutput, oldRemote + legacyRecoveryAfterCredentialRefresh, legacyRecoverySave = oldHook, oldSave + }() + commit := strings.Repeat("d", 40) + legacyRecoveryLauncherCommit = func() string { return commit } + legacyRecoveryInput = strings.NewReader("UPDATE CREDENTIAL REFERENCES\n") + var output bytes.Buffer + legacyRecoveryOutput = &output + remoteCalls := 0 + legacyRecoveryRemote = func(p guidedProfile, gotDraft coordinatorDraft, dir string, attempt *flowAttempt, gotCommit string) error { + remoteCalls++ + if p.ReleaseCommit != expiredAWSRecoveryRelease || p.Image != oldProfile.Image || p.Credentials != draft.Credentials || + !reflect.DeepEqual(gotDraft, draft) || attempt.ID != attemptID || gotCommit != commit { + t.Fatal("remote recovery did not receive the fully revalidated frozen context") + } + return nil + } + args := []string{name, "--settings-root", settingsRoot} + if err := runLegacyPublicationRecovery(args); err != nil { + t.Fatal(err) + } + if remoteCalls != 1 { + t.Fatalf("remote calls=%d, want 1", remoteCalls) + } + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + var state roleFlowState + if err := setupReadJSON(filepath.Join(dir, "workflow", "state.json"), &state); err != nil { + t.Fatal(err) + } + attempt, err := validateFrozenRecoveryState(state.Profile, state) + if err != nil || !legacyRecoveryCompleted(state, attempt) { + t.Fatalf("completion not retained: attempt=%v err=%v", attempt, err) + } + legacyRecoveryLauncherCommit = func() string { return strings.Repeat("f", 40) } + legacyRecoveryInput = strings.NewReader("") + legacyRecoveryRemote = func(guidedProfile, coordinatorDraft, string, *flowAttempt, string) error { + t.Fatal("durably completed recovery must not rerun the remote operation") + return nil + } + if err := runLegacyPublicationRecovery(args); err != nil { + t.Fatal(err) + } +} + +func TestRecoveryContextMountIsInternalAndReadOnly(t *testing.T) { + o := roleTestOptions(t) + o.recoveryContext = privateRoleTestDir(t) + args, err := dockerRoleArgs(o, []string{"relay", "coordinator", "recover-initial-1828", "--attempt-id", "flow-" + strings.Repeat("a", 32)}, 501, 20) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(args, " ") + if !strings.Contains(joined, "src="+o.recoveryContext+",dst=/recovery,readonly") { + t.Fatalf("recovery context is not mounted read-only: %s", joined) + } + o.recoveryContext = "" + args, err = dockerRoleArgs(o, []string{"relay", "coordinator", "publish"}, 501, 20) + if err != nil { + t.Fatal(err) + } + if strings.Contains(strings.Join(args, " "), "dst=/recovery") { + t.Fatal("ordinary role invocation unexpectedly received recovery context") + } +} + +func TestPostRotationMutationsStopBeforeRemoteRecovery(t *testing.T) { + mutations := map[string]func(t *testing.T, settingsRoot, name string, draft coordinatorDraft){ + "profile": func(t *testing.T, settingsRoot, name string, _ coordinatorDraft) { + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + path := filepath.Join(dir, "profile.json") + var p guidedProfile + if err := setupReadJSON(path, &p); err != nil { + t.Fatal(err) + } + p.Image = "sha256:" + strings.Repeat("9", 64) + if err := saveJSONAtomic(path, p); err != nil { + t.Fatal(err) + } + }, + "workflow command": func(t *testing.T, settingsRoot, name string, _ coordinatorDraft) { + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + path := filepath.Join(dir, "workflow", "state.json") + var s roleFlowState + if err := setupReadJSON(path, &s); err != nil { + t.Fatal(err) + } + s.Attempts[0].Command[3] = "--closed" + if err := saveJSONAtomic(path, s); err != nil { + t.Fatal(err) + } + }, + "draft storage": func(t *testing.T, _ string, _ string, draft coordinatorDraft) { + draft.Storage["published-bucket"] = "changed-bucket" + if err := saveJSONAtomic(filepath.Join(draft.Work, "coordinator-setup", "draft.json"), draft); err != nil { + t.Fatal(err) + } + }, + "attempt status": func(t *testing.T, settingsRoot, name string, _ coordinatorDraft) { + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + path := filepath.Join(dir, "workflow", "state.json") + var s roleFlowState + if err := setupReadJSON(path, &s); err != nil { + t.Fatal(err) + } + s.Attempts[0].Status = "succeeded" + if err := saveJSONAtomic(path, s); err != nil { + t.Fatal(err) + } + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + settingsRoot, ceremony, _, draft, _ := frozen1828RecoveryFixture(t) + oldCommit, oldInput, oldOutput, oldRemote, oldHook := legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote, legacyRecoveryAfterCredentialRefresh + defer func() { + legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote, legacyRecoveryAfterCredentialRefresh = oldCommit, oldInput, oldOutput, oldRemote, oldHook + }() + legacyRecoveryLauncherCommit = func() string { return strings.Repeat("d", 40) } + legacyRecoveryInput = strings.NewReader("UPDATE CREDENTIAL REFERENCES\n") + legacyRecoveryOutput = &bytes.Buffer{} + legacyRecoveryAfterCredentialRefresh = func() error { mutate(t, settingsRoot, ceremony, draft); return nil } + legacyRecoveryRemote = func(guidedProfile, coordinatorDraft, string, *flowAttempt, string) error { + t.Fatal("mutation reached remote recovery") + return nil + } + if err := runLegacyPublicationRecovery([]string{ceremony, "--settings-root", settingsRoot}); err == nil { + t.Fatal("post-rotation mutation was accepted") + } + }) + } +} + +func TestRemoteSuccessCheckpointNotLandedRerunsCreateOnly(t *testing.T) { + settingsRoot, name, _, _, _ := frozen1828RecoveryFixture(t) + oldCommit, oldInput, oldOutput, oldRemote, oldSave := legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote, legacyRecoverySave + defer func() { + legacyRecoveryLauncherCommit, legacyRecoveryInput, legacyRecoveryOutput, legacyRecoveryRemote, legacyRecoverySave = oldCommit, oldInput, oldOutput, oldRemote, oldSave + }() + commit := strings.Repeat("d", 40) + legacyRecoveryLauncherCommit = func() string { return commit } + legacyRecoveryInput = strings.NewReader("UPDATE CREDENTIAL REFERENCES\n") + legacyRecoveryOutput = &bytes.Buffer{} + source := filepath.Join(t.TempDir(), "blob") + writeRecoveryFixtureFile(t, source, "exact retained bytes") + ref := state.Ref{Name: "phase1/chain-0000.json", SHA256: "sha256:" + strings.Repeat("a", 64)} + sig := state.Ref{Name: "phase1/chain-0000.sig", SHA256: "sha256:" + strings.Repeat("b", 64)} + pointer := state.Pointer{Schema: state.Schema, CeremonyID: "sha256:" + strings.Repeat("c", 64), Phase: "phase1", Index: 0, Chain: ref, ChainSignature: sig, UpdatedAt: "2026-01-01T00:00:00Z", Files: []state.Ref{ref, sig}} + raw, _ := pointer.Encode() + pointerPath := filepath.Join(t.TempDir(), "head.json") + writeRecoveryFixtureFile(t, pointerPath, string(raw)) + store := &publicationStoreFake{objects: map[string][]byte{}} + remoteCalls := 0 + legacyRecoveryRemote = func(guidedProfile, coordinatorDraft, string, *flowAttempt, string) error { + remoteCalls++ + if err := reconcilePublicationObject(store, "blob", source, "blob"); err != nil { + return err + } + present, _ := store.Head("head") + return reconcileInitialPointer(store, "head", pointerPath, pointer, present) + } + legacyRecoverySave = func(string, any) error { return errors.New("checkpoint did not land") } + args := []string{name, "--settings-root", settingsRoot} + if err := runLegacyPublicationRecovery(args); err == nil { + t.Fatal("missing checkpoint should remain an error") + } + putsAfterFirst := store.puts + legacyRecoveryInput = strings.NewReader("") + legacyRecoverySave = saveJSONAtomic + if err := runLegacyPublicationRecovery(args); err != nil { + t.Fatal(err) + } + if remoteCalls != 2 || store.puts != putsAfterFirst { + t.Fatalf("remoteCalls=%d puts=%d firstPuts=%d", remoteCalls, store.puts, putsAfterFirst) + } +} + +func TestAmbiguousRecoveryCheckpointIsReconciled(t *testing.T) { + settingsRoot, name, _, _, attemptID := frozen1828RecoveryFixture(t) + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + statePath := filepath.Join(dir, "workflow", "state.json") + var state roleFlowState + if err := setupReadJSON(statePath, &state); err != nil { + t.Fatal(err) + } + commit := strings.Repeat("e", 40) + if err := markLegacyRecoveryComplete(&state, attemptID, commit, "2026-01-01T00:00:00Z"); err != nil { + t.Fatal(err) + } + ambiguous := errors.New("ambiguous fsync result") + err := persistLegacyRecoveryCompletion(statePath, state, state.Profile, attemptID, func(path string, value any) error { + if err := saveJSONAtomic(path, value); err != nil { + return err + } + return ambiguous + }) + if err != nil { + t.Fatalf("landed checkpoint must reconcile despite ambiguous return: %v", err) + } +} + +func TestGenericPublishHasNoRecoverySwitch(t *testing.T) { + if err := runPublish([]string{"--recover-initial"}); err == nil || !strings.Contains(err.Error(), "flag provided but not defined") { + t.Fatalf("direct recovery switch unexpectedly accepted: %v", err) + } + if err := runRecoverInitial1828([]string{"--attempt-id", "flow-" + strings.Repeat("a", 32)}); err == nil || !strings.Contains(err.Error(), "locked recovery profile") { + t.Fatalf("internal recovery ran without the locked 1828 context: %v", err) + } +} + +func TestFrozen1828InternalRecoveryReconcilesExactObjects(t *testing.T) { + settingsRoot, name, profile, _, attemptID := frozen1828RecoveryFixture(t) + dir, _ := guidedDirectory(settingsRoot, name, "coordinator") + public := filepath.Join(profile.Work, "ceremony", "public") + for path, contents := range map[string]string{ + filepath.Join(public, "ceremony.json"): "signed-definition", + filepath.Join(public, "ceremony.sig"): "definition-signature", + filepath.Join(public, "ownership-destination.ccs"): "signed-r1cs", + filepath.Join(public, "phase1", "genesis.bin"): "signed-genesis", + filepath.Join(profile.Trust, "coordinator-public-key.hex"): strings.Repeat("1", 64), + } { + writeRecoveryFixtureFile(t, path, contents) + } + r1csPath := filepath.Join(public, "ownership-destination.ccs") + genesisPath := filepath.Join(public, "phase1", "genesis.bin") + r1csHash, _ := setupFileHash(r1csPath) + genesisHash, _ := setupFileHash(genesisPath) + r1csHash, genesisHash = "sha256:"+r1csHash, "sha256:"+genesisHash + r1csInfo, _ := os.Stat(r1csPath) + genesisInfo, _ := os.Stat(genesisPath) + ceremonyID := "sha256:" + strings.Repeat("c", 64) + blake := "blake2b256:" + strings.Repeat("0", 64) + definitionResult := fmt.Sprintf(`{"schema":"proof-tool-mpc-command-result-v1","ok":true,"command":"inspect definition","definition_inspection":{"schema":"proof-tool-mpc-definition-inspection-v1","ceremony_id":"%s","mode":"rehearsal","phase1_participants":["participant-1"],"phase2_participants":["participant-1"],"r1cs":{"name":"ownership-destination.ccs","digest":{"sha256":"%s","blake2b256":"%s","size":%d}}}}`, ceremonyID, r1csHash, blake, r1csInfo.Size()) + chainResult := fmt.Sprintf(`{"schema":"proof-tool-mpc-command-result-v1","ok":true,"command":"inspect chain","chain_inspection":{"schema":"proof-tool-mpc-chain-inspection-v1","ceremony_id":"%s","phase":"phase1","accepted_count":0,"artifacts":[{"name":"phase1/genesis.bin","digest":{"sha256":"%s","blake2b256":"%s","size":%d}}],"records":[]}}`, ceremonyID, genesisHash, blake, genesisInfo.Size()) + tool := filepath.Join(t.TempDir(), "mpc-ceremony") + script := "#!/bin/sh\ncase \"$3 $4\" in\n 'inspect definition') printf '%s\\n' '" + definitionResult + "' ;;\n 'inspect chain') printf '%s\\n' '" + chainResult + "' ;;\n *) exit 2 ;;\nesac\n" + if err := os.WriteFile(tool, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + storagePath := filepath.Join(profile.Work, "ceremony", "config", "relay-storage.json") + config, err := loadStorageConfig(storagePath) + if err != nil { + t.Fatal(err) + } + config.CeremonyBinary = tool + if err := saveJSONAtomic(storagePath, config); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(dir, "workflow", "state.json") + var workflow roleFlowState + if err := setupReadJSON(statePath, &workflow); err != nil { + t.Fatal(err) + } + storageHash, _ := setupFileHash(storagePath) + workflow.Attempts[0].InputBindings[workflow.Attempts[0].Command[5]] = storageHash + if err := saveJSONAtomic(statePath, workflow); err != nil { + t.Fatal(err) + } + fake := &publicationStoreFake{objects: map[string][]byte{}} + oldContext, oldWork, oldTrust, oldStore := legacyRecoveryContextRoot, legacyRecoveryWorkRoot, legacyRecoveryTrustRoot, legacyRecoveryStore + defer func() { + legacyRecoveryContextRoot, legacyRecoveryWorkRoot, legacyRecoveryTrustRoot, legacyRecoveryStore = oldContext, oldWork, oldTrust, oldStore + }() + legacyRecoveryContextRoot, legacyRecoveryWorkRoot, legacyRecoveryTrustRoot = dir, profile.Work, profile.Trust + legacyRecoveryStore = func(access.StorageConfig) publicationStore { return fake } + args := []string{"--attempt-id", attemptID} + if err := runRecoverInitial1828(args); err != nil { + t.Fatal(err) + } + firstPuts := fake.puts + if firstPuts == 0 { + t.Fatal("integrated recovery created no objects") + } + if err := runRecoverInitial1828(args); err != nil { + t.Fatal(err) + } + if fake.puts != firstPuts { + t.Fatalf("idempotent internal rerun added writes: first=%d after=%d", firstPuts, fake.puts) + } +} diff --git a/cmd/relay/main.go b/cmd/relay/main.go index 53206c1..208974c 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -88,6 +88,7 @@ func usage() { relay ceremony open NAME --role ROLE [--grant FILE] [--resume-candidate DIR] relay ceremony open NAME --role ROLE --action ACTION [-- TOOL ARGS...] relay ceremony guide NAME --role ROLE + relay ceremony recover-publication NAME [--settings-root DIR] relay ceremony prepare --name NAME --role ROLE --release RELEASE --work DIR --trust DIR --keys DIR relay role --role ROLE --image DIGEST --work DIR [mount flags] -- TOOL ARGS... relay coordinator configure-storage [provider and ceremony flags] --out FILE @@ -176,6 +177,8 @@ func runCoordinator(args []string) error { return runEvidenceInbox(args[1:]) case "publish": return runPublish(args[1:]) + case "recover-initial-1828": + return runRecoverInitial1828(args[1:]) default: return fmt.Errorf("unknown coordinator command %q", args[0]) } @@ -253,6 +256,8 @@ func runCeremony(args []string) error { return runRolePrepare(args[1:]) case "guide": return runRoleFlow(args[1:]) + case "recover-publication": + return runLegacyPublicationRecovery(args[1:]) case "setup": return runGuidedSetup(args[1:]) case "open": diff --git a/cmd/relay/publication_recovery.go b/cmd/relay/publication_recovery.go new file mode 100644 index 0000000..c2844a9 --- /dev/null +++ b/cmd/relay/publication_recovery.go @@ -0,0 +1,394 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/zksecurity/relay/internal/state" + "github.com/zksecurity/relay/internal/store" + "github.com/zksecurity/relay/internal/transcript" +) + +type publicationStore interface { + Head(string) (bool, error) + Size(string) (int64, error) + Get(string, string) error + PutNoReplace(string, string) error +} + +func copyRecoverySnapshot(source, destination string) error { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("recovery inputs must be regular files, not links or special files") + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return err + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + _, copyErr := io.Copy(output, input) + closeErr := output.Close() + if copyErr != nil || closeErr != nil { + _ = os.Remove(destination) + return errors.Join(copyErr, closeErr) + } + return nil +} + +func publicationLogicalName(root, target string, files []transcript.File) (string, error) { + for _, file := range files { + local, err := transcript.Resolve(root, file.Name) + if err != nil { + return "", err + } + if sameLocalPath(local, target) { + return file.Name, nil + } + } + return "", errors.New("authenticated publication file set does not contain a required input") +} + +func snapshotInitialPublication(o roleOpts, chainPath, chainSignaturePath string) (roleOpts, transcript.Definition, transcript.Chain, []transcript.File, func(), error) { + definition, err := o.inspector().Definition() + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + chain, err := o.inspector().Chain(chainPath, chainSignaturePath) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + if chain.Phase != "phase1" || chain.AcceptedCount() != 0 { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, errors.New("recovery is limited to the retained initial phase1 publication at index 0") + } + files, err := transcript.TranscriptFiles(o.root, chain) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + for _, file := range files { + if strings.HasPrefix(file.Name, "phase1/closure/") || strings.HasPrefix(file.Name, "phase1/beacon/") || strings.HasPrefix(file.Name, "phase1/sealed/") { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, errors.New("initial publication recovery refuses phase-ending files") + } + } + definitionName, err := publicationLogicalName(o.root, o.definition, files) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + definitionSignatureName, err := publicationLogicalName(o.root, o.definitionSig, files) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + chainName, err := publicationLogicalName(o.root, chainPath, files) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + chainSignatureName, err := publicationLogicalName(o.root, chainSignaturePath, files) + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + temp, err := os.MkdirTemp("", "relay-retained-publication-") + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + cleanup := func() { _ = os.RemoveAll(temp) } + seen := map[string]bool{} + for _, file := range files { + if seen[file.Name] { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, fmt.Errorf("duplicate publication file %q", file.Name) + } + seen[file.Name] = true + source, err := transcript.Resolve(o.root, file.Name) + if err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + destination, err := transcript.Resolve(temp, file.Name) + if err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + if err := copyRecoverySnapshot(source, destination); err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, fmt.Errorf("snapshot %s: %w", file.Name, err) + } + } + trustedKey := filepath.Join(temp, ".trust", "coordinator-public-key.hex") + if err := copyRecoverySnapshot(o.coordinatorKey, trustedKey); err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, fmt.Errorf("snapshot coordinator public key: %w", err) + } + snapshot := o + snapshot.root = temp + snapshot.definition = filepath.Join(temp, filepath.FromSlash(definitionName)) + snapshot.definitionSig = filepath.Join(temp, filepath.FromSlash(definitionSignatureName)) + snapshot.coordinatorKey = trustedKey + snapshotChainPath := filepath.Join(temp, filepath.FromSlash(chainName)) + snapshotSignaturePath := filepath.Join(temp, filepath.FromSlash(chainSignatureName)) + snapshotDefinition, err := snapshot.inspector().Definition() + if err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, fmt.Errorf("authenticate snapshotted definition: %w", err) + } + snapshotChain, err := snapshot.inspector().Chain(snapshotChainPath, snapshotSignaturePath) + if err != nil { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, fmt.Errorf("authenticate snapshotted chain: %w", err) + } + snapshotFiles, err := transcript.TranscriptFiles(temp, snapshotChain) + if err != nil || !reflect.DeepEqual(files, snapshotFiles) { + cleanup() + if err != nil { + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, err + } + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, errors.New("publication file set changed while it was being snapshotted") + } + if !reflect.DeepEqual(definition, snapshotDefinition) || chain.Phase != snapshotChain.Phase || chain.AcceptedCount() != snapshotChain.AcceptedCount() { + cleanup() + return o, transcript.Definition{}, transcript.Chain{}, nil, func() {}, errors.New("authenticated publication changed while it was being snapshotted") + } + return snapshot, snapshotDefinition, snapshotChain, snapshotFiles, cleanup, nil +} + +func verifyRemotePublicationObject(client publicationStore, key, source, label string) error { + expected, expectedSize, err := transcript.DigestFile(source) + if err != nil { + return err + } + remoteSize, err := client.Size(key) + if err != nil { + return fmt.Errorf("inspect retained object size %s: %w", label, err) + } + if remoteSize != expectedSize { + return fmt.Errorf("integrity conflict at %s: existing object size differs from the retained authenticated file", key) + } + dir, err := os.MkdirTemp("", "relay-publication-recovery-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + destination := filepath.Join(dir, "object") + if err := client.Get(key, destination); err != nil { + return fmt.Errorf("read retained object %s: %w", label, err) + } + actual, actualSize, err := transcript.DigestFile(destination) + if err != nil { + return err + } + if actual != expected || actualSize != expectedSize { + return fmt.Errorf("integrity conflict at %s: existing object differs from the retained authenticated file", key) + } + return nil +} + +func reconcilePublicationObject(client publicationStore, key, source, label string) error { + present, err := client.Head(key) + if err != nil { + return fmt.Errorf("inspect retained object %s: %w", label, err) + } + if !present { + putErr := client.PutNoReplace(key, source) + if putErr != nil && !errors.Is(putErr, store.ErrExists) { + // The request may have reached storage before the response failed. Read + // the exact key before deciding whether another invocation is needed. + if verifyErr := verifyRemotePublicationObject(client, key, source, label); verifyErr == nil { + return nil + } else { + return errors.Join(fmt.Errorf("create retained object %s: %w", label, putErr), verifyErr) + } + } + } + return verifyRemotePublicationObject(client, key, source, label) +} + +func decodeRecoveryPointer(raw []byte) (state.Pointer, error) { + var pointer state.Pointer + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&pointer); err != nil { + return pointer, fmt.Errorf("decode existing phase head: %w", err) + } + if err := decoder.Decode(new(any)); err != io.EOF { + return pointer, errors.New("existing phase head has trailing JSON") + } + return pointer, pointer.Validate() +} + +func equivalentInitialPointer(actual, expected state.Pointer) bool { + updated, err := time.Parse(time.RFC3339, actual.UpdatedAt) + if err != nil { + return false + } + _, offset := updated.Zone() + return actual.Schema == expected.Schema && actual.CeremonyID == expected.CeremonyID && + actual.Phase == "phase1" && actual.Index == 0 && !actual.Closed && + actual.Chain == expected.Chain && actual.ChainSignature == expected.ChainSignature && + reflect.DeepEqual(actual.Files, expected.Files) && offset == 0 +} + +func verifyExistingInitialPointer(client publicationStore, key string, expected state.Pointer) error { + size, err := client.Size(key) + if err != nil { + return fmt.Errorf("inspect existing initial phase head size: %w", err) + } + if size <= 0 || size > 1<<20 { + return errors.New("existing initial phase head exceeds the recovery size limit") + } + dir, err := os.MkdirTemp("", "relay-publication-head-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + path := filepath.Join(dir, "head.json") + if err := client.Get(key, path); err != nil { + return fmt.Errorf("read existing initial phase head: %w", err) + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + actual, err := decodeRecoveryPointer(raw) + if err != nil { + return err + } + if !equivalentInitialPointer(actual, expected) { + return errors.New("existing phase1 head is not the exact retained index-0 publication; stopped without overwriting it") + } + return nil +} + +func reconcileInitialPointer(client publicationStore, key, pointerPath string, expected state.Pointer, present bool) error { + if !present { + putErr := client.PutNoReplace(key, pointerPath) + if putErr != nil && !errors.Is(putErr, store.ErrExists) { + if verifyErr := verifyExistingInitialPointer(client, key, expected); verifyErr != nil { + return errors.Join(fmt.Errorf("create initial phase head: %w", putErr), verifyErr) + } + } + } + return verifyExistingInitialPointer(client, key, expected) +} + +func validateRecoveryFile(file transcript.File, r1cs transcript.ArtifactRef, sum string, size int64) (bool, error) { + if file.HasDigest() && (sum != file.Digest.SHA256 || size != file.Digest.Size) { + return false, fmt.Errorf("%s: retained file does not match its authenticated chain digest", file.Name) + } + if file.Name != r1cs.Name { + return false, nil + } + if sum != r1cs.Digest.SHA256 || size != r1cs.Digest.Size { + return true, fmt.Errorf("%s: retained circuit does not match the signed ceremony definition", file.Name) + } + return true, nil +} + +// recoverInitialPublication is intentionally narrower than ordinary publish. +// It can only finish phase1 index zero and every write is create-only. +func recoverInitialPublication(o roleOpts, chainPath, chainSignaturePath string) error { + return recoverInitialPublicationWithStore(o, o.client, chainPath, chainSignaturePath) +} + +func recoverInitialPublicationWithStore(o roleOpts, client publicationStore, chainPath, chainSignaturePath string) error { + o, definition, chain, files, cleanup, err := snapshotInitialPublication(o, chainPath, chainSignaturePath) + if err != nil { + return err + } + defer cleanup() + chainPath = chain.ChainPath + chainSignaturePath = chain.ChainSignaturePath + r1cs, err := definition.R1CS() + if err != nil { + return err + } + type plannedObject struct { + name, local, sum string + } + var plan []plannedObject + var chainRef, signatureRef state.Ref + r1csFound := false + refs := make([]state.Ref, 0, len(files)) + for _, file := range files { + local, err := transcript.Resolve(o.root, file.Name) + if err != nil { + return err + } + sum, size, err := transcript.DigestFile(local) + if err != nil { + return fmt.Errorf("%s: %w", file.Name, err) + } + isR1CS, err := validateRecoveryFile(file, r1cs, sum, size) + if err != nil { + return err + } + if isR1CS { + r1csFound = true + } + ref := state.Ref{Name: file.Name, SHA256: sum} + refs = append(refs, ref) + if sameLocalPath(local, chainPath) { + chainRef = ref + } + if sameLocalPath(local, chainSignaturePath) { + signatureRef = ref + } + plan = append(plan, plannedObject{name: file.Name, local: local, sum: sum}) + } + if chainRef.Name == "" || signatureRef.Name == "" { + return errors.New("authenticated file set did not contain the retained chain and signature") + } + if !r1csFound { + return errors.New("authenticated file set did not contain the circuit named by the signed definition") + } + pointer := state.Pointer{Schema: state.Schema, CeremonyID: definition.CeremonyID, Phase: "phase1", Index: 0, Chain: chainRef, ChainSignature: signatureRef, UpdatedAt: time.Now().UTC().Format(time.RFC3339), Files: refs} + key := state.Key(definition.CeremonyID, "phase1") + headPresent, err := client.Head(key) + if err != nil { + return fmt.Errorf("inspect initial phase head: %w", err) + } + if headPresent { + if err := verifyExistingInitialPointer(client, key, pointer); err != nil { + return err + } + } + for _, object := range plan { + if err := reconcilePublicationObject(client, store.Key(object.sum), object.local, object.name); err != nil { + return err + } + fmt.Printf(" verified %s\n", object.name) + } + encoded, err := pointer.Encode() + if err != nil { + return err + } + temp, err := os.MkdirTemp("", "relay-publication-pointer-") + if err != nil { + return err + } + defer os.RemoveAll(temp) + pointerPath := filepath.Join(temp, "head.json") + if err := os.WriteFile(pointerPath, encoded, 0o600); err != nil { + return err + } + if err := reconcileInitialPointer(client, key, pointerPath, pointer, headPresent); err != nil { + return err + } + fmt.Println("reconciled the retained phase1 index-0 publication; no object was overwritten") + return nil +} diff --git a/cmd/relay/publication_recovery_test.go b/cmd/relay/publication_recovery_test.go new file mode 100644 index 0000000..eb8bd0d --- /dev/null +++ b/cmd/relay/publication_recovery_test.go @@ -0,0 +1,248 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/zksecurity/relay/internal/access" + "github.com/zksecurity/relay/internal/state" + "github.com/zksecurity/relay/internal/store" + "github.com/zksecurity/relay/internal/transcript" +) + +type publicationStoreFake struct { + objects map[string][]byte + putErr error + puts int + putHook func(string, []byte) error +} + +func (f *publicationStoreFake) Head(key string) (bool, error) { + _, ok := f.objects[key] + return ok, nil +} +func (f *publicationStoreFake) Size(key string) (int64, error) { + raw, ok := f.objects[key] + if !ok { + return 0, errors.New("not found") + } + return int64(len(raw)), nil +} +func (f *publicationStoreFake) Get(key, path string) error { + raw, ok := f.objects[key] + if !ok { + return errors.New("not found") + } + return os.WriteFile(path, raw, 0o600) +} +func (f *publicationStoreFake) PutNoReplace(key, path string) error { + f.puts++ + if _, ok := f.objects[key]; ok { + return store.ErrExists + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if f.putHook != nil { + return f.putHook(key, raw) + } + f.objects[key] = raw + return f.putErr +} + +func TestReconcilePublicationObject(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "genesis.bin") + if err := os.WriteFile(source, []byte("retained genesis"), 0o600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + initial []byte + putErr error + wantErr string + wantPuts int + }{ + {name: "missing", wantPuts: 1}, + {name: "matching", initial: []byte("retained genesis")}, + {name: "conflict", initial: []byte("different"), wantErr: "integrity conflict"}, + {name: "ambiguous write after effect", putErr: errors.New("expired response"), wantPuts: 1}, + } { + t.Run(test.name, func(t *testing.T) { + fake := &publicationStoreFake{objects: map[string][]byte{}, putErr: test.putErr} + if test.initial != nil { + fake.objects["blob"] = test.initial + } + err := reconcilePublicationObject(fake, "blob", source, "phase1/genesis.bin") + if test.wantErr == "" && err != nil { + t.Fatal(err) + } + if test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("error=%v, want substring %q", err, test.wantErr) + } + if fake.puts != test.wantPuts { + t.Fatalf("puts=%d, want %d", fake.puts, test.wantPuts) + } + }) + } +} + +func TestEquivalentInitialPointerIgnoresOnlyTimestamp(t *testing.T) { + ref := state.Ref{Name: "phase1/chain-0000.json", SHA256: "sha256:" + strings.Repeat("a", 64)} + sig := state.Ref{Name: "phase1/chain-0000.sig", SHA256: "sha256:" + strings.Repeat("b", 64)} + want := state.Pointer{Schema: state.Schema, CeremonyID: "sha256:" + strings.Repeat("c", 64), Phase: "phase1", Index: 0, Chain: ref, ChainSignature: sig, UpdatedAt: "2026-01-01T00:00:00Z", Files: []state.Ref{ref, sig}} + got := want + got.UpdatedAt = "2026-01-01T00:00:01Z" + if !equivalentInitialPointer(got, want) { + t.Fatal("timestamp-only difference should reconcile") + } + got.Index = 1 + if equivalentInitialPointer(got, want) { + t.Fatal("advanced head must not reconcile") + } +} + +func TestReconcileInitialPointerHeadRace(t *testing.T) { + ref := state.Ref{Name: "phase1/chain-0000.json", SHA256: "sha256:" + strings.Repeat("a", 64)} + sig := state.Ref{Name: "phase1/chain-0000.sig", SHA256: "sha256:" + strings.Repeat("b", 64)} + want := state.Pointer{Schema: state.Schema, CeremonyID: "sha256:" + strings.Repeat("c", 64), Phase: "phase1", Index: 0, Chain: ref, ChainSignature: sig, UpdatedAt: "2026-01-01T00:00:00Z", Files: []state.Ref{ref, sig}} + raw, err := want.Encode() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "head.json") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + mutate func(*state.Pointer) + wantErr string + }{ + {name: "equivalent", mutate: func(pointer *state.Pointer) { pointer.UpdatedAt = "2026-01-01T00:00:01Z" }}, + {name: "conflicting", mutate: func(pointer *state.Pointer) { pointer.Index = 1 }, wantErr: "not the exact retained"}, + } { + t.Run(test.name, func(t *testing.T) { + fake := &publicationStoreFake{objects: map[string][]byte{}} + fake.putHook = func(key string, _ []byte) error { + competing := want + test.mutate(&competing) + encoded, encodeErr := competing.Encode() + if encodeErr != nil { + return encodeErr + } + fake.objects[key] = encoded + return store.ErrExists + } + err := reconcileInitialPointer(fake, "state/head.json", path, want, false) + if test.wantErr == "" && err != nil { + t.Fatal(err) + } + if test.wantErr != "" && (err == nil || !strings.Contains(err.Error(), test.wantErr)) { + t.Fatalf("error=%v, want substring %q", err, test.wantErr) + } + }) + } +} + +func TestDecodeRecoveryPointerRejectsUnknownFields(t *testing.T) { + raw := []byte(`{"schema":"relay-state-v1","ceremony_id":"sha256:` + strings.Repeat("c", 64) + `","phase":"phase1","index":0,"chain":{"name":"chain","sha256":"sha256:` + strings.Repeat("a", 64) + `"},"chain_signature":{"name":"sig","sha256":"sha256:` + strings.Repeat("b", 64) + `"},"updated_at":"2026-01-01T00:00:00Z","closed":false,"files":[],"surprise":true}`) + if _, err := decodeRecoveryPointer(raw); err == nil { + t.Fatal("unknown pointer fields must not be accepted during recovery") + } +} + +func TestUnresolvedLegacyPublicationIsExact(t *testing.T) { + command := []string{"relay", "coordinator", "publish", "--verify", "--storage", "/work/ceremony/config/relay-storage.json", "--chain", "/work/ceremony/public/phase1/chain-0000.json", "--chain-signature", "/work/ceremony/public/phase1/chain-0000.sig"} + publication := flowAttempt{ID: "flow-" + strings.Repeat("a", 32), Task: "publish", Stage: "storage", Status: "failed", OperationSchema: flowOperationSchema, RecoveryClass: recoveryPublication, Command: command, InputBindings: map[string]string{command[5]: "hash-storage", command[7]: "hash-chain", command[9]: "hash-signature"}} + got, err := unresolvedLegacyPublication(roleFlowState{Attempts: []flowAttempt{publication}}) + if err != nil || got.ID != publication.ID { + t.Fatalf("got=%v err=%v", got, err) + } + publication.Command = append(publication.Command, "--closed") + if _, err := unresolvedLegacyPublication(roleFlowState{Attempts: []flowAttempt{publication}}); err == nil { + t.Fatal("modified publication recipe must not use initial recovery") + } +} + +func TestUnresolvedLegacyPublicationUsesLatestScopedAttempt(t *testing.T) { + command := []string{"relay", "coordinator", "publish", "--verify", "--storage", "/work/ceremony/config/relay-storage.json", "--chain", "/work/ceremony/public/phase1/chain-0000.json", "--chain-signature", "/work/ceremony/public/phase1/chain-0000.sig"} + bindings := map[string]string{command[5]: "hash-storage", command[7]: "hash-chain", command[9]: "hash-signature"} + old := flowAttempt{ID: "flow-" + strings.Repeat("a", 32), Task: "inspect", Stage: "enrollments", Status: "failed", RecoveryClass: recoveryReadOnly} + target := flowAttempt{ID: "flow-" + strings.Repeat("b", 32), Task: "publish", Stage: "storage", Status: "failed", OperationSchema: flowOperationSchema, RecoveryClass: recoveryPublication, Command: command, InputBindings: bindings} + if got, err := unresolvedLegacyPublication(roleFlowState{Attempts: []flowAttempt{old, target}}); err != nil || got.ID != target.ID { + t.Fatalf("got=%v err=%v", got, err) + } + succeeded := target + succeeded.Status = "succeeded" + if _, err := unresolvedLegacyPublication(roleFlowState{Attempts: []flowAttempt{target, succeeded}}); err == nil { + t.Fatal("a later successful storage publication must supersede the old failure") + } +} + +func TestCopyRecoverySnapshotIsIndependent(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source") + destination := filepath.Join(dir, "snapshot", "object") + if err := os.WriteFile(source, []byte("authenticated"), 0o600); err != nil { + t.Fatal(err) + } + if err := copyRecoverySnapshot(source, destination); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("changed later"), 0o600); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(destination) + if err != nil || string(raw) != "authenticated" { + t.Fatalf("snapshot=%q err=%v", raw, err) + } +} + +func TestAWSStorageRecoveryAllowsOnlyCredentialChange(t *testing.T) { + work := t.TempDir() + settings := storageSettingsFixture() + config, err := settings.infrastructure() + if err != nil { + t.Fatal(err) + } + config.Schema = access.StorageConfigSchema + config.CeremonyID = "sha256:" + strings.Repeat("c", 64) + config.CeremonyPath = "/work/ceremony/public/ceremony.json" + config.CeremonySignature = "/work/ceremony/public/ceremony.sig" + config.CoordinatorPublicKey = "/trust/coordinator-public-key.hex" + config.CeremonyBinary = "mpc-ceremony" + path := filepath.Join(work, "relay-storage.json") + if err := writeJSONNoReplace(path, config, 0o600); err != nil { + t.Fatal(err) + } + draft := coordinatorDraft{Storage: settings.Settings} + f := roleFlow{state: roleFlowState{Profile: guidedProfile{Work: work}}} + if err := awsStorageUnchanged("/work/relay-storage.json", draft, f); err != nil { + t.Fatal(err) + } + draft.Storage = map[string]string{} + for key, value := range settings.Settings { + draft.Storage[key] = value + } + draft.Storage["published-bucket"] = "different-public-fixture" + if err := awsStorageUnchanged("/work/relay-storage.json", draft, f); err == nil { + t.Fatal("changed publication bucket must be refused") + } +} + +func TestValidateRecoveryFileRequiresSignedR1CS(t *testing.T) { + r1cs := transcript.ArtifactRef{Name: "ownership-destination.ccs", Digest: transcript.Digest{SHA256: "sha256:" + strings.Repeat("a", 64), Size: 12}} + file := transcript.File{Name: r1cs.Name} + if _, err := validateRecoveryFile(file, r1cs, "sha256:"+strings.Repeat("b", 64), 12); err == nil { + t.Fatal("changed R1CS must be rejected even though TranscriptFiles has no digest on the root file") + } + matched, err := validateRecoveryFile(file, r1cs, r1cs.Digest.SHA256, r1cs.Digest.Size) + if err != nil || !matched { + t.Fatalf("matched=%v err=%v", matched, err) + } +} diff --git a/cmd/relay/testdata/frozen-1828-recovery/draft.json b/cmd/relay/testdata/frozen-1828-recovery/draft.json new file mode 100644 index 0000000..793b3f1 --- /dev/null +++ b/cmd/relay/testdata/frozen-1828-recovery/draft.json @@ -0,0 +1 @@ +{"Schema":"","Name":"frozen-1828-golden","Release":"role-images-1828c720da11a3f9a83ba905a6c3352b6ca06615","Work":"/fixture/work","Trust":"/fixture/trust","Keys":"/fixture/keys","Mode":"","Circuit":"","Status":"","CreatedAt":"","Identities":{"coordinator":{"id":"","display_name":"","key_id":"","ed25519_public_key_hex":"","public_key_fingerprint":""},"release_signer":{"id":"","display_name":"","key_id":"","ed25519_public_key_hex":"","public_key_fingerprint":""},"auditors":null,"roster":null},"Policy":{"phase1_policy":{"participants":null,"minimum":0},"phase2_policy":{"participants":null,"minimum":0},"beacon_policy":{"provider":"","network":"","chain_hash_hex":"","public_key_hex":"","scheme":"","genesis_time_unix":0,"period_seconds":0,"extraction":"","minimum_challenge_bytes":0,"minimum_witness_lead_seconds":0,"future_round_required":false}},"Binaries":null,"Storage":{"grant-role-arn":"arn:aws:iam::123456789012:role/grants","grant-role-max-ttl":"1h","inbox-bucket":"private-fixture","issuer-profile":"issuer","profile":"coordinator","provider":"aws","published-base-url":"https://ceremony.example","published-bucket":"public-fixture","region":"us-east-1"},"Credentials":"/fixture/new.aws","PolicyTemplate":""} diff --git a/cmd/relay/testdata/frozen-1828-recovery/profile.json b/cmd/relay/testdata/frozen-1828-recovery/profile.json new file mode 100644 index 0000000..c0b00b6 --- /dev/null +++ b/cmd/relay/testdata/frozen-1828-recovery/profile.json @@ -0,0 +1 @@ +{"release_commit":"1828c720da11a3f9a83ba905a6c3352b6ca06615","schema":"relay-guided-role-v1","name":"frozen-1828-golden","role":"coordinator","image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","platform":"linux/arm64","work":"/fixture/work","trust":"/fixture/trust","keys":"/fixture/keys","aws_credentials":"/fixture/old.aws"} diff --git a/cmd/relay/testdata/frozen-1828-recovery/state.json b/cmd/relay/testdata/frozen-1828-recovery/state.json new file mode 100644 index 0000000..b141cbd --- /dev/null +++ b/cmd/relay/testdata/frozen-1828-recovery/state.json @@ -0,0 +1 @@ +{"Schema":"relay-role-flow-v2","Name":"frozen-1828-golden","Role":"coordinator","CatalogDigest":"","Profile":{"release_commit":"1828c720da11a3f9a83ba905a6c3352b6ca06615","schema":"relay-guided-role-v1","name":"frozen-1828-golden","role":"coordinator","image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","platform":"linux/arm64","work":"/fixture/work","trust":"/fixture/trust","keys":"/fixture/keys","aws_credentials":"/fixture/old.aws"},"Stage":0,"Values":{},"Attempts":[{"ID":"flow-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","Task":"publish","Stage":"storage","Status":"failed","operation_schema":"relay-guided-operation-v1","recovery_class":"publication","image_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","platform":"linux/arm64","Command":["relay","coordinator","publish","--verify","--storage","/work/ceremony/config/relay-storage.json","--chain","/work/ceremony/public/phase1/chain-0000.json","--chain-signature","/work/ceremony/public/phase1/chain-0000.sig"],"StartedAt":"","FinishedAt":"","Note":"","mounts":{"/keys":"/fixture/keys","/trust":"/fixture/trust","/work":"/fixture/work"},"InputBindings":{"/work/ceremony/config/relay-storage.json":"1111111111111111111111111111111111111111111111111111111111111111","/work/ceremony/public/phase1/chain-0000.json":"2222222222222222222222222222222222222222222222222222222222222222","/work/ceremony/public/phase1/chain-0000.sig":"3333333333333333333333333333333333333333333333333333333333333333"}}],"PublicBindings":null} diff --git a/internal/store/store.go b/internal/store/store.go index 884d08c..9e93bae 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -23,6 +23,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" ) @@ -173,6 +174,19 @@ func (c Client) Head(key string) (bool, error) { return true, nil } +// Size returns the provider-reported byte length of an existing object. +func (c Client) Size(key string) (int64, error) { + raw, err := c.run("head-object", "--bucket", c.Bucket, "--key", key, "--query", "ContentLength", "--output", "text") + if err != nil { + return 0, err + } + size, err := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64) + if err != nil || size < 0 { + return 0, errors.New("object storage returned an invalid content length") + } + return size, nil +} + // Put writes an object, replacing any existing one. // // This is used only for the mutable state pointer, which must be overwritable diff --git a/release/release-notes.md b/release/release-notes.md index 4000b8f..197871b 100644 --- a/release/release-notes.md +++ b/release/release-notes.md @@ -1,5 +1,25 @@ ## What changed +- Added a narrowly scoped recovery command for the affected 1828 coordinator + release when its first Phase 1 publication was interrupted by expired AWS + session credentials. The command accepts only the retained authenticated + index-0 publication, rotates only credential-file references, verifies any + existing remote bytes, uses create-only writes for missing objects and the + initial head, and resumes the original frozen workflow only after complete + reconciliation. It refuses advanced, closed, conflicting, ambiguous or + differently pinned workflows. + Ordinary `coordinator publish` has no recovery switch. The hotfix launcher + invokes a container-only compatibility command with the locked 1828 profile + and workflow mounted read-only, then revalidates the complete profile, + runtime, storage target and retained attempt after credential rotation. + A retry also recognizes a completion checkpoint that landed despite an + ambiguous local save result, without publishing again. + In the original coordinator preparation, choose `6) Storage settings`, then + AWS and the same existing resources to capture a fresh credential snapshot; + do not change any storage value. Then run the hotfix launcher's + `relay ceremony recover-publication CEREMONY_NAME`; on success, resume with + the original release's start script. + - Reworked the ceremony flow map into seven explicit lanes covering identity collection, enrollment and storage distribution, participant custody turns, per-head mirror evidence, closure and beacon timing, phase transition, @@ -89,6 +109,11 @@ ## Tessera compatibility +The recovery command does not change setup contracts, signed definitions, +ceremony data, proof-tool pins, Tessera fields or selected releases. It is an +explicit compatibility bridge for one frozen Relay release and does not make +the hotfix release the ceremony runtime. No Tessera change is required. + Setup contracts, ceremony data, and Tessera request fields are unchanged. The ceremony-flow documentation does not change Tessera integration behavior. The compiled workflow recipe adds handoff tasks while preserving existing task