From 4b415b9e20c01f29c69e534e7f2288c32df086a3 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 00:00:48 -0700 Subject: [PATCH 01/20] [RAPTOR-19525] test(workload): rework fakeFilesClient into self-consistent server model Replace the shallow fakeFilesClient stub with an in-memory server model that records per-path content and SHA-256, models stage REPLACE merge semantics, supports configurable AllFiles pagination, returns numFiles counting all files in the resulting version, and exposes per-method call counters and fault-injection hooks. Guard every shared map with a mutex so -race is clean under 4-way concurrent uploads. Add a syncedProject helper that builds a temp project tree with initialized workload state (config.json + manifest.json matching file hashes) for reuse by later features. Add contract tests for VAL-UPLOAD-015 (empty plan issues zero upload-side calls) and VAL-UPLOAD-017 (dry-run issues zero upload-side calls), with a positive control proving the counters have teeth. Zero production (non-_test.go) files modified. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/engine_test.go | 89 --- .../sync/fake_files_client_self_test.go | 516 ++++++++++++++ .../workload/sync/fake_files_client_test.go | 638 ++++++++++++++++++ internal/workload/sync/project_helper_test.go | 88 +++ .../workload/sync/upload_scaffolding_test.go | 174 +++++ 5 files changed, 1416 insertions(+), 89 deletions(-) create mode 100644 internal/workload/sync/fake_files_client_self_test.go create mode 100644 internal/workload/sync/fake_files_client_test.go create mode 100644 internal/workload/sync/project_helper_test.go create mode 100644 internal/workload/sync/upload_scaffolding_test.go diff --git a/internal/workload/sync/engine_test.go b/internal/workload/sync/engine_test.go index 17cdeadb1..613b87191 100644 --- a/internal/workload/sync/engine_test.go +++ b/internal/workload/sync/engine_test.go @@ -16,10 +16,8 @@ package sync import ( "errors" - "io" "os" "path/filepath" - stdsync "sync" "testing" "time" @@ -56,93 +54,6 @@ func (f *fakeArtifactStore) PatchCodeRef(artifactID, catalogID, catalogVersionID return f.PatchFn(artifactID, catalogID, catalogVersionID) } -// fakeFilesClient is the in-memory FilesAPI fake used by engine tests. -// Unexpected methods return errors so off-happy-path drift fails loudly. -type fakeFilesClient struct { - allFiles map[string]filesapi.FileMeta - catalogID string - versionID string - stageID string - uploadedFiles map[string][]byte - deletedPaths []string - mu stdsync.Mutex -} - -func (f *fakeFilesClient) CreateCatalog() (*filesapi.CatalogResp, error) { - if f.catalogID == "" { - return nil, errors.New("fakeFilesClient.CreateCatalog: no catalogID configured") - } - - return &filesapi.CatalogResp{CatalogID: f.catalogID, CatalogVersionID: ""}, nil -} - -func (f *fakeFilesClient) CreateStage(_ string) (*filesapi.StageResp, error) { - if f.stageID == "" { - return nil, errors.New("fakeFilesClient.CreateStage: no stageID configured") - } - - return &filesapi.StageResp{CatalogID: f.catalogID, StageID: f.stageID}, nil -} - -func (f *fakeFilesClient) UploadToStage(_, _, name string, _ int64, body io.Reader) error { - data, err := io.ReadAll(body) - if err != nil { - return err - } - - f.mu.Lock() - defer f.mu.Unlock() - - if f.uploadedFiles == nil { - f.uploadedFiles = map[string][]byte{} - } - - f.uploadedFiles[name] = data - - return nil -} - -func (f *fakeFilesClient) ApplyStage(_, _, _ string) (*filesapi.ApplyStageResp, error) { - if f.versionID == "" { - return nil, errors.New("fakeFilesClient.ApplyStage: no versionID configured") - } - - return &filesapi.ApplyStageResp{ - CatalogID: f.catalogID, - CatalogVersionID: f.versionID, - NumFiles: len(f.uploadedFiles), - }, nil -} - -func (f *fakeFilesClient) UploadFromZipNew(_ string, _ int64, _ io.Reader) (*filesapi.FromFileResp, error) { - return nil, errors.New("fakeFilesClient: UploadFromZipNew not expected") -} - -func (f *fakeFilesClient) UploadFromZipExisting(_, _, _ string, _ int64, _ io.Reader) (*filesapi.FromFileResp, error) { - return nil, errors.New("fakeFilesClient: UploadFromZipExisting not expected") -} - -func (f *fakeFilesClient) PollStatus(_ string) (*filesapi.StatusResp, error) { - return nil, errors.New("fakeFilesClient: PollStatus not expected") -} - -func (f *fakeFilesClient) AllFiles(_, _ string) (map[string]filesapi.FileMeta, error) { - return f.allFiles, nil -} - -func (f *fakeFilesClient) DownloadFile(_, _, _ string, _ io.Writer) (string, int64, error) { - return "", 0, errors.New("fakeFilesClient: DownloadFile not expected") -} - -func (f *fakeFilesClient) DeleteFiles(_ string, paths []string) (*filesapi.DeleteFilesResp, error) { - f.deletedPaths = append(f.deletedPaths, paths...) - return &filesapi.DeleteFilesResp{}, nil -} - -func (f *fakeFilesClient) ListVersions(_ string, _ int) ([]filesapi.CatalogVersion, error) { - return nil, errors.New("fakeFilesClient: ListVersions not expected") -} - func initProject(t *testing.T, files map[string]string) string { t.Helper() diff --git a/internal/workload/sync/fake_files_client_self_test.go b/internal/workload/sync/fake_files_client_self_test.go new file mode 100644 index 000000000..121433d7d --- /dev/null +++ b/internal/workload/sync/fake_files_client_self_test.go @@ -0,0 +1,516 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "sort" + "strings" + "sync" + "testing" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sha256Hex computes the SHA-256 hex of content, matching what the fake +// records and what the real server returns. +func sha256Hex(data []byte) string { + h := sha256.Sum256(data) + + return hex.EncodeToString(h[:]) +} + +// stageUpload uploads files through the stage path and applies, returning the +// ApplyStage response. This is the helper for tests that exercise the fake's +// server-state model without going through the full engine. +func stageUpload(t *testing.T, fake *fakeFilesClient, files map[string][]byte) *filesapi.ApplyStageResp { + t.Helper() + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + for path, data := range files { + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, int64(len(data)), bytes.NewReader(data)) + require.NoError(t, err) + } + + resp, err := fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + return resp +} + +// TestFakeServerState_StagePathRecordsContentAndChecksum verifies that after an +// upload+apply, AllFiles for the new version returns exactly the paths uploaded +// with fileChecksum equal to the SHA-256 hex of the recorded bytes. +func TestFakeServerState_StagePathRecordsContentAndChecksum(t *testing.T) { + files := map[string][]byte{ + "app.py": []byte("print('hello')\n"), + "utils/helper.py": []byte("def help(): pass\n"), + } + + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + resp := stageUpload(t, fake, files) + require.Equal(t, "ver-1", resp.CatalogVersionID) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + // AllFiles must return exactly the paths uploaded, with correct checksums. + assert.Len(t, all, len(files)) + + for path, data := range files { + fm, ok := all[path] + require.True(t, ok, "path %s missing from AllFiles", path) + assert.Equal(t, sha256Hex(data), fm.Hash, "checksum for %s", path) + assert.Equal(t, int64(len(data)), fm.Size, "size for %s", path) + } +} + +// TestFakeServerState_StagePathMergeSemantics verifies that ApplyStage merges +// staged files into the prior version (REPLACE merge), not replacing the +// whole catalog. Uploading 1 file over a 3-file version yields a 4-file +// version. +func TestFakeServerState_StagePathMergeSemantics(t *testing.T) { + // Pre-populate a version with 3 files. + priorFiles := map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte("a")), Size: 1}, + "b.py": {Hash: sha256Hex([]byte("b")), Size: 1}, + "c.py": {Hash: sha256Hex([]byte("c")), Size: 1}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + // Upload 1 new file over the 3-file version. + resp := stageUpload(t, fake, map[string][]byte{ + "d.py": []byte("d"), + }) + + // numFiles must count ALL files in the resulting version (3 + 1 = 4), + // not just the ones uploaded (1). + assert.Equal(t, 4, resp.NumFiles, "numFiles must count all files in the resulting version, not just uploaded") + + all, err := fake.AllFiles(fake.catalogID, "ver-2") + require.NoError(t, err) + + // The resulting version must contain all 4 files: 3 from the prior + // version plus 1 newly uploaded. + assert.Len(t, all, 4) + + for path := range priorFiles { + assert.Contains(t, all, path, "prior file %s must survive the merge", path) + } + + assert.Contains(t, all, "d.py", "new file must be in the version") + assert.Equal(t, sha256Hex([]byte("d")), all["d.py"].Hash) +} + +// TestFakeServerState_StagePathReplacesExisting verifies that uploading a file +// that already exists in the prior version replaces its content (REPLACE +// semantics for staged paths). +func TestFakeServerState_StagePathReplacesExisting(t *testing.T) { + priorFiles := map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("old")), Size: 3}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + newContent := []byte("new content here") + + resp := stageUpload(t, fake, map[string][]byte{"app.py": newContent}) + assert.Equal(t, 1, resp.NumFiles, "version still has 1 file after replace") + + all, err := fake.AllFiles(fake.catalogID, "ver-2") + require.NoError(t, err) + + assert.Equal(t, sha256Hex(newContent), all["app.py"].Hash, "checksum must reflect new content") + assert.Equal(t, int64(len(newContent)), all["app.py"].Size) +} + +// TestFakeServerState_ZipPathRecordsContent verifies that the zip upload path +// extracts the archive and records per-file content with SHA-256 checksums. +func TestFakeServerState_ZipPathRecordsContent(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "main.py": "if __name__ == '__main__': pass\n", + }) + + // Build a real zip from the project files (same as the production ZipUploader). + plan := &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalSize: 11}, + {Path: "main.py", LocalSize: 34}, + }, + } + + zipPath, err := buildZip(dir, plan.Uploads) + require.NoError(t, err) + + zipData, err := os.ReadFile(zipPath) + require.NoError(t, err) + + fake := &fakeFilesClient{ + catalogID: "cid-zip", + versionID: "ver-zip", + } + + resp, err := fake.UploadFromZipNew("wapi-sync.zip", int64(len(zipData)), bytes.NewReader(zipData)) + require.NoError(t, err) + + assert.Equal(t, "cid-zip", resp.CatalogID) + assert.Equal(t, "ver-zip", resp.CatalogVersionID) + + all, err := fake.AllFiles(fake.catalogID, "ver-zip") + require.NoError(t, err) + + // The zip path must record the same content and checksums as the stage path. + assert.Len(t, all, 2) + + for _, fa := range plan.Uploads { + fm, ok := all[fa.Path] + require.True(t, ok, "path %s missing from zip AllFiles", fa.Path) + + // The checksum must match the SHA-256 of the file content on disk. + data, err := os.ReadFile(dir + "/" + fa.Path) + require.NoError(t, err) + + assert.Equal(t, sha256Hex(data), fm.Hash, "zip checksum for %s", fa.Path) + assert.Equal(t, int64(len(data)), fm.Size, "zip size for %s", fa.Path) + } +} + +// TestFakePagination_PageSizeOne verifies that AllFiles with page size 1 and +// two files returns both files (the fake follows the next link). If the fake +// failed to follow next links, only the first file would be returned. +func TestFakePagination_PageSizeOne(t *testing.T) { + files := map[string][]byte{ + "alpha.py": []byte("a"), + "beta.py": []byte("b"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withPageSize(1) + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + // Both files must be returned despite page size 1. + assert.Len(t, all, 2, "AllFiles must follow next links and return all files") + assert.Contains(t, all, "alpha.py") + assert.Contains(t, all, "beta.py") +} + +// TestFakePagination_SinglePage verifies that when all files fit in one page, +// AllFiles returns them all without needing to follow a next link. +func TestFakePagination_SinglePage(t *testing.T) { + files := map[string][]byte{ + "alpha.py": []byte("a"), + "beta.py": []byte("b"), + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withPageSize(10) // larger than the number of files + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Len(t, all, 2) +} + +// TestFakeNumFiles_DefaultsToAllFiles verifies that ApplyStage returns numFiles +// equal to the count of ALL files in the resulting version, not just the ones +// uploaded. +func TestFakeNumFiles_DefaultsToAllFiles(t *testing.T) { + priorFiles := map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte("a")), Size: 1}, + "b.py": {Hash: sha256Hex([]byte("b")), Size: 1}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withVersion("cid-1", "ver-1", priorFiles) + + // Upload 1 file over a 2-file version. + resp := stageUpload(t, fake, map[string][]byte{"c.py": []byte("c")}) + assert.Equal(t, 3, resp.NumFiles, "numFiles = all files in version (2+1=3), not uploaded count (1)") +} + +// TestFakeNumFiles_Override verifies that the numFilesOverride hook makes +// ApplyStage return a configured value instead of the real count. +func TestFakeNumFiles_Override(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withNumFilesOverride(99) + + resp := stageUpload(t, fake, map[string][]byte{"app.py": []byte("x")}) + assert.Equal(t, 99, resp.NumFiles, "numFilesOverride must override the real count") +} + +// TestFakeCallCounters verifies that per-method call counters are incremented +// correctly for stage-create, upload-to-stage, apply-stage, and AllFiles. +func TestFakeCallCounters(t *testing.T) { + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + // No calls yet. + assert.Equal(t, 0, fake.CreateStageCalls()) + assert.Equal(t, 0, fake.UploadToStageCalls()) + assert.Equal(t, 0, fake.ApplyStageCalls()) + assert.Equal(t, 0, fake.AllFilesCalls()) + assert.Equal(t, 0, fake.UploadFromZipCalls()) + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + assert.Equal(t, 1, fake.CreateStageCalls()) + + for i := 0; i < 3; i++ { + path := "file" + string(rune('a'+i)) + ".py" + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, 1, strings.NewReader("x")) + require.NoError(t, err) + } + + assert.Equal(t, 3, fake.UploadToStageCalls()) + + _, err = fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + assert.Equal(t, 1, fake.ApplyStageCalls()) + + _, err = fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + _, err = fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Equal(t, 2, fake.AllFilesCalls()) +} + +// TestFakeFaultInjection_DropPath verifies that the dropPathFromApply hook +// omits a path from the resulting version. Passes with the fault off, fails +// with it on (the path is missing from AllFiles). +func TestFakeFaultInjection_DropPath(t *testing.T) { + files := map[string][]byte{ + "app.py": []byte("app"), + "config.py": []byte("config"), + } + + // Without the fault: both files present. + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stageUpload(t, fake, files) + + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + assert.Len(t, all, 2, "without fault, both files present") + + // With the fault: app.py is dropped from the resulting version. + fake2 := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-2", + }).withDropPath("app.py") + + stageUpload(t, fake2, files) + + all2, err := fake2.AllFiles(fake2.catalogID, "ver-2") + require.NoError(t, err) + + assert.Len(t, all2, 1, "with fault, dropped path is missing") + assert.NotContains(t, all2, "app.py", "app.py must be dropped") + assert.Contains(t, all2, "config.py", "other files must survive") +} + +// TestFakeFaultInjection_WrongChecksum verifies that the wrongChecksum hook +// makes AllFiles return a wrong checksum for a chosen path. Passes with the +// fault off, fails with it on. +func TestFakeFaultInjection_WrongChecksum(t *testing.T) { + data := []byte("content here") + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stageUpload(t, fake, map[string][]byte{"app.py": data}) + + // Without the fault: correct checksum. + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + assert.Equal(t, sha256Hex(data), all["app.py"].Hash) + + // With the fault: wrong checksum. + wrong := strings.Repeat("z", 64) + fake2 := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withWrongChecksum("app.py", wrong) + + stageUpload(t, fake2, map[string][]byte{"app.py": data}) + + all2, err := fake2.AllFiles(fake2.catalogID, "ver-1") + require.NoError(t, err) + + assert.Equal(t, wrong, all2["app.py"].Hash, "wrong checksum must be returned") + assert.NotEqual(t, sha256Hex(data), all2["app.py"].Hash, "wrong checksum must differ from real") +} + +// TestFakeFaultInjection_FailNthUpload verifies that the failNthUpload hook +// fails the Nth UploadToStage call. Passes with the fault off, fails with it on. +func TestFakeFaultInjection_FailNthUpload(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withFailNthUpload(2) + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + // First upload succeeds. + err = fake.UploadToStage(fake.catalogID, stage.StageID, "a.py", 1, strings.NewReader("a")) + require.NoError(t, err, "first upload must succeed") + + // Second upload fails. + err = fake.UploadToStage(fake.catalogID, stage.StageID, "b.py", 1, strings.NewReader("b")) + require.Error(t, err, "second upload must fail with fault injection") + require.Contains(t, err.Error(), "injected failure") + + // Third upload succeeds (fault only fires on the Nth call). + err = fake.UploadToStage(fake.catalogID, stage.StageID, "c.py", 1, strings.NewReader("c")) + require.NoError(t, err, "third upload must succeed after the fault fired") + + assert.Equal(t, 3, fake.UploadToStageCalls()) +} + +// TestFakeFaultInjection_FailApplyStage verifies that the failApplyStage hook +// makes ApplyStage return an error after all files have been staged. +func TestFakeFaultInjection_FailApplyStage(t *testing.T) { + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withFailApplyStage() + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + err = fake.UploadToStage(fake.catalogID, stage.StageID, "app.py", 3, strings.NewReader("app")) + require.NoError(t, err, "upload must succeed; fault is on ApplyStage only") + + _, err = fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.Error(t, err, "ApplyStage must fail with fault injection") + require.Contains(t, err.Error(), "injected failure") +} + +// TestFakeRaceFree_ConcurrentUploads verifies that the fake is race-free when +// the uploader runs with 4-way concurrency over 8+ files. Run with -race to +// detect data races. +func TestFakeRaceFree_ConcurrentUploads(t *testing.T) { + // 8 files, uploaded concurrently by 4 goroutines (matching UploadConcurrency). + numFiles := 8 + + fake := &fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + } + + stage, err := fake.CreateStage(fake.catalogID) + require.NoError(t, err) + + var wg sync.WaitGroup + + for i := 0; i < numFiles; i++ { + path := "file" + string(rune('a'+i)) + ".py" + data := []byte(strings.Repeat("x", i+1)) // varying sizes including 0-byte-like + + wg.Add(1) + + go func() { + defer wg.Done() + + err := fake.UploadToStage(fake.catalogID, stage.StageID, path, int64(len(data)), bytes.NewReader(data)) + assert.NoError(t, err) + }() + } + + wg.Wait() + + resp, err := fake.ApplyStage(fake.catalogID, stage.StageID, filesapi.OverwriteReplace) + require.NoError(t, err) + + assert.Equal(t, numFiles, resp.NumFiles) + assert.Equal(t, numFiles, fake.UploadToStageCalls()) + + // Verify all files are present with correct checksums. + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Len(t, all, numFiles, "all files must be recorded despite concurrent uploads") + + // Verify a few checksums deterministically. + paths := make([]string, 0, numFiles) + for i := 0; i < numFiles; i++ { + paths = append(paths, "file"+string(rune('a'+i))+".py") + } + + sort.Strings(paths) + + for i, p := range paths { + expected := sha256Hex([]byte(strings.Repeat("x", i+1))) + assert.Equal(t, expected, all[p].Hash, "checksum for %s", p) + } +} diff --git a/internal/workload/sync/fake_files_client_test.go b/internal/workload/sync/fake_files_client_test.go new file mode 100644 index 000000000..565476228 --- /dev/null +++ b/internal/workload/sync/fake_files_client_test.go @@ -0,0 +1,638 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "archive/zip" + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + stdsync "sync" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload/fileops" +) + +// fakeFilesClient is a self-consistent in-memory model of the Files API +// server. Unlike a shallow stub, it records per-path content and its SHA-256 +// so tests can assert "what the server ended up holding" after an upload. +// +// Server state is modeled as versions: each ApplyStage (or zip upload) creates +// a new version whose files are the REPLACE-merge of the staged paths into the +// prior version. AllFiles serves exactly what was recorded, keyed by version, +// with configurable pagination via a next-link simulation. ApplyStage returns +// a numFiles that defaults to the count of ALL files in the resulting version +// (matching the real API semantics), not just the ones uploaded. +// +// Per-method call counters let tests assert that specific network calls were +// or were not issued. Fault-injection hooks are all opt-in: a test that does +// not ask for a fault gets a faithful server. +// +// Every shared map is guarded by a mutex so -race is clean when the uploader +// runs concurrently (UploadConcurrency = 4). +type fakeFilesClient struct { + mu stdsync.Mutex + + // Configured IDs returned by the fake server. CreateCatalog returns + // catalogID, CreateStage returns stageID, ApplyStage/zip uploads return + // versionID. + catalogID string + stageID string + versionID string + + // Server state: versionID → (path → FileMeta with SHA-256 hash + size). + versions map[string]map[string]filesapi.FileMeta + + // Per-catalog latest version ID, for REPLACE-merge semantics. + latestVersion map[string]string + + // Current staging area: path → content bytes (stage path only). + stagedFiles map[string][]byte + + // Backward-compatible fields: recorded uploaded content and deleted + // paths, kept so existing tests that inspect them still work. + uploadedFiles map[string][]byte + deletedPaths []string + + // Call counters (guarded by mu). + createCatalogCalls int + createStageCalls int + uploadToStageCalls int + applyStageCalls int + uploadFromZipCalls int + allFilesCalls int + + // Configurable AllFiles page size. 0 = return all at once (no pagination + // simulation). When > 0, AllFiles internally splits files into pages and + // follows next links, mirroring the real client's pagination behavior. + allFilesPageSize int + + // Fault-injection hooks (all opt-in; zero values are no-ops). + dropPathFromApply string // omit this path from the resulting version after apply + wrongChecksumPath string // return wrong checksum for this path in AllFiles + wrongChecksumValue string // the wrong checksum to substitute + failNthUpload int // fail the Nth UploadToStage call (1-indexed; 0 = disabled) + failApplyStage bool // ApplyStage returns error after staging succeeded + numFilesOverride *int // if non-nil, ApplyStage returns this numFiles; otherwise real count +} + +// --- Builder methods for test setup (chainable, mutex-safe) --- + +// withVersion pre-populates a version in the fake's server state. Useful for +// testing REPLACE-merge semantics without going through the upload flow. +func (f *fakeFilesClient) withVersion(catalogID, versionID string, files map[string]filesapi.FileMeta) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[versionID] = files + f.latestVersion[catalogID] = versionID + + return f +} + +// withPageSize sets the AllFiles page size for pagination simulation. +func (f *fakeFilesClient) withPageSize(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.allFilesPageSize = n + + return f +} + +// withDropPath configures the fake to omit the given path from the resulting +// version after ApplyStage, simulating a server-side dropped upload. +func (f *fakeFilesClient) withDropPath(path string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.dropPathFromApply = path + + return f +} + +// withWrongChecksum configures AllFiles to return the given checksum for the +// given path instead of the real SHA-256, simulating a server-side checksum +// mismatch. +func (f *fakeFilesClient) withWrongChecksum(path, checksum string) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.wrongChecksumPath = path + f.wrongChecksumValue = checksum + + return f +} + +// withFailNthUpload configures the fake to fail the Nth UploadToStage call +// (1-indexed). 0 disables the hook. +func (f *fakeFilesClient) withFailNthUpload(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.failNthUpload = n + + return f +} + +// withFailApplyStage configures ApplyStage to return an error after all files +// have been successfully staged, simulating a server-side apply failure. +func (f *fakeFilesClient) withFailApplyStage() *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.failApplyStage = true + + return f +} + +// withNumFilesOverride configures ApplyStage to return the given numFiles +// instead of the real count of all files in the resulting version. +func (f *fakeFilesClient) withNumFilesOverride(n int) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + f.numFilesOverride = &n + + return f +} + +// --- Counter accessors (mutex-safe for reading after concurrent work) --- + +func (f *fakeFilesClient) CreateCatalogCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.createCatalogCalls +} + +func (f *fakeFilesClient) CreateStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.createStageCalls +} + +func (f *fakeFilesClient) UploadToStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.uploadToStageCalls +} + +func (f *fakeFilesClient) ApplyStageCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.applyStageCalls +} + +func (f *fakeFilesClient) UploadFromZipCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.uploadFromZipCalls +} + +func (f *fakeFilesClient) AllFilesCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.allFilesCalls +} + +// --- filesapi.Client implementation --- + +func (f *fakeFilesClient) CreateCatalog() (*filesapi.CatalogResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.createCatalogCalls++ + + if f.catalogID == "" { + return nil, errors.New("fakeFilesClient.CreateCatalog: no catalogID configured") + } + + return &filesapi.CatalogResp{CatalogID: f.catalogID}, nil +} + +func (f *fakeFilesClient) CreateStage(_ string) (*filesapi.StageResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.createStageCalls++ + + if f.stageID == "" { + return nil, errors.New("fakeFilesClient.CreateStage: no stageID configured") + } + + // Initialize a fresh staging area for this stage. + f.stagedFiles = make(map[string][]byte) + + return &filesapi.StageResp{CatalogID: f.catalogID, StageID: f.stageID}, nil +} + +func (f *fakeFilesClient) UploadToStage(_, _, name string, _ int64, body io.Reader) error { + // Read the body outside the mutex: each call has its own reader and + // holding the mutex during I/O would serialize uploads unnecessarily. + data, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("read upload body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadToStageCalls++ + + // Fault injection: fail the Nth upload. + if f.failNthUpload > 0 && f.uploadToStageCalls == f.failNthUpload { + return fmt.Errorf("fakeFilesClient.UploadToStage: injected failure on upload %d", f.uploadToStageCalls) + } + + if f.stagedFiles == nil { + f.stagedFiles = make(map[string][]byte) + } + + f.stagedFiles[name] = data + + // Backward-compatible: record uploaded content for tests that inspect it. + if f.uploadedFiles == nil { + f.uploadedFiles = make(map[string][]byte) + } + + f.uploadedFiles[name] = data + + return nil +} + +// mergeStagedFiles merges staged files into a copy of the latest version's +// files, matching the real API's stage REPLACE semantics: staged paths +// replace existing entries, unmentioned paths from the prior version remain. +func (f *fakeFilesClient) mergeStagedFiles(catalogID string) map[string]filesapi.FileMeta { + newVersion := make(map[string]filesapi.FileMeta) + + if latest, ok := f.latestVersion[catalogID]; ok { + if files, ok := f.versions[latest]; ok { + for k, v := range files { + newVersion[k] = v + } + } + } + + for path, data := range f.stagedFiles { + h := sha256.Sum256(data) + newVersion[path] = filesapi.FileMeta{ + Hash: hex.EncodeToString(h[:]), + Size: int64(len(data)), + } + } + + return newVersion +} + +func (f *fakeFilesClient) ApplyStage(catalogID, _, _ string) (*filesapi.ApplyStageResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.applyStageCalls++ + + // Fault injection: fail ApplyStage after staging succeeded. + if f.failApplyStage { + return nil, errors.New("fakeFilesClient.ApplyStage: injected failure") + } + + if f.versionID == "" { + return nil, errors.New("fakeFilesClient.ApplyStage: no versionID configured") + } + + newVersion := f.mergeStagedFiles(catalogID) + + // Fault injection: drop a path from the resulting version. + if f.dropPathFromApply != "" { + delete(newVersion, f.dropPathFromApply) + } + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = newVersion + f.latestVersion[catalogID] = f.versionID + + // Clear the staging area for the next stage. + f.stagedFiles = make(map[string][]byte) + + // numFiles defaults to the count of ALL files in the resulting version, + // not just the ones uploaded. This matches the real API semantics. + numFiles := len(newVersion) + if f.numFilesOverride != nil { + numFiles = *f.numFilesOverride + } + + return &filesapi.ApplyStageResp{ + CatalogID: f.catalogID, + CatalogVersionID: f.versionID, + NumFiles: numFiles, + }, nil +} + +func (f *fakeFilesClient) UploadFromZipNew(_ string, _ int64, body io.Reader) (*filesapi.FromFileResp, error) { + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read zip body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadFromZipCalls++ + + if f.catalogID == "" { + return nil, errors.New("fakeFilesClient.UploadFromZipNew: no catalogID configured") + } + + zipFiles, err := extractZipFiles(data) + if err != nil { + return nil, err + } + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = zipFiles + f.latestVersion[f.catalogID] = f.versionID + + // Inline completion (no StatusID), matching the real API for small archives. + return &filesapi.FromFileResp{ + CatalogID: f.catalogID, + CatalogVersionID: f.versionID, + }, nil +} + +func (f *fakeFilesClient) UploadFromZipExisting(catalogID, _, _ string, _ int64, body io.Reader) (*filesapi.FromFileResp, error) { + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read zip body: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploadFromZipCalls++ + + zipFiles, err := extractZipFiles(data) + if err != nil { + return nil, err + } + + // REPLACE merge: start with the latest version's files, then merge. + newVersion := make(map[string]filesapi.FileMeta) + + if latest, ok := f.latestVersion[catalogID]; ok { + if files, ok := f.versions[latest]; ok { + for k, v := range files { + newVersion[k] = v + } + } + } + + for k, v := range zipFiles { + newVersion[k] = v + } + + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[f.versionID] = newVersion + f.latestVersion[catalogID] = f.versionID + + return &filesapi.FromFileResp{ + CatalogID: catalogID, + CatalogVersionID: f.versionID, + }, nil +} + +func (f *fakeFilesClient) PollStatus(_ string) (*filesapi.StatusResp, error) { + return &filesapi.StatusResp{Status: filesapi.StatusCompleted}, nil +} + +// applyWrongChecksum substitutes the configured wrong checksum for the +// chosen path in the result map, simulating a server-side checksum mismatch. +func (f *fakeFilesClient) applyWrongChecksum(result map[string]filesapi.FileMeta) { + if f.wrongChecksumPath == "" { + return + } + + if fm, exists := result[f.wrongChecksumPath]; exists { + fm.Hash = f.wrongChecksumValue + result[f.wrongChecksumPath] = fm + } +} + +// paginateFiles simulates the server's paginated response and the client's +// next-link following. With a page size of 1 and N files, the fake processes +// N pages; if it failed to follow next links, only the first file would be +// returned. A no-op when pageSize is 0 or when all files fit in one page. +func paginateFiles(files map[string]filesapi.FileMeta, pageSize int) map[string]filesapi.FileMeta { + if pageSize <= 0 || len(files) <= pageSize { + return files + } + + paths := make([]string, 0, len(files)) + for k := range files { + paths = append(paths, k) + } + + sort.Strings(paths) + + collected := make(map[string]filesapi.FileMeta, len(paths)) + + // Process page by page, following next links. + idx := 0 + for idx < len(paths) { + end := idx + pageSize + if end > len(paths) { + end = len(paths) + } + + for _, p := range paths[idx:end] { + collected[p] = files[p] + } + + // Advance to the next page (follow the next link). + idx = end + } + + return collected +} + +func (f *fakeFilesClient) AllFiles(_, versionID string) (map[string]filesapi.FileMeta, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.allFilesCalls++ + + // Get the files for the requested version from server state. + files, ok := f.versions[versionID] + if !ok { + return make(map[string]filesapi.FileMeta), nil + } + + // Copy so the caller cannot mutate our state. + result := make(map[string]filesapi.FileMeta, len(files)) + for k, v := range files { + result[k] = v + } + + // Fault injection: return a wrong checksum for a chosen path. + f.applyWrongChecksum(result) + + // Simulate pagination: split into pages and follow next links, mirroring + // the real client. A bug here (not following next links) would cause + // AllFiles to return only the first page, which the pagination test catches. + result = paginateFiles(result, f.allFilesPageSize) + + return result, nil +} + +func (f *fakeFilesClient) DownloadFile(_, _, _ string, _ io.Writer) (string, int64, error) { + return "", 0, errors.New("fakeFilesClient: DownloadFile not expected") +} + +func (f *fakeFilesClient) DeleteFiles(catalogID string, paths []string) (*filesapi.DeleteFilesResp, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.deletedPaths = append(f.deletedPaths, paths...) + + if len(paths) == 0 { + return &filesapi.DeleteFilesResp{}, nil + } + + // Create a new version with the deleted paths removed from the latest version. + latest, ok := f.latestVersion[catalogID] + if !ok { + return &filesapi.DeleteFilesResp{}, nil + } + + currentFiles, ok := f.versions[latest] + if !ok { + return &filesapi.DeleteFilesResp{}, nil + } + + newVersion := make(map[string]filesapi.FileMeta, len(currentFiles)) + for k, v := range currentFiles { + newVersion[k] = v + } + + for _, p := range paths { + delete(newVersion, p) + } + + newVerID := fmt.Sprintf("%s-del-%d", f.versionID, len(f.versions)) + if f.versions == nil { + f.versions = make(map[string]map[string]filesapi.FileMeta) + } + + if f.latestVersion == nil { + f.latestVersion = make(map[string]string) + } + + f.versions[newVerID] = newVersion + f.latestVersion[catalogID] = newVerID + + return &filesapi.DeleteFilesResp{ + CatalogID: catalogID, + CatalogVersionID: newVerID, + NumFiles: len(newVersion), + }, nil +} + +func (f *fakeFilesClient) ListVersions(_ string, _ int) ([]filesapi.CatalogVersion, error) { + return nil, errors.New("fakeFilesClient: ListVersions not expected") +} + +// extractZipFiles reads a zip archive from raw bytes and returns a map of +// path → FileMeta with the SHA-256 hash and size of each entry's content. +// This models what the server does when it extracts an uploaded archive. +func extractZipFiles(data []byte) (map[string]filesapi.FileMeta, error) { + zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open zip: %w", err) + } + + files := make(map[string]filesapi.FileMeta) + + for _, zf := range zipReader.File { + content, err := readZipEntry(zf) + if err != nil { + return nil, err + } + + h := sha256.Sum256(content) + path := fileops.NormalizePath(zf.Name) + files[path] = filesapi.FileMeta{ + Hash: hex.EncodeToString(h[:]), + Size: int64(len(content)), + } + } + + return files, nil +} + +// readZipEntry reads and closes a single zip file entry, returning its +// uncompressed content. +func readZipEntry(zf *zip.File) ([]byte, error) { + rc, err := zf.Open() + if err != nil { + return nil, fmt.Errorf("open zip entry %s: %w", zf.Name, err) + } + + defer func() { _ = rc.Close() }() + + content, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("read zip entry %s: %w", zf.Name, err) + } + + return content, nil +} diff --git a/internal/workload/sync/project_helper_test.go b/internal/workload/sync/project_helper_test.go new file mode 100644 index 000000000..dfbefc0fa --- /dev/null +++ b/internal/workload/sync/project_helper_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/require" +) + +// syncedProject creates a temp project tree with initialized workload state +// (config.json + manifest.json) where the manifest reflects the current file +// hashes, simulating a project that has already been synced. The config +// points at the given catalogID and versionID, and the manifest's file +// entries match the SHA-256 of every file on disk (user files + .drignore). +// +// This is the reusable helper for tests that need a project in a "synced" +// state: local == base, so a Plan with no disk changes produces an empty plan, +// and a Plan after modifying a file produces exactly that file as an upload. +func syncedProject(t *testing.T, files map[string]string, catalogID, versionID string) string { + t.Helper() + + dir := initProject(t, files) + + // Update config with catalog and version to simulate a prior sync. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + cfg.CatalogID = &catalogID + cfg.LastSyncedVersionID = &versionID + require.NoError(t, wapi.SaveConfig(dir, cfg)) + + // Build manifest with hashes of ALL files on disk: user files plus the + // .drignore template that Initialize writes. Every manifest entry must + // match the real file hash so that local == base and the plan is empty. + manifestFiles := make(map[string]wapi.FileMeta) + + for rel := range files { + hash, size, err := hashLocal(t, dir, rel) + require.NoError(t, err) + + manifestFiles[rel] = wapi.FileMeta{Hash: hash, Size: size} + } + + // Include .drignore (created by Initialize) so it too is in sync. + hash, size, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + manifestFiles[ignore.FileName] = wapi.FileMeta{Hash: hash, Size: size} + + syncedAt := time.Now().UTC() + manifest := wapi.Manifest{ + Version: wapi.ManifestVersion, + SyncedAt: &syncedAt, + SyncedVersionID: &versionID, + Files: manifestFiles, + } + + require.NoError(t, wapi.SaveManifest(dir, manifest)) + + return dir +} + +// modifyFile rewrites a file in dir with new content, for tests that need +// pending changes after a syncedProject setup. +func modifyFile(t *testing.T, dir, rel, content string) { + t.Helper() + + abs := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.WriteFile(abs, []byte(content), 0o644)) +} diff --git a/internal/workload/sync/upload_scaffolding_test.go b/internal/workload/sync/upload_scaffolding_test.go new file mode 100644 index 000000000..cf8a1a369 --- /dev/null +++ b/internal/workload/sync/upload_scaffolding_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// assertZeroUploadCalls asserts that no upload-side network calls were issued +// on the fake: stage create, upload-to-stage, apply-stage, and zip upload all +// zero. This is the core assertion for VAL-UPLOAD-015 and VAL-UPLOAD-017. +func assertZeroUploadCalls(t *testing.T, fake *fakeFilesClient) { + t.Helper() + + assert.Equal(t, 0, fake.CreateStageCalls(), "CreateStage must not be called") + assert.Equal(t, 0, fake.UploadToStageCalls(), "UploadToStage must not be called") + assert.Equal(t, 0, fake.ApplyStageCalls(), "ApplyStage must not be called") + assert.Equal(t, 0, fake.UploadFromZipCalls(), "UploadFromZip must not be called") +} + +// TestEmptyPlan_ZeroUploadCalls verifies VAL-UPLOAD-015: when the plan has +// zero uploads (all files unchanged), no upload API calls are issued. The +// per-method counters on the fake prove this at the Go level. +func TestEmptyPlan_ZeroUploadCalls(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "main.py": "def main(): pass\n", + }, catalogID, versionID) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // An empty plan (all files unchanged) issues zero upload calls. + assertZeroUploadCalls(t, fake) +} + +// TestDryRun_ZeroUploadCalls verifies VAL-UPLOAD-017: a --dry-run run with +// pending changes issues zero upload-side calls on the fake. The plan is +// non-empty (a file was modified), but Run returns after Plan without calling +// Execute, so no upload methods are invoked. +func TestDryRun_ZeroUploadCalls(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change so the plan is non-empty. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + // Run stops after Plan because DryRun is true. The plan is non-empty + // (a file was modified), but Execute is never called, so no upload + // methods are invoked. The positive control (TestNonEmptyPlan_UploadCallsAreNonZero) + // proves the same setup with DryRun=false DOES issue upload calls, so + // the zero-count here is not vacuously true. + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // No upload-side calls despite a non-empty plan. + assertZeroUploadCalls(t, fake) +} + +// TestNonEmptyPlan_UploadCallsAreNonZero is the positive control for +// VAL-UPLOAD-015 and VAL-UPLOAD-017: a non-empty plan in a non-dry-run sync +// DOES issue upload-side calls. This proves the counters work and the +// zero-call assertions above are not vacuously true. +func TestNonEmptyPlan_UploadCallsAreNonZero(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // A non-empty plan in a non-dry-run sync must issue upload calls. + assert.Equal(t, 1, fake.CreateStageCalls(), "CreateStage must be called for a non-empty plan") + assert.Positive(t, fake.UploadToStageCalls(), "UploadToStage must be called for a non-empty plan") + assert.Equal(t, 1, fake.ApplyStageCalls(), "ApplyStage must be called for a non-empty plan") + assert.Equal(t, 0, fake.UploadFromZipCalls(), "zip path must not be used for a small upload") +} From cd30b65a449e9167e0d6097de3becb7161a64968 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 00:30:03 -0700 Subject: [PATCH 02/20] [RAPTOR-19525] fix(workload): hash uploads while streaming so the manifest records what was sent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest recorded the Phase-2 planned hash for uploaded files, not the hash of the bytes actually sent. A file rewritten between plan and upload left BASE describing content the server never received, and the next sync then reported "Up to date." forever — the TOCTOU poisoning reproduced 3/3 through the real binary. ApplyUploads now returns UploadOutcome{CatalogID, VersionID, Sent} where Sent carries the per-path SHA-256 and size of the bytes that crossed the wire. The stage path wraps the open file in io.TeeReader(f, sha256.New()) and derives content-length from f.Stat() on the already-open handle (never a fresh os.Stat). The zip path wraps the archive entry writer in io.MultiWriter(w, sha256.New()) and takes the streamed size from io.Copy's return. The shared hash-while-streaming piece is factored into hashstream.go so jscpd stays quiet. Phase 6 seeds each uploaded path's manifest entry from outcome.Sent[path] and hard-fails with an error naming the path if an entry is missing — a per-path fallback to the Phase-2 hash is the original bug and must not exist anywhere. The Engine struct gains an uploadOutcome field as the channel between Phase 5 and Phase 6 (runPhases has no per-phase return threading). Manifest schema stays at version 1; no new fields are added to manifest.json or config.json. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/engine.go | 1 + .../sync/fake_files_client_self_test.go | 2 +- internal/workload/sync/hashstream.go | 44 +++ internal/workload/sync/phase5_execute.go | 13 +- internal/workload/sync/phase6_state.go | 33 +- internal/workload/sync/phase6_state_test.go | 305 ++++++++++++++++++ internal/workload/sync/upload_stage.go | 90 ++++-- internal/workload/sync/upload_stage_test.go | 166 ++++++++++ internal/workload/sync/upload_zip.go | 65 ++-- internal/workload/sync/uploader.go | 20 +- 10 files changed, 681 insertions(+), 58 deletions(-) create mode 100644 internal/workload/sync/hashstream.go create mode 100644 internal/workload/sync/phase6_state_test.go create mode 100644 internal/workload/sync/upload_stage_test.go diff --git a/internal/workload/sync/engine.go b/internal/workload/sync/engine.go index 883dc77c8..30c71e256 100644 --- a/internal/workload/sync/engine.go +++ b/internal/workload/sync/engine.go @@ -105,6 +105,7 @@ type Engine struct { rollback *Rollback newCatalogID string newVersionID string + uploadOutcome *UploadOutcome conflictCopies []string result *Result startedAt time.Time diff --git a/internal/workload/sync/fake_files_client_self_test.go b/internal/workload/sync/fake_files_client_self_test.go index 121433d7d..022e14e3e 100644 --- a/internal/workload/sync/fake_files_client_self_test.go +++ b/internal/workload/sync/fake_files_client_self_test.go @@ -173,7 +173,7 @@ func TestFakeServerState_ZipPathRecordsContent(t *testing.T) { }, } - zipPath, err := buildZip(dir, plan.Uploads) + zipPath, _, err := buildZip(dir, plan.Uploads) require.NoError(t, err) zipData, err := os.ReadFile(zipPath) diff --git a/internal/workload/sync/hashstream.go b/internal/workload/sync/hashstream.go new file mode 100644 index 000000000..5c5acda26 --- /dev/null +++ b/internal/workload/sync/hashstream.go @@ -0,0 +1,44 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "crypto/sha256" + "encoding/hex" + "hash" +) + +// newStreamHasher returns a SHA-256 hasher for computing the hash of bytes +// as they are streamed to their destination, without a separate read pass. +// Both uploaders use this so the hash recorded in the manifest is of the +// bytes that actually crossed the wire, not the bytes hashed during planning. +// A file rewritten between plan and upload must leave BASE describing what +// the server received, and only a hash of the streamed bytes can guarantee +// that. +func newStreamHasher() hash.Hash { + return sha256.New() +} + +// streamedEntry finalizes a hasher that observed the streamed bytes into a +// FileEntry carrying the hex-encoded SHA-256 and the byte count. The size +// comes from the caller — f.Stat on the already-open handle for the stage +// path, io.Copy's return for the zip path — so it always describes the same +// bytes the hasher observed, never the Phase-2 planned size. +func streamedEntry(h hash.Hash, size int64) FileEntry { + return FileEntry{ + Hash: hex.EncodeToString(h.Sum(nil)), + Size: size, + } +} diff --git a/internal/workload/sync/phase5_execute.go b/internal/workload/sync/phase5_execute.go index 80c616dc6..b535c4076 100644 --- a/internal/workload/sync/phase5_execute.go +++ b/internal/workload/sync/phase5_execute.go @@ -276,13 +276,20 @@ func applyRemoteDeletesAndUploads(e *Engine, codeRef codeRefRef) (string, string if len(e.plan.Uploads) > 0 { uploader := ChooseUploader(e.plan) - cid, vid, err := uploader.ApplyUploads(e, e.plan.Uploads) + outcome, err := uploader.ApplyUploads(e, e.plan.Uploads) if err != nil { return "", "", err } - newCatalogID = cid - newVersionID = vid + // Store the outcome on the engine so Phase 6 can read + // outcome.Sent[path]. The phase pipeline runs each phase + // independently via runPhases with no per-phase return + // threading, so Engine state is the only channel between + // Phase 5 and Phase 6. + e.uploadOutcome = &outcome + + newCatalogID = outcome.CatalogID + newVersionID = outcome.VersionID } if newVersionID != "" && newVersionID != codeRef.CatalogVersionID { diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index 26e1cf810..897337760 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -52,7 +52,11 @@ func phase6State(e *Engine) error { return fmt.Errorf("save config: %w", err) } - manifest := buildNewBaseManifest(e, versionForState, now) + manifest, err := buildNewBaseManifest(e, versionForState, now) + if err != nil { + return fmt.Errorf("build manifest: %w", err) + } + if err := wapi.SaveManifest(e.projectDir, manifest); err != nil { return fmt.Errorf("save manifest: %w", err) } @@ -72,9 +76,13 @@ func phase6State(e *Engine) error { return nil } -// buildNewBaseManifest computes NEW_BASE = REMOTE + uploads (local hashes) -// - deletes, with conflicts resolved as remote-wins. -func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) wapi.Manifest { +// buildNewBaseManifest computes NEW_BASE = REMOTE + uploads (streamed hashes) +// - deletes, with conflicts resolved as remote-wins. Each uploaded path's +// hash and size come from the UploadOutcome recorded in Phase 5 — the bytes +// that actually crossed the wire, not the Phase-2 planned hash. A missing +// Sent entry is a hard error naming the path: a per-path fallback to the +// planned hash IS the original poisoning bug and must not exist here. +func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) (wapi.Manifest, error) { files := make(map[string]wapi.FileMeta, len(e.remote)) for path, fe := range e.remote { @@ -82,7 +90,20 @@ func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) } for _, fa := range e.plan.Uploads { - files[fa.Path] = wapi.FileMeta{Hash: fa.LocalHash, Size: fa.LocalSize} + if e.uploadOutcome == nil { + return wapi.Manifest{}, fmt.Errorf("internal: no upload outcome recorded for %s", fa.Path) + } + + sent, ok := e.uploadOutcome.Sent[fa.Path] + if !ok { + // Refuse rather than fall back: phase6 overwrites unconditionally, + // so a per-path fallback to fa.LocalHash silently reintroduces + // the poisoning. Either every upload has a Sent entry, or the + // sync fails. + return wapi.Manifest{}, fmt.Errorf("internal: no streamed hash recorded for %s", fa.Path) + } + + files[fa.Path] = wapi.FileMeta{Hash: sent.Hash, Size: sent.Size} } for _, fa := range e.plan.Deletes { @@ -105,7 +126,7 @@ func buildNewBaseManifest(e *Engine, syncedVersionID string, syncedAt time.Time) SyncedAt: &syncedAtCopy, SyncedVersionID: &versionCopy, Files: files, - } + }, nil } // syncHistoryEntry assembles the JSONL line written to history.log. diff --git a/internal/workload/sync/phase6_state_test.go b/internal/workload/sync/phase6_state_test.go new file mode 100644 index 000000000..c08bdd156 --- /dev/null +++ b/internal/workload/sync/phase6_state_test.go @@ -0,0 +1,305 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// emptyHash is the SHA-256 of the empty string, matching what a torn-to-empty +// file produces when streamed. +const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +// TestExecuteRecordsStreamedHash is the deterministic form of the TOCTOU that +// poisoned the manifest 3/3 times through the real binary. Plan() computes a +// hash, then the file is rewritten, then Execute(plan) streams the new bytes. +// The manifest must record the streamed hash, not the Phase-2 planned hash. +func TestExecuteRecordsStreamedHash(t *testing.T) { + tests := []struct { + name string + planContent string // file content at Plan time + execContent string // file content at Execute time + }{ + { + name: "same_size_rewrite", + planContent: "print('ho')\n", // 11 bytes + execContent: "print('ok')\n", // 11 bytes, different content + }, + { + name: "grows", + planContent: "print('hi')\n", // 11 bytes + execContent: "print('hello')\n", // 14 bytes + }, + { + name: "shrinks", + planContent: "print('hello')\n", // 14 bytes + execContent: "print('hi')\n", // 11 bytes + }, + { + name: "torn_to_empty", + planContent: "print('hi')\n", // 11 bytes + execContent: "", // 0 bytes + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + // Start from a synced project so Plan sees exactly one upload. + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees app.py as an upload. + modifyFile(t, dir, "app.py", tc.planContent) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 1, "exactly one upload expected") + require.Equal(t, "app.py", plan.Uploads[0].Path) + + // The Phase-2 planned hash is of the content present at Plan time. + plannedHash := plan.Uploads[0].LocalHash + plannedSize := plan.Uploads[0].LocalSize + + // The streamed hash is of the content present at Execute time. + streamedContent := []byte(tc.execContent) + streamedHash := sha256Hex(streamedContent) + streamedSize := int64(len(streamedContent)) + + // The test is only meaningful if the planned and streamed hashes differ. + require.NotEqual(t, plannedHash, streamedHash, + "planned and streamed hashes must differ for the test to be meaningful") + + // Between Plan and Execute, rewrite the file. This is the TOCTOU window. + modifyFile(t, dir, "app.py", tc.execContent) + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + // Read the manifest and verify it records the streamed bytes. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + fm, ok := manifest.Files["app.py"] + require.True(t, ok, "app.py must be in the manifest") + + assert.Equal(t, streamedHash, fm.Hash, + "manifest must record the streamed hash, not the Phase-2 planned hash") + assert.NotEqual(t, plannedHash, fm.Hash, + "manifest must NOT record the Phase-2 planned hash") + assert.Equal(t, streamedSize, fm.Size, + "manifest must record the streamed size, not the Phase-2 planned size") + + // Size inequality only holds when the content actually changed size; + // the same_size case has identical planned and streamed sizes by + // construction, and that is correct. + if plannedSize != streamedSize { + assert.NotEqual(t, plannedSize, fm.Size, + "manifest must NOT record the Phase-2 planned size when sizes differ") + } + + // The torn-to-empty case has a specific expected hash. + if tc.execContent == "" { + assert.Equal(t, emptyHash, fm.Hash, + "torn-to-empty must record the SHA-256 of the empty string") + assert.Equal(t, int64(0), fm.Size, + "torn-to-empty must record size 0") + } + + // Every hash must be 64 lowercase hex characters and size >= 0. + assert.Len(t, fm.Hash, 64, "hash must be 64 hex characters") + assert.GreaterOrEqual(t, fm.Size, int64(0), "size must be >= 0") + }) + } +} + +// TestPhase6MissingSentHardFails verifies the hard rule: if Sent[path] is +// missing for an uploaded path, buildNewBaseManifest must return an error +// naming the path rather than falling back to the Phase-2 planned hash. A +// per-path fallback IS the original poisoning bug. +func TestPhase6MissingSentHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, // missing app.py + }, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") +} + +// TestPhase6NilOutcomeHardFails verifies that a nil uploadOutcome with a +// non-empty upload list also hard-fails naming the path. +func TestPhase6NilOutcomeHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: nil, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "app.py") +} + +// TestPhase6EmptyPlanStillWritesManifest verifies that a plan with zero +// uploads still produces a correct manifest. The upload loop does not execute, +// so the hard-fail does not misfire on an empty Sent map. A nil uploadOutcome +// is safe when there are no uploads. +func TestPhase6EmptyPlanStillWritesManifest(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{}, + Downloads: []FileAction{}, + Deletes: []FileAction{}, + Conflicts: []FileAction{}, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + uploadOutcome: nil, // no uploads, so nil is fine + } + + manifest, err := buildNewBaseManifest(e, "ver-new", time.Now()) + require.NoError(t, err) + assert.Equal(t, wapi.ManifestVersion, manifest.Version) + assert.Len(t, manifest.Files, 1) + assert.Equal(t, sha256Hex([]byte("print('hi')\n")), manifest.Files["app.py"].Hash) +} + +// TestManifestSchemaUnchanged verifies that the written manifest carries +// "version": 1 and that no new fields are added to manifest.json or config.json. +func TestManifestSchemaUnchanged(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + result, err := e.Run() + require.NoError(t, err) + require.NotNil(t, result) + + // Verify manifest.json schema. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, 1, manifest.Version, "manifest version must stay 1") + + // Verify the manifest has exactly the expected top-level keys by reading + // the raw JSON and checking no unexpected fields were added. + rawManifest, err := os.ReadFile(dir + "/.datarobot/workload/manifest.json") + require.NoError(t, err) + + // The manifest must contain "version": 1 and the standard fields only. + assert.Contains(t, string(rawManifest), `"version": 1`) + assert.Contains(t, string(rawManifest), `"files"`) + assert.Contains(t, string(rawManifest), `"syncedVersionId"`) + + // Verify config.json has no new fields. + rawConfig, err := os.ReadFile(dir + "/.datarobot/workload/config.json") + require.NoError(t, err) + + assert.Contains(t, string(rawConfig), `"artifactId"`) + assert.Contains(t, string(rawConfig), `"catalogId"`) + assert.Contains(t, string(rawConfig), `"lastSyncedVersionId"`) + + // syncedVersionId in manifest must equal the version in the result. + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, result.NewVersion, *manifest.SyncedVersionID, + "syncedVersionId must equal the version in the sync summary") + + // config.json's LastSyncedVersionID must equal manifest's syncedVersionId. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, *manifest.SyncedVersionID, *cfg.LastSyncedVersionID, + "config LastSyncedVersionID must equal manifest syncedVersionId") +} diff --git a/internal/workload/sync/upload_stage.go b/internal/workload/sync/upload_stage.go index 8949f092b..6efa14b67 100644 --- a/internal/workload/sync/upload_stage.go +++ b/internal/workload/sync/upload_stage.go @@ -16,6 +16,7 @@ package sync import ( "fmt" + "io" "os" "path/filepath" "sync" @@ -27,28 +28,34 @@ import ( // missing, create stage, upload each file, apply stage. type StageUploader struct{} -// ApplyUploads pushes files via the stage workflow. -func (StageUploader) ApplyUploads(e *Engine, files []FileAction) (string, string, error) { +// ApplyUploads pushes files via the stage workflow and returns the per-path +// streamed hashes so Phase 6 can record what the server actually received. +func (StageUploader) ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) { catalogID, err := ensureCatalog(e) if err != nil { - return "", "", err + return UploadOutcome{}, err } stage, err := e.files.CreateStage(catalogID) if err != nil { - return "", "", fmt.Errorf("create stage: %w", err) + return UploadOutcome{}, fmt.Errorf("create stage: %w", err) } - if err := uploadFilesParallel(e, catalogID, stage.StageID, files); err != nil { - return "", "", err + sent, err := uploadFilesParallel(e, catalogID, stage.StageID, files) + if err != nil { + return UploadOutcome{}, err } apply, err := e.files.ApplyStage(catalogID, stage.StageID, filesapi.OverwriteReplace) if err != nil { - return "", "", fmt.Errorf("apply stage: %w", err) + return UploadOutcome{}, fmt.Errorf("apply stage: %w", err) } - return catalogID, apply.CatalogVersionID, nil + return UploadOutcome{ + CatalogID: catalogID, + VersionID: apply.CatalogVersionID, + Sent: sent, + }, nil } // ensureCatalog returns the catalog ID, creating a new one when neither @@ -66,12 +73,23 @@ func ensureCatalog(e *Engine) (string, error) { return cat.CatalogID, nil } -// uploadFilesParallel uploads files up to UploadConcurrency. The first -// error closes done to stop other workers from starting; in-flight -// workers still finish their current upload before the function returns. -func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileAction) error { +// uploadResult carries one file's streamed hash and size from a worker +// goroutine to the orchestrator via a buffered result channel, matching the +// existing errCh convention rather than adding a mutex-guarded map. +type uploadResult struct { + path string + entry FileEntry +} + +// uploadFilesParallel uploads files up to UploadConcurrency and collects +// per-path streamed hashes. The first error closes done to stop other workers +// from starting; in-flight workers still finish their current upload before +// the function returns. Both errCh and resCh are buffered to len(files) so a +// send never blocks or drops. resCh is closed by the orchestrator only, after +// wg.Wait, following the same convention as errCh. +func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileAction) (map[string]FileEntry, error) { if len(files) == 0 { - return nil + return nil, nil } done := make(chan struct{}) @@ -83,6 +101,7 @@ func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileActio sem := make(chan struct{}, UploadConcurrency) errCh := make(chan error, len(files)) + resCh := make(chan uploadResult, len(files)) var wg sync.WaitGroup @@ -103,39 +122,68 @@ func uploadFilesParallel(e *Engine, catalogID, stageID string, files []FileActio defer func() { <-sem }() - if err := uploadOneToStage(e, catalogID, stageID, fa); err != nil { + entry, err := uploadOneToStage(e, catalogID, stageID, fa) + if err != nil { select { case errCh <- err: cancel() default: } + + return } + + resCh <- uploadResult{path: fa.Path, entry: entry} }() } wg.Wait() close(errCh) + close(resCh) if err := <-errCh; err != nil { - return err + return nil, err + } + + sent := make(map[string]FileEntry, len(files)) + + for r := range resCh { + sent[r.path] = r.entry } - return nil + return sent, nil } -func uploadOneToStage(e *Engine, catalogID, stageID string, fa FileAction) error { +func uploadOneToStage(e *Engine, catalogID, stageID string, fa FileAction) (FileEntry, error) { abs := filepath.Join(e.projectDir, filepath.FromSlash(fa.Path)) f, err := os.Open(abs) if err != nil { - return fmt.Errorf("open %s: %w", fa.Path, err) + return FileEntry{}, fmt.Errorf("open %s: %w", fa.Path, err) } defer func() { _ = f.Close() }() - if err := e.files.UploadToStage(catalogID, stageID, fa.Path, fa.LocalSize, f); err != nil { - return fmt.Errorf("upload %s: %w", fa.Path, err) + // Content-length from the already-open handle, not a fresh os.Stat: a + // fresh stat reopens the TOCTOU window and can follow a symlink swapped + // in since the plan phase. Do not abort merely because this differs from + // fa.LocalSize — a mid-run edit is legitimate; upload the current bytes + // and let the recorded hash be the truth. + stat, err := f.Stat() + if err != nil { + return FileEntry{}, fmt.Errorf("stat %s: %w", fa.Path, err) + } + + size := stat.Size() + + // Hash the exact bytes streamed, not the Phase-2 planned hash. TeeReader + // is the right primitive: UploadToStage pipes the body through io.Copy, + // so every byte that reaches the wire passes through the hasher. + h := newStreamHasher() + + if err := e.files.UploadToStage(catalogID, stageID, fa.Path, size, io.TeeReader(f, h)); err != nil { + return FileEntry{}, fmt.Errorf("upload %s: %w", fa.Path, err) } - return nil + return streamedEntry(h, size), nil } diff --git a/internal/workload/sync/upload_stage_test.go b/internal/workload/sync/upload_stage_test.go new file mode 100644 index 000000000..5a2407c4c --- /dev/null +++ b/internal/workload/sync/upload_stage_test.go @@ -0,0 +1,166 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestStageUploadHashesStreamedBytes verifies that the per-path hash in +// UploadOutcome.Sent is computed from the bytes the server actually received, +// not from a re-hash of the file on disk. The fake records the bytes it +// received, so the assertion is "the recorded hash matches the received +// bytes' SHA-256" rather than "it matches the disk file.". +func TestStageUploadHashesStreamedBytes(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees an upload. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + // The outcome must carry a Sent entry for app.py. + require.NotNil(t, e.uploadOutcome, "Phase 5 must store the upload outcome on the engine") + + sent, ok := e.uploadOutcome.Sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + // The streamed hash must equal the SHA-256 of the bytes the fake + // received, proving the hash is of the streamed bytes, not a re-hash + // of the disk file. + receivedBytes, ok := fake.uploadedFiles["app.py"] + require.True(t, ok, "fake must have recorded the uploaded bytes") + assert.Equal(t, sha256Hex(receivedBytes), sent.Hash, + "Sent hash must match the SHA-256 of the bytes the server received") + assert.Equal(t, int64(len(receivedBytes)), sent.Size, + "Sent size must match the byte count the server received") +} + +// TestStageUploadContentLengthFromOpenHandle verifies that content-length is +// derived from Stat() on the already-open handle, not from the Phase-2 planned +// size. A file that changes size between Plan and Execute must have the +// streamed size match the new size, not the planned size. This is only possible +// if the size comes from the open handle at upload time. +func TestStageUploadContentLengthFromOpenHandle(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('orig')\n", + }, catalogID, versionID) + + // Introduce a pending change so Plan sees an upload. + modifyFile(t, dir, "app.py", "print('ho')\n") // 11 bytes + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.Len(t, plan.Uploads, 1) + + plannedSize := plan.Uploads[0].LocalSize + + // Between Plan and Execute, grow the file. The streamed size must + // match the new size, not the Phase-2 planned size. + newContent := "print('hello world')\n" // 19 bytes + modifyFile(t, dir, "app.py", newContent) + + require.NotEqual(t, plannedSize, int64(len(newContent)), + "planned and streamed sizes must differ for the test to be meaningful") + + result, err := e.Execute(plan) + require.NoError(t, err) + require.NotNil(t, result) + + require.NotNil(t, e.uploadOutcome) + + sent, ok := e.uploadOutcome.Sent["app.py"] + require.True(t, ok) + + // The streamed size must match the bytes at Execute time, not the + // Phase-2 planned size. This proves content-length came from the open + // handle's Stat, not from fa.LocalSize. + assert.Equal(t, int64(len(newContent)), sent.Size, + "streamed size must match the file at Execute time, not the Phase-2 planned size") + assert.NotEqual(t, plannedSize, sent.Size, + "streamed size must differ from the Phase-2 planned size") + + // The hash must also match the new content. + assert.Equal(t, sha256Hex([]byte(newContent)), sent.Hash, + "streamed hash must match the content at Execute time") + + // The fake must have received exactly the new bytes. + receivedBytes := fake.uploadedFiles["app.py"] + assert.Equal(t, []byte(newContent), receivedBytes, + "server must have received the bytes present at Execute time") +} diff --git a/internal/workload/sync/upload_zip.go b/internal/workload/sync/upload_zip.go index 444e4a203..8bb17f0b7 100644 --- a/internal/workload/sync/upload_zip.go +++ b/internal/workload/sync/upload_zip.go @@ -30,49 +30,55 @@ import ( // POST it to FilesAPI, poll until terminal. type ZipUploader struct{} -// ApplyUploads zips the files, POSTs, and polls until done. -func (ZipUploader) ApplyUploads(e *Engine, files []FileAction) (string, string, error) { - zipPath, err := buildZip(e.projectDir, files) +// ApplyUploads zips the files, POSTs, polls until done, and returns the +// per-path streamed hashes so Phase 6 can record what entered the archive. +func (ZipUploader) ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) { + zipPath, sent, err := buildZip(e.projectDir, files) if err != nil { - return "", "", err + return UploadOutcome{}, err } defer func() { _ = os.Remove(zipPath) }() zipFile, err := os.Open(zipPath) if err != nil { - return "", "", fmt.Errorf("open built zip: %w", err) + return UploadOutcome{}, fmt.Errorf("open built zip: %w", err) } defer func() { _ = zipFile.Close() }() stat, err := zipFile.Stat() if err != nil { - return "", "", fmt.Errorf("stat built zip: %w", err) + return UploadOutcome{}, fmt.Errorf("stat built zip: %w", err) } resp, err := postZip(e, zipFile, stat.Size()) if err != nil { - return "", "", err + return UploadOutcome{}, err } // Small archives complete inline (201, no statusId); larger ones // come back 202 with a statusId we then poll. if resp.StatusID != "" { if err := waitForCompletion(e, resp.StatusID); err != nil { - return "", "", err + return UploadOutcome{}, err } } - return resp.CatalogID, resp.CatalogVersionID, nil + return UploadOutcome{ + CatalogID: resp.CatalogID, + VersionID: resp.CatalogVersionID, + Sent: sent, + }, nil } -// buildZip writes a zip archive to a temp file. Buffering on disk -// keeps very large zips from pinning a multi-GiB allocation. -func buildZip(projectDir string, files []FileAction) (string, error) { +// buildZip writes a zip archive to a temp file and returns the per-path +// streamed hashes. Buffering on disk keeps very large zips from pinning a +// multi-GiB allocation. +func buildZip(projectDir string, files []FileAction) (string, map[string]FileEntry, error) { tmp, err := os.CreateTemp("", "wapi-sync-*.zip") if err != nil { - return "", fmt.Errorf("create zip tempfile: %w", err) + return "", nil, fmt.Errorf("create zip tempfile: %w", err) } defer func() { _ = tmp.Close() }() @@ -81,27 +87,32 @@ func buildZip(projectDir string, files []FileAction) (string, error) { defer func() { _ = zw.Close() }() + sent := make(map[string]FileEntry, len(files)) + for _, fa := range files { abs := filepath.Join(projectDir, filepath.FromSlash(fa.Path)) - if err := addToZip(zw, abs, fa.Path); err != nil { + entry, err := addToZip(zw, abs, fa.Path) + if err != nil { _ = os.Remove(tmp.Name()) - return "", err + return "", nil, err } + + sent[fa.Path] = entry } if err := zw.Close(); err != nil { _ = os.Remove(tmp.Name()) - return "", fmt.Errorf("close zip writer: %w", err) + return "", nil, fmt.Errorf("close zip writer: %w", err) } - return tmp.Name(), nil + return tmp.Name(), sent, nil } -func addToZip(zw *zip.Writer, src, archivePath string) error { +func addToZip(zw *zip.Writer, src, archivePath string) (FileEntry, error) { in, err := os.Open(src) if err != nil { - return fmt.Errorf("open %s for zip: %w", src, err) + return FileEntry{}, fmt.Errorf("open %s for zip: %w", src, err) } defer func() { _ = in.Close() }() @@ -110,14 +121,22 @@ func addToZip(zw *zip.Writer, src, archivePath string) error { w, err := zw.CreateHeader(hdr) if err != nil { - return fmt.Errorf("zip header for %s: %w", archivePath, err) + return FileEntry{}, fmt.Errorf("zip header for %s: %w", archivePath, err) } - if _, err := io.Copy(w, in); err != nil { - return fmt.Errorf("copy %s into zip: %w", archivePath, err) + // Hash the bytes entering the archive, not the Phase-2 planned hash. + // MultiWriter mirrors the download verification at download.go: a file + // rewritten between plan and zip-build must leave BASE describing what + // the server extracted. The size comes from io.Copy's return, not from + // fa.LocalSize, so it always describes the bytes that entered the archive. + h := newStreamHasher() + + n, err := io.Copy(io.MultiWriter(w, h), in) + if err != nil { + return FileEntry{}, fmt.Errorf("copy %s into zip: %w", archivePath, err) } - return nil + return streamedEntry(h, n), nil } // postZip dispatches to UploadFromZipNew (first-sync, no catalog) or diff --git a/internal/workload/sync/uploader.go b/internal/workload/sync/uploader.go index b5e2e13b5..7ff454990 100644 --- a/internal/workload/sync/uploader.go +++ b/internal/workload/sync/uploader.go @@ -14,11 +14,23 @@ package sync -// Uploader pushes a SyncPlan's Uploads and returns the resulting -// (catalogID, newVersionID). When catalogID is empty (first-sync against -// an empty artifact) the implementation creates a new catalog. +// UploadOutcome is what an Uploader actually accomplished. Sent carries the +// hash and size of the bytes that really crossed the wire, keyed by the same +// forward-slash relative path used everywhere else in the manifest. Phase 6 +// seeds each uploaded file's manifest entry from Sent, never from the Phase-2 +// planned hash — a per-path fallback to the planned hash is the original +// poisoning bug and must not exist anywhere in the code. +type UploadOutcome struct { + CatalogID string + VersionID string + Sent map[string]FileEntry +} + +// Uploader pushes a SyncPlan's Uploads and returns the resulting outcome. +// When the artifact has no catalog (first-sync against an empty artifact) +// the implementation creates a new one. type Uploader interface { - ApplyUploads(e *Engine, files []FileAction) (catalogID, versionID string, err error) + ApplyUploads(e *Engine, files []FileAction) (UploadOutcome, error) } // ChooseUploader picks stage for small change sets (tight error semantics) From 14d8383a8a744efd0817e77b09b821acc8c91ff4 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 00:41:06 -0700 Subject: [PATCH 03/20] [RAPTOR-19525] test(workload): cover zip uploader hash-while-streaming Add internal/workload/sync/upload_zip_test.go, the first test file for the zip uploader, covering the archive path's hash-while-streaming behaviour introduced by upload-outcome-streamed-hashes. Tests: - TestZipBuild_HashesStreamedBytes: a normal multi-file archive whose per-path Sent hashes equal the SHA-256 of the file contents, with a genuine read-back (open the produced archive, extract entries, hash the extracted bytes and confirm they match the recorded Sent hash). - TestZipBuild_SameSizeContentChange: a file rewritten to different bytes of the SAME size between plan and addToZip, asserting the Sent hash is the archived bytes' hash and not the Phase-2 planned hash, with read-back confirming the archive contains the build-time bytes. - TestZipBuild_SizeChange: grow and shrink cases, asserting Sent size equals the byte count that entered the archive (io.Copy's return) rather than fa.LocalSize, with hash and size self-consistent and read-back confirming the extracted bytes match. - TestStageAndZipProduceIdenticalSent: given the same project tree, the stage path and the zip path produce identical Sent maps (same hashes and sizes for every file), so Phase 6 would write identical manifests. Read-back approach: call buildZip (the production function), open the archive it returns the path to, and hash the extracted entries. This asserts "the archive contains what we claimed" rather than "we hashed something." Every test was proven to fail first by temporarily reverting addToZip's hash-while-streaming (all hash assertions failed) and buildZip's streamed-size usage (size assertions failed with the planned size instead of the streamed size). Both reverts were restored; no production code was modified. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/upload_zip_test.go | 392 ++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 internal/workload/sync/upload_zip_test.go diff --git a/internal/workload/sync/upload_zip_test.go b/internal/workload/sync/upload_zip_test.go new file mode 100644 index 000000000..106db1cc4 --- /dev/null +++ b/internal/workload/sync/upload_zip_test.go @@ -0,0 +1,392 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "archive/zip" + "bytes" + "io" + "os" + "sort" + "testing" + + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// readZipEntries opens a zip archive at zipPath, extracts every entry, and +// returns a map of archive-path → uncompressed content bytes. This is the +// genuine read-back assertion: we open the archive that buildZip produced +// and hash the extracted entries, rather than re-hashing the source files. +// The assertion is "the archive contains what we claimed," not "we hashed +// something.". +func readZipEntries(t *testing.T, zipPath string) map[string][]byte { + t.Helper() + + data, err := os.ReadFile(zipPath) + require.NoError(t, err) + + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + + entries := make(map[string][]byte) + + for _, zf := range zr.File { + rc, err := zf.Open() + require.NoError(t, err) + + content, err := io.ReadAll(rc) + + _ = rc.Close() + + require.NoError(t, err) + + entries[zf.Name] = content + } + + return entries +} + +// fileActionsFrom builds a sorted slice of FileActions from a map of +// relative paths to content. LocalHash and LocalSize are set from the +// content so the actions are realistic, but addToZip and uploadOneToStage +// do not read them — they hash the bytes that actually enter the archive. +func fileActionsFrom(files map[string]string) []FileAction { + actions := make([]FileAction, 0, len(files)) + + for path, content := range files { + actions = append(actions, FileAction{ + Path: path, + LocalHash: sha256Hex([]byte(content)), + LocalSize: int64(len(content)), + }) + } + + sort.Slice(actions, func(i, j int) bool { return actions[i].Path < actions[j].Path }) + + return actions +} + +// TestZipBuild_HashesStreamedBytes verifies that buildZip records per-path +// Sent hashes equal to the SHA-256 of each file's content, and that the +// bytes read back out of the produced archive hash to the same values. This +// is the baseline correctness test for the zip path's hash-while-streaming +// behaviour: a normal multi-file archive with no mid-build changes. +// +// Read-back approach: we call buildZip (the production function), open the +// archive it returns the path to, extract each entry, and hash the extracted +// bytes. This proves the archive contains what the Sent map claims, rather +// than just that we hashed the source file. +// Fulfills VAL-UPLOAD-009 (sub-condition e: zip path produces hashes matching +// shasum of disk content). +func TestZipBuild_HashesStreamedBytes(t *testing.T) { + files := map[string]string{ + "app.py": "print('hello world')\n", + "utils/helper.py": "def help(): pass\n", + "config.yaml": "key: value\n", + "README.md": "# Project\n\nA test project.\n", + } + + dir := initProject(t, files) + + actions := fileActionsFrom(files) + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err) + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + // Read back the produced archive and extract entries. + archived := readZipEntries(t, zipPath) + + // Assert each file's Sent hash equals SHA-256 of its content, and + // the bytes read back from the archive hash to the same value. + for path, content := range files { + entry, ok := sent[path] + require.True(t, ok, "Sent must have an entry for %s", path) + + expectedHash := sha256Hex([]byte(content)) + expectedSize := int64(len(content)) + + assert.Equal(t, expectedHash, entry.Hash, + "Sent hash for %s must equal SHA-256 of file content", path) + assert.Equal(t, expectedSize, entry.Size, + "Sent size for %s must equal file byte count", path) + + // Read-back: the archive must contain the same bytes, and the + // extracted bytes must hash to the same value recorded in Sent. + archivedContent, ok := archived[path] + require.True(t, ok, "archive must contain entry for %s", path) + + assert.Equal(t, []byte(content), archivedContent, + "archived bytes for %s must match file content", path) + assert.Equal(t, expectedHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes for %s must match Sent hash", path) + } +} + +// TestZipBuild_SameSizeContentChange verifies that a file rewritten to +// different bytes of the SAME size between plan and addToZip records a Sent +// hash equal to the archived bytes' hash, NOT the Phase-2 planned hash. This +// is the zip-path analogue of the silent-poison TOCTOU case: the same-size +// rewrite produces no error and the hash must describe what entered the +// archive, not what was on disk at plan time. +// Fulfills VAL-UPLOAD-010. +func TestZipBuild_SameSizeContentChange(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('orig')\n", + }) + + // Phase-2 "planned" content: what was on disk at plan time. + planContent := "print('aaaa')\n" // 12 bytes + + modifyFile(t, dir, "app.py", planContent) + + plannedHash := sha256Hex([]byte(planContent)) + plannedSize := int64(len(planContent)) + + // Between plan and zip-build, rewrite to different bytes of the + // SAME size. This is the silent-poison window. + buildContent := "print('bbbb')\n" // 12 bytes, different content + + modifyFile(t, dir, "app.py", buildContent) + + require.Equal(t, plannedSize, int64(len(buildContent)), + "planned and build content must be the same size for this test") + + // The FileAction carries the Phase-2 planned hash and size. addToZip + // must NOT use these — it must hash the bytes that enter the archive. + actions := []FileAction{{ + Path: "app.py", + LocalHash: plannedHash, + LocalSize: plannedSize, + }} + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err) + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + entry, ok := sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + buildHash := sha256Hex([]byte(buildContent)) + + // The Sent hash must be the archived bytes' hash, not the Phase-2 + // planned hash. + assert.Equal(t, buildHash, entry.Hash, + "Sent hash must equal SHA-256 of the archived bytes, not the Phase-2 planned hash") + assert.NotEqual(t, plannedHash, entry.Hash, + "Sent hash must differ from the Phase-2 planned hash") + + // Read-back: confirm the archive contains the build-time bytes, and + // the extracted bytes hash to the recorded Sent hash. + archived := readZipEntries(t, zipPath) + + archivedContent := archived["app.py"] + require.NotNil(t, archivedContent, "archive must contain app.py") + + assert.Equal(t, []byte(buildContent), archivedContent, + "archive must contain the build-time bytes, not the plan-time bytes") + assert.Equal(t, buildHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes must match the recorded Sent hash") + assert.NotEqual(t, plannedHash, sha256Hex(archivedContent), + "extracted bytes must NOT hash to the Phase-2 planned hash") +} + +// TestZipBuild_SizeChange verifies that a file that grows or shrinks between +// plan and addToZip records a Sent size equal to the byte count that actually +// entered the archive (from io.Copy's return), not fa.LocalSize (the Phase-2 +// planned size). The hash and size must be self-consistent — both describe +// the same streamed bytes — and the upload must not fail. +// Fulfills VAL-UPLOAD-020. +func TestZipBuild_SizeChange(t *testing.T) { + tests := []struct { + name string + planContent string + buildContent string + }{ + { + name: "grows", + planContent: "print('hi')\n", // 11 bytes + buildContent: "print('hello')\n", // 14 bytes + }, + { + name: "shrinks", + planContent: "print('hello')\n", // 14 bytes + buildContent: "print('hi')\n", // 11 bytes + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('orig')\n", + }) + + // Phase-2 planned content. + modifyFile(t, dir, "app.py", tc.planContent) + + plannedHash := sha256Hex([]byte(tc.planContent)) + plannedSize := int64(len(tc.planContent)) + + // Between plan and zip-build, change the file size. + modifyFile(t, dir, "app.py", tc.buildContent) + + buildHash := sha256Hex([]byte(tc.buildContent)) + buildSize := int64(len(tc.buildContent)) + + require.NotEqual(t, plannedSize, buildSize, + "planned and build sizes must differ for this test to be meaningful") + + actions := []FileAction{{ + Path: "app.py", + LocalHash: plannedHash, + LocalSize: plannedSize, + }} + + zipPath, sent, err := buildZip(dir, actions) + require.NoError(t, err, "upload must not fail due to a size change") + + t.Cleanup(func() { _ = os.Remove(zipPath) }) + + entry, ok := sent["app.py"] + require.True(t, ok, "Sent must have an entry for app.py") + + // Sent size must equal the bytes that entered the archive, + // not the Phase-2 planned size (fa.LocalSize). + assert.Equal(t, buildSize, entry.Size, + "Sent size must equal the bytes that entered the archive, not fa.LocalSize") + assert.NotEqual(t, plannedSize, entry.Size, + "Sent size must differ from the Phase-2 planned size") + + // Hash and size must be self-consistent: both describe the + // same streamed bytes. + assert.Equal(t, buildHash, entry.Hash, + "Sent hash must equal SHA-256 of the build-time bytes") + assert.Equal(t, buildSize, entry.Size, + "Sent size must equal the build-time byte count") + + // Read-back: confirm the archive contains the build-time + // bytes, and the extracted bytes match the recorded Sent. + archived := readZipEntries(t, zipPath) + + archivedContent := archived["app.py"] + require.NotNil(t, archivedContent, "archive must contain app.py") + + assert.Equal(t, []byte(tc.buildContent), archivedContent, + "archive must contain the build-time bytes") + assert.Equal(t, buildHash, sha256Hex(archivedContent), + "SHA-256 of extracted bytes must match the recorded Sent hash") + assert.Equal(t, buildSize, int64(len(archivedContent)), + "extracted byte count must match the recorded Sent size") + }) + } +} + +// TestStageAndZipProduceIdenticalSent verifies that given the same project +// tree, the stage path and the zip path produce identical Sent maps (same +// hashes and sizes for every file). Since Phase 6 seeds the manifest from +// Sent, identical Sent maps mean identical manifests. Both uploaders are +// driven directly through their exported ApplyUploads methods against +// self-consistent fakes, so the comparison exercises the real production +// code paths on both sides. +// Fulfills VAL-UPLOAD-009 (sub-condition e: stage and zip paths both produce +// manifests where every hash matches shasum of disk content). +func TestStageAndZipProduceIdenticalSent(t *testing.T) { + files := map[string]string{ + "app.py": "print('hello world')\n", + "utils/helper.py": "def help(): pass\n", + "config.yaml": "key: value\n", + "README.md": "# Project\n", + } + + dir := initProject(t, files) + + actions := fileActionsFrom(files) + + const catalogID = "cid-equiv" + + cid := catalogID + + // Stage path: drive StageUploader.ApplyUploads directly. The engine + // only needs projectDir, files, and config.CatalogID for the upload + // code path — no phases run. + stageFake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-stage", + } + + stageEngine := &Engine{ + projectDir: dir, + files: stageFake, + config: wapi.Config{CatalogID: &cid}, + } + + stageOutcome, err := StageUploader{}.ApplyUploads(stageEngine, actions) + require.NoError(t, err) + + // Zip path: drive ZipUploader.ApplyUploads directly with a separate + // fake so the two do not share server state. + zipFake := &fakeFilesClient{ + catalogID: catalogID, + versionID: "ver-zip", + } + + zipEngine := &Engine{ + projectDir: dir, + files: zipFake, + config: wapi.Config{CatalogID: &cid}, + } + + zipOutcome, err := ZipUploader{}.ApplyUploads(zipEngine, actions) + require.NoError(t, err) + + // Both paths must produce Sent maps with the same number of entries. + require.Len(t, stageOutcome.Sent, len(actions), + "stage Sent must have one entry per file") + require.Len(t, zipOutcome.Sent, len(actions), + "zip Sent must have one entry per file") + + // Both must produce identical hashes and sizes for every file, and + // both must match the SHA-256 of the file content on disk. + for _, fa := range actions { + stageEntry, ok := stageOutcome.Sent[fa.Path] + require.True(t, ok, "stage Sent must have entry for %s", fa.Path) + + zipEntry, ok := zipOutcome.Sent[fa.Path] + require.True(t, ok, "zip Sent must have entry for %s", fa.Path) + + assert.Equal(t, stageEntry.Hash, zipEntry.Hash, + "stage and zip must produce the same hash for %s", fa.Path) + assert.Equal(t, stageEntry.Size, zipEntry.Size, + "stage and zip must produce the same size for %s", fa.Path) + + // Both must match the SHA-256 and byte count of the file content. + expectedHash := sha256Hex([]byte(files[fa.Path])) + expectedSize := int64(len(files[fa.Path])) + + assert.Equal(t, expectedHash, stageEntry.Hash, + "stage hash for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, expectedHash, zipEntry.Hash, + "zip hash for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, expectedSize, stageEntry.Size, + "stage size for %s must equal byte count", fa.Path) + assert.Equal(t, expectedSize, zipEntry.Size, + "zip size for %s must equal byte count", fa.Path) + } +} From 90c7fc16f29aaf17196975f06e7bd7b032e174fc Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 14:13:36 -0700 Subject: [PATCH 04/20] [RAPTOR-19525] test(workload): prove failed/partial uploads never advance persisted state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add regression coverage proving that a failed or partial upload never advances manifest.json or config.json, and that the Phase 6 hard-fail on a missing Sent entry actually holds. Eleven new test cases across four files: upload_failure_test.go (VAL-UPLOAD-011, VAL-UPLOAD-013, VAL-UPLOAD-021): - Missing Sent entry: buildNewBaseManifest hard-fails naming the path; phase6State does not write manifest.json. Also documents the SaveConfig-before-SaveManifest ordering hazard (config IS advanced when buildNewBaseManifest fails — not reachable through normal operation but a real defect, reported in handoff). - Partial Sent from worker error: uploadFilesParallel returns an error, Phase 6 never runs, state untouched. - One file fails late, all files fail, ApplyStage fails: each leaves manifest.json byte-identical, config.json unchanged, working tree rolled back, ApplyStage not called (or called but not persisted). - File deleted between Plan and Execute: error names the path, no panic, manifest/config unchanged, remaining files not applied. upload_concurrency_test.go (VAL-UPLOAD-012): - 4-way concurrent upload of 13 mixed-size files (0-byte, small, chunk-boundary 32 KiB) under -race: no race, exactly one Sent entry per file, no duplicate upload, all hashes match content. dry_run_integrity_test.go (VAL-UPLOAD-016): - Dry-run with pending changes: manifest.json content AND mtime unchanged, config.json unchanged, zero upload calls, zero AllFiles calls. Notes in comments what remains for staging validator (real mtime across a process invocation). interruption_test.go (VAL-CROSS-011): - Simulated mid-upload interruption via fault injection: first sync exits non-zero without advancing state, rollback directory remains; second sync restores stale rollback via Phase 0 and converges to a manifest matching the fake's server state. Notes in comments what remains for staging validator (real SIGINT delivery). Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../workload/sync/dry_run_integrity_test.go | 151 +++++ internal/workload/sync/interruption_test.go | 241 ++++++++ .../workload/sync/upload_concurrency_test.go | 165 ++++++ internal/workload/sync/upload_failure_test.go | 554 ++++++++++++++++++ 4 files changed, 1111 insertions(+) create mode 100644 internal/workload/sync/dry_run_integrity_test.go create mode 100644 internal/workload/sync/interruption_test.go create mode 100644 internal/workload/sync/upload_concurrency_test.go create mode 100644 internal/workload/sync/upload_failure_test.go diff --git a/internal/workload/sync/dry_run_integrity_test.go b/internal/workload/sync/dry_run_integrity_test.go new file mode 100644 index 000000000..7494297eb --- /dev/null +++ b/internal/workload/sync/dry_run_integrity_test.go @@ -0,0 +1,151 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDryRun_LeavesStateUntouched verifies that a --dry-run run with pending +// changes leaves manifest.json (content AND mtime), config.json, and the +// fake's server state all unchanged. The plan is non-empty (a file was +// modified), but Run returns after Plan without calling Execute, so no +// upload-side calls are issued and no state files are written. +// +// Go-level coverage: +// - manifest.json content is byte-identical before and after. +// - manifest.json mtime is unchanged (SaveManifest is never called; the +// file is not touched at all, so even the mtime survives). +// - config.json content is byte-identical (SaveConfig is never called). +// - The fake's upload counters are all zero (no stage, upload, apply, or +// zip calls). +// - AllFiles is not called (the artifact is not drifted, so the fast path +// copies BASE to REMOTE without a round-trip). +// +// What remains for a staging validator to confirm through the real binary: +// - Real mtime preservation across a process invocation (a Go test shares +// a process and a filesystem cache; a separate `dr` process touching +// the file would reset the mtime even if the content is unchanged). +// - The server's AllFiles listing is unchanged after the dry-run (the Go +// test asserts zero upload calls, which implies this, but the staging +// validator confirms it against the real server). +// +// Fulfills VAL-UPLOAD-016. +func TestDryRun_LeavesStateUntouched(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Introduce a pending change so the plan is non-empty. + modifyFile(t, dir, "app.py", "print('changed')\n") + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: "ver-should-not-be-used", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + // Capture pre-run state: manifest bytes, manifest mtime, config bytes. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + preManifest, err := os.ReadFile(mPath) + require.NoError(t, err) + + // Set a known mtime well in the past so any touch (even at the same + // content) is detectable regardless of filesystem mtime resolution. + knownTime := time.Now().Add(-1 * time.Hour) + require.NoError(t, os.Chtimes(mPath, knownTime, knownTime)) + + preInfo, err := os.Stat(mPath) + require.NoError(t, err) + + preManifestMtime := preInfo.ModTime() + + preConfig, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + // Run the dry-run. It must stop after Plan (DryRun is true). + result, err := e.Run() + require.NoError(t, err) + + require.NotNil(t, result, "dry-run must return a result") + + // The plan must be non-empty — the positive control that proves the + // zero-call assertions below are not vacuously true. + plan, planErr := e.Plan() + require.NoError(t, planErr) + + assert.False(t, plan.IsEmpty(), + "plan must be non-empty (a file was modified) for the zero-call assertions to be meaningful") + + // manifest.json content must be byte-identical. + postManifest, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, preManifest, postManifest, + "manifest.json content must be unchanged after a dry-run") + + // manifest.json mtime must be unchanged — SaveManifest was never called. + postInfo, err := os.Stat(mPath) + require.NoError(t, err) + + assert.True(t, postInfo.ModTime().Equal(preManifestMtime), + "manifest.json mtime must be unchanged after a dry-run (SaveManifest not called)") + + // config.json content must be byte-identical. + postConfig, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, preConfig, postConfig, + "config.json content must be unchanged after a dry-run") + + // No upload-side calls: the fake's server state is untouched. + assert.Equal(t, 0, fake.CreateStageCalls(), + "CreateStage must not be called in a dry-run") + assert.Equal(t, 0, fake.UploadToStageCalls(), + "UploadToStage must not be called in a dry-run") + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called in a dry-run") + assert.Equal(t, 0, fake.UploadFromZipCalls(), + "UploadFromZip must not be called in a dry-run") + assert.Equal(t, 0, fake.AllFilesCalls(), + "AllFiles must not be called in a dry-run on a non-drifted artifact (fast path)") +} diff --git a/internal/workload/sync/interruption_test.go b/internal/workload/sync/interruption_test.go new file mode 100644 index 000000000..5f512619d --- /dev/null +++ b/internal/workload/sync/interruption_test.go @@ -0,0 +1,241 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestInterruption_MidUpload_NonZeroAndConverges simulates a sync interrupted +// mid-upload and verifies that (1) the interrupted sync exits non-zero without +// advancing manifest.json or config.json, (2) the rollback directory remains +// for stale-rollback recovery, and (3) a following sync converges to a manifest +// matching the fake's server state. +// +// The interruption is simulated by injecting an upload failure (withFailNthUpload) +// rather than by delivering a real SIGINT. The effect is the same from the +// engine's perspective: Phase 5 fails, Phase 6 never runs, and the rollback +// directory is left on disk. The second sync's Phase 0 restores the stale +// rollback via RestoreStaleIfPresent, then proceeds normally. +// +// The fake for the second sync is pre-populated with the prior version (the +// server state from the original successful sync) so that ApplyStage's REPLACE +// merge produces a version containing both the uploaded files and the +// unchanged files (like .drignore). Without this, the fake's server would +// only hold the uploaded files and the convergence check would fail for +// unchanged paths that the manifest correctly carries forward from BASE. +// +// Go-level coverage: +// - The first sync returns an error (non-zero) without advancing manifest +// or config. +// - The rollback directory exists after the failed sync (it is NOT cleaned +// up by Restore, only by Discard on success or by the next sync's Phase 0). +// - The second sync's Phase 0 restores the stale rollback +// (e.StaleRollbackRestored() == true) and removes the rollback directory. +// - The second sync converges: the manifest's hashes match the fake's +// AllFiles server state for every file. +// +// What remains for a staging validator to confirm through the real binary: +// - Real SIGINT delivery to a running `dr` process (a Go test cannot +// deliver a signal to itself in a meaningful way; the test simulates the +// interruption via a fault-injected upload failure, which has the same +// engine-level effect: Phase 5 fails, Phase 6 never runs, the rollback +// directory remains for the next sync's stale-rollback recovery). +// - The real binary exits with a non-zero code on SIGINT (the Go test +// asserts the engine returns an error, which maps to a non-zero exit in +// the CLI, but the signal-to-exit-code path through the CLI's signal +// handler is the CLI's responsibility, not the engine's). +// +// Fulfills VAL-CROSS-011. +func TestInterruption_MidUpload_NonZeroAndConverges(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + } + + dir := syncedProject(t, files, catalogID, versionID) + + // Compute the prior version's server state (old content hashes) before + // modifying any files. The second sync's fake must be pre-populated with + // this so ApplyStage's REPLACE merge yields a version containing both + // the uploaded files and the unchanged ones (like .drignore). + priorVersion := make(map[string]filesapi.FileMeta, len(files)+1) + + for rel := range files { + hash, size, err := hashLocal(t, dir, rel) + require.NoError(t, err) + + priorVersion[rel] = filesapi.FileMeta{Hash: hash, Size: size} + } + + // Include .drignore (created by Initialize) so it is in the server state. + drHash, drSize, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + priorVersion[ignore.FileName] = filesapi.FileMeta{Hash: drHash, Size: drSize} + + // Now modify the files to introduce pending changes. + for rel, content := range mods { + modifyFile(t, dir, rel, content) + } + + // Capture pre-sync state. + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py"}) + + // --- First sync: simulate interruption mid-upload --- + + fake1 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(2) + + e1, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake1, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + plan1, err := e1.Plan() + require.NoError(t, err) + + require.Len(t, plan1.Uploads, 3, "all three files should be pending uploads") + + _, err = e1.Execute(plan1) + + require.Error(t, err, "interrupted sync must exit non-zero (error)") + assert.Contains(t, err.Error(), "upload", + "error must come from the upload step") + + require.NoError(t, e1.Close()) + + // manifest.json and config.json must not be advanced. + assertStateUntouched(t, dir, pre) + + // The rollback directory must exist — it is the stale rollback that + // the next sync's Phase 0 will restore. + rollDir := wapi.RollbackDir(dir) + + _, err = os.Stat(rollDir) + require.NoError(t, err, + "rollback directory must exist after a failed sync (for stale-rollback recovery)") + + // ApplyStage must not have been called — the upload failure prevented it. + assert.Equal(t, 0, fake1.ApplyStageCalls(), + "ApplyStage must not be called when the upload was interrupted") + + // --- Second sync: converge --- + + // Pre-populate the fake with the prior version so ApplyStage's REPLACE + // merge produces a version containing both uploaded and unchanged files. + fake2 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-2", + versionID: "ver-new", + }).withVersion(catalogID, versionID, priorVersion) + + e2, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake2, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + // Phase 0 must restore the stale rollback from the first sync. + plan2, err := e2.Plan() + require.NoError(t, err) + + assert.True(t, e2.StaleRollbackRestored(), + "Phase 0 must restore the stale rollback from the interrupted sync") + + require.Len(t, plan2.Uploads, 3, + "the same pending changes must still produce uploads") + + // Execute the second sync — it must converge. + result, err := e2.Execute(plan2) + require.NoError(t, err, "the second sync must succeed and converge") + + require.NotNil(t, result) + + // The rollback directory must be gone after a successful sync. + _, err = os.Stat(rollDir) + assert.True(t, os.IsNotExist(err), + "rollback directory must be removed after a successful sync") + + // The manifest must match the fake's server state for every file. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + all, err := fake2.AllFiles(catalogID, "ver-new") + require.NoError(t, err) + + // Every file in the manifest must have a hash matching the server. + for path, fm := range manifest.Files { + serverFM, ok := all[path] + require.True(t, ok, "server must have %s", path) + + assert.Equal(t, fm.Hash, serverFM.Hash, + "manifest hash for %s must match the server's checksum (converged)", path) + assert.Equal(t, fm.Size, serverFM.Size, + "manifest size for %s must match the server's size", path) + } + + // The manifest's syncedVersionId must equal the new version. + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, "ver-new", *manifest.SyncedVersionID, + "manifest syncedVersionId must equal the new version after convergence") + + // config.json's LastSyncedVersionID must also equal the new version. + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-new", *cfg.LastSyncedVersionID, + "config LastSyncedVersionID must equal the new version after convergence") +} diff --git a/internal/workload/sync/upload_concurrency_test.go b/internal/workload/sync/upload_concurrency_test.go new file mode 100644 index 000000000..16fe3fbcf --- /dev/null +++ b/internal/workload/sync/upload_concurrency_test.go @@ -0,0 +1,165 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// chunkBoundarySize is the default io.Copy buffer size (32 KiB). A file whose +// size is an exact multiple of this value exercises the chunk-boundary edge +// case: the last io.Copy chunk is exactly one buffer, not a partial. +const chunkBoundarySize = 32 * 1024 + +// TestConcurrentUpload_RaceFree_NoDroppedResults drives the stage uploader +// with 4-way concurrency (UploadConcurrency) over 13 files of mixed sizes — +// including 0-byte, small, and chunk-boundary sizes — and asserts: +// - No data race (automatic under `go test -race`). +// - Exactly one Sent entry per planned upload (no dropped result from a +// blocked channel send or an early worker return). +// - No file is uploaded twice (UploadToStageCalls == len(files)). +// - Every Sent hash equals the SHA-256 of the file content on disk. +// +// The result channel is buffered to len(files) so a send never blocks, and +// the orchestrator drains it after wg.Wait. A bug that closed resCh early, +// used an unbuffered channel, or returned before sending would drop a result +// and the Sent-count assertion would catch it. +// Fulfills VAL-UPLOAD-012. +func TestConcurrentUpload_RaceFree_NoDroppedResults(t *testing.T) { + // 13 files of mixed sizes: 0-byte, small, chunk-boundary, and larger. + fileSpecs := []struct { + path string + size int + }{ + {"f00.dat", 0}, + {"f01.dat", 0}, + {"f02.dat", 1}, + {"f03.dat", 10}, + {"f04.dat", 100}, + {"f05.dat", 500}, + {"f06.dat", 1000}, + {"f07.dat", chunkBoundarySize}, + {"f08.dat", chunkBoundarySize}, + {"f09.dat", chunkBoundarySize + 1}, + {"f10.dat", chunkBoundarySize * 2}, + {"f11.dat", 50000}, + {"f12.dat", 0}, + } + + dir := initProject(t, nil) + + // Write the files and build FileActions with known content. + actions := make([]FileAction, 0, len(fileSpecs)) + + expectedHashes := make(map[string]string, len(fileSpecs)) + + for _, spec := range fileSpecs { + content := generateContent(spec.path, spec.size) + + abs := filepath.Join(dir, spec.path) + require.NoError(t, os.WriteFile(abs, content, 0o644)) + + actions = append(actions, FileAction{ + Path: spec.path, + LocalHash: sha256Hex(content), + LocalSize: int64(len(content)), + }) + + expectedHashes[spec.path] = sha256Hex(content) + } + + const catalogID = "cid-conc" + + cid := catalogID + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: "ver-conc", + } + + engine := &Engine{ + projectDir: dir, + files: fake, + config: wapi.Config{CatalogID: &cid}, + } + + outcome, err := StageUploader{}.ApplyUploads(engine, actions) + + require.NoError(t, err, "all uploads must succeed with no fault injection") + + // Exactly one Sent entry per planned upload — no dropped result. + require.Len(t, outcome.Sent, len(fileSpecs), + "Sent must have exactly one entry per planned file (no dropped results)") + + // No file uploaded twice: UploadToStageCalls must equal the file count. + assert.Equal(t, len(fileSpecs), fake.UploadToStageCalls(), + "UploadToStage must be called exactly once per file (no duplicate uploads)") + + // ApplyStage called exactly once (the stage path applies in a single call). + assert.Equal(t, 1, fake.ApplyStageCalls(), + "ApplyStage must be called exactly once") + + // Every Sent entry must have the correct hash and size. + for _, fa := range actions { + entry, ok := outcome.Sent[fa.Path] + require.True(t, ok, "Sent must have an entry for %s", fa.Path) + + assert.Equal(t, expectedHashes[fa.Path], entry.Hash, + "Sent hash for %s must equal SHA-256 of file content", fa.Path) + assert.Equal(t, fa.LocalSize, entry.Size, + "Sent size for %s must equal file byte count", fa.Path) + } + + // The fake's server state must reflect every uploaded file with the + // correct checksum. This is the self-consistency check: AllFiles returns + // exactly what was uploaded. + all, err := fake.AllFiles(catalogID, "ver-conc") + require.NoError(t, err) + + assert.Len(t, all, len(fileSpecs), + "server must hold exactly one entry per uploaded file") + + for _, fa := range actions { + fm, ok := all[fa.Path] + require.True(t, ok, "server must have %s", fa.Path) + + assert.Equal(t, expectedHashes[fa.Path], fm.Hash, + "server checksum for %s must equal SHA-256 of content", fa.Path) + assert.Equal(t, fa.LocalSize, fm.Size, + "server size for %s must equal file byte count", fa.Path) + } +} + +// generateContent produces deterministic content of the given size for a +// given path. The content is a repeating pattern of the path name so each +// file has distinct bytes (avoiding hash collisions) while being fully +// deterministic across runs. +func generateContent(path string, size int) []byte { + if size == 0 { + return []byte{} + } + + pattern := path + ":" + + return []byte(strings.Repeat(pattern, size/len(pattern)+1))[:size] +} diff --git a/internal/workload/sync/upload_failure_test.go b/internal/workload/sync/upload_failure_test.go new file mode 100644 index 000000000..28df26122 --- /dev/null +++ b/internal/workload/sync/upload_failure_test.go @@ -0,0 +1,554 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// preSyncState captures the on-disk state of manifest.json, config.json, and +// the project files so a test can assert nothing was advanced after a failed +// sync. The manifest and config are captured as raw bytes for byte-identical +// comparison; the file contents are captured per-path so a rollback failure +// that leaves a modified file is caught. +type preSyncState struct { + manifestBytes []byte + configBytes []byte + fileContents map[string][]byte +} + +// captureState reads manifest.json, config.json, and the given project files +// as raw bytes. The caller lists the relative paths whose content matters +// for the rollback assertion. +func captureState(t *testing.T, dir string, files []string) preSyncState { + t.Helper() + + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + contents := make(map[string][]byte, len(files)) + + for _, rel := range files { + content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + + contents[rel] = content + } + + return preSyncState{ + manifestBytes: mBytes, + configBytes: cBytes, + fileContents: contents, + } +} + +// assertStateUntouched asserts manifest.json and config.json are byte-identical +// to their pre-sync content and every captured project file is unchanged. +// This is the core integrity assertion for every failure mode: a failed sync +// must never advance persisted state. +func assertStateUntouched(t *testing.T, dir string, pre preSyncState) { + t.Helper() + + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content") + + for rel, expected := range pre.fileContents { + content, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + require.NoError(t, err) + + assert.Equal(t, expected, content, + "project file %s must be unchanged after a failed sync (rollback)", rel) + } +} + +// newSyncedEngine builds an engine over a synced project with the given files +// modified to introduce pending changes. The fake and artifact store are +// returned so the caller can inspect counters and configure fault injection +// before running the engine. +func newSyncedEngine(t *testing.T, files map[string]string, mods map[string]string, fake *fakeFilesClient) (*Engine, string) { + t.Helper() + + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, files, catalogID, versionID) + + for rel, content := range mods { + modifyFile(t, dir, rel, content) + } + + e, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e, dir +} + +// --- VAL-UPLOAD-011: Missing streamed hash hard-fails and persists nothing --- + +// TestMissingSentEntry_BuildNewBaseManifestHardFails verifies that +// buildNewBaseManifest returns an error naming the missing path when the +// Sent map lacks an entry for an uploaded file. The error must NOT fall back +// to the Phase-2 planned hash — a per-path fallback IS the original poisoning +// bug. This is the unit-level guard; the engine-level guard +// (TestMissingSentEntry_Phase6DoesNotAdvanceManifest) verifies the on-disk +// consequence. +func TestMissingSentEntry_BuildNewBaseManifestHardFails(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, // missing app.py + }, + } + + _, err := buildNewBaseManifest(e, "ver", time.Now()) + + require.Error(t, err, "missing Sent entry must hard-fail, not fall back") + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") +} + +// TestMissingSentEntry_Phase6DoesNotAdvanceManifest verifies that when +// buildNewBaseManifest fails inside phase6State (because Sent is missing an +// entry), manifest.json is not written — it retains its exact pre-sync +// content. This is the on-disk consequence of the hard-fail. +// +// NOTE: This test also exposes the SaveConfig-before-SaveManifest ordering +// hazard: SaveConfig runs before buildNewBaseManifest, so config.json IS +// advanced even though the manifest is not. The missing-Sent scenario is not +// reachable through normal operation (uploadFilesParallel returns an error +// if any upload fails, so a partial Sent never reaches Phase 6), but the +// hazard is real and documented in architecture.md section 3.2. See +// discoveredIssues in the handoff. +func TestMissingSentEntry_Phase6DoesNotAdvanceManifest(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Construct an engine in the state Phase 6 would see after a successful + // Phase 5 where the Sent map is missing app.py. This is not reachable + // through the engine (uploadFilesParallel returns an error on any upload + // failure), so we test phase6State directly. + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: "ver-new", + Sent: map[string]FileEntry{}, // missing app.py + }, + newCatalogID: "cid-new", + newVersionID: "ver-new", + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when Sent is missing") + assert.Contains(t, err.Error(), "app.py", + "error must name the missing path") + + // manifest.json must NOT be written — SaveManifest runs after + // buildNewBaseManifest, which failed. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + // config.json: SaveConfig runs BEFORE buildNewBaseManifest in phase6State. + // When buildNewBaseManifest fails, SaveConfig has already persisted the + // advanced config (LastSyncedVersionID = "ver-new"). This is the + // SaveConfig-before-SaveManifest ordering hazard documented in + // architecture.md section 3.2. The missing-Sent scenario is not reachable + // through normal operation (uploadFilesParallel returns an error on any + // upload failure, so a partial Sent never reaches Phase 6), so this + // hazard is theoretical. We assert the current behaviour and document + // the gap; see discoveredIssues in the handoff. + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + // If the hazard were fixed (SaveConfig moved after buildNewBaseManifest, + // or made conditional on manifest success), this would assert equality. + // Today config IS advanced, so we assert the difference and name it. + assert.NotEqual(t, pre.configBytes, cBytes, + "config.json IS advanced by SaveConfig before buildNewBaseManifest fails (known hazard)") + + // The advanced config must point at the new version, proving the hazard + // is specifically the SaveConfig-before-SaveManifest ordering. + advancedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, advancedCfg.LastSyncedVersionID) + assert.Equal(t, "ver-new", *advancedCfg.LastSyncedVersionID, + "config LastSyncedVersionID must be advanced (the hazard)") + + // The project file must be unchanged (Phase 6 does not modify the tree). + content, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + + assert.Equal(t, pre.fileContents["app.py"], content, + "app.py must be unchanged") +} + +// TestMissingSentEntry_DoesNotFallBackToPhase2Hash verifies that the error +// from buildNewBaseManifest does not silently produce a manifest with the +// Phase-2 hash. The manifest is not written at all, so there is no entry to +// check — but we also verify the returned manifest is the zero value so no +// caller can accidentally use it. +func TestMissingSentEntry_DoesNotFallBackToPhase2Hash(t *testing.T) { + e := &Engine{ + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{}, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid", + VersionID: "ver", + Sent: map[string]FileEntry{}, + }, + } + + manifest, err := buildNewBaseManifest(e, "ver", time.Now()) + + require.Error(t, err) + assert.Empty(t, manifest.Files, + "no manifest must be produced when Sent is missing — no fallback") +} + +// --- VAL-UPLOAD-011b / VAL-UPLOAD-013: Failure modes through the engine --- + +// TestPartialSent_WorkerError_FailsCleanly verifies that when some workers +// succeed and one errors (producing a partial Sent internally), the sync +// fails, Phase 6 never runs, and manifest.json and config.json retain their +// pre-sync content. The partial Sent is never merged with a fallback because +// uploadFilesParallel returns an error, not a partial map. +// Fulfills VAL-UPLOAD-011 (partial Sent) and VAL-UPLOAD-013. +func TestPartialSent_WorkerError_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAA\n", + "b.py": "BBB\n", + "c.py": "CCC\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(2) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 3, "all three files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when a worker errors") + assert.Contains(t, err.Error(), "upload", + "error must come from the upload step") + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called when an upload fails — no partial apply") +} + +// TestUploadFailure_OneFileFailsLate_FailsCleanly verifies that when one file +// fails after others have been uploaded to the staging area, the sync fails, +// Phase 6 never runs, and all persisted state is untouched. Files staged but +// never applied must not appear in the manifest. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_OneFileFailsLate_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + "d.py": "ddd\n", + "e.py": "eee\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + "d.py": "DDDD\n", + "e.py": "EEEE\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(4) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 5, "all five files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py", "c.py", "d.py", "e.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when one file fails late") + assert.Contains(t, err.Error(), "upload", + "error must come from the upload step") + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called — staged-but-unapplied files must not reach the manifest") +} + +// TestUploadFailure_AllFilesFail_FailsCleanly verifies that when every upload +// fails, the sync fails immediately, Phase 6 never runs, and all persisted +// state is untouched. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_AllFilesFail_FailsCleanly(t *testing.T) { + files := map[string]string{ + "app.py": "print('orig')\n", + } + + mods := map[string]string{ + "app.py": "print('changed')\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailNthUpload(1) + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 1, "one file should be pending upload") + + pre := captureState(t, dir, []string{"app.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when all uploads fail") + assert.Contains(t, err.Error(), "app.py", + "error must name the failing path") + + assertStateUntouched(t, dir, pre) + + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called when all uploads fail") +} + +// TestUploadFailure_ApplyStageFails_FailsCleanly verifies that when all files +// are staged successfully but ApplyStage fails, the sync fails, Phase 6 +// never runs, and all persisted state is untouched. Files staged but never +// applied must not appear in the manifest. +// Fulfills VAL-UPLOAD-013. +func TestUploadFailure_ApplyStageFails_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + } + + fake := (&fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + }).withFailApplyStage() + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 2, "both files should be pending uploads") + + pre := captureState(t, dir, []string{"a.py", "b.py"}) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when ApplyStage fails") + assert.Contains(t, err.Error(), "apply stage", + "error must come from the apply-stage step") + + assertStateUntouched(t, dir, pre) + + // ApplyStage WAS called (it failed), but the resulting version must not + // be persisted. The manifest must not record hashes for staged-but- + // unapplied files. + assert.Equal(t, 1, fake.ApplyStageCalls(), + "ApplyStage must be called (it failed)") +} + +// --- VAL-UPLOAD-021: File deleted between Plan and Execute --- + +// TestDeletedFileBetweenPlanAndExecute_FailsCleanly verifies that a planned +// upload file deleted from disk between Plan() and Execute() produces an error +// naming that path (no panic), with manifest and config unchanged, rollback +// performed, and remaining planned files not applied. The first error stops +// the pipeline: ApplyStage is never called. +// Fulfills VAL-UPLOAD-021. +func TestDeletedFileBetweenPlanAndExecute_FailsCleanly(t *testing.T) { + files := map[string]string{ + "a.py": "aaa\n", + "b.py": "bbb\n", + "c.py": "ccc\n", + } + + mods := map[string]string{ + "a.py": "AAAA\n", + "b.py": "BBBB\n", + "c.py": "CCCC\n", + } + + fake := &fakeFilesClient{ + catalogID: "cid-synced", + stageID: "stage-1", + versionID: "ver-new", + } + + e, dir := newSyncedEngine(t, files, mods, fake) + + plan, err := e.Plan() + require.NoError(t, err) + + require.Len(t, plan.Uploads, 3, "all three files should be pending uploads") + + // Capture the state of the files that will NOT be deleted. + pre := captureState(t, dir, []string{"a.py", "c.py"}) + + // Between Plan and Execute, delete b.py from disk. os.Open will return + // os.ErrNotExist when the uploader tries to read it. + require.NoError(t, os.Remove(filepath.Join(dir, "b.py"))) + + _, err = e.Execute(plan) + + require.Error(t, err, "sync must fail when a planned file is missing") + assert.Contains(t, err.Error(), "b.py", + "error must name the deleted path") + assert.NotContains(t, err.Error(), "panic", + "error must not be a panic") + + // manifest.json and config.json must be byte-identical to pre-sync. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + mBytes, err := os.ReadFile(mPath) + require.NoError(t, err) + + assert.Equal(t, pre.manifestBytes, mBytes, + "manifest.json must be byte-identical to its pre-sync content") + + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content") + + // The remaining files (not deleted by the test) must be unchanged — + // the rollback restored the working tree to its pre-Execute state. + for _, rel := range []string{"a.py", "c.py"} { + content, err := os.ReadFile(filepath.Join(dir, rel)) + require.NoError(t, err) + + assert.Equal(t, pre.fileContents[rel], content, + "file %s must be unchanged after the failed sync", rel) + } + + // ApplyStage must not be called — the first error stops the pipeline. + assert.Equal(t, 0, fake.ApplyStageCalls(), + "ApplyStage must not be called when a planned file is missing") +} From f1cbeb505e967ce5d833890d977c4c296c4dcb2f Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 14:34:57 -0700 Subject: [PATCH 05/20] [RAPTOR-19525] fix(workload): write manifest before config in Phase 6 so config never advances ahead of manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 6 previously wrote config.json before manifest.json. A SaveManifest I/O failure (reachable in normal operation) left config advanced past a stale manifest — the manifest-poisoning shape this mission exists to close: the next sync sees no drift, fast-paths, copies the stale BASE to REMOTE, and reports "Up to date." forever. Reorder to buildNewBaseManifest → SaveManifest → SaveConfig. The failure direction is now safe: a SaveConfig failure leaves the manifest advanced and config stale, so the next sync detects drift, fetches AllFiles, and rebuilds BASE from real remote data — self-healing. The reorder is data-safe: nothing between the two writes reads config from disk. Flip TestMissingSentEntry_Phase6DoesNotAdvanceManifest to assert config is NOT advanced (was asserting it IS advanced as a known hazard). Add TestSaveManifestFailure_DoesNotAdvanceConfig and TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs to guard both failure directions. The write pair is still not transactional: each write is individually atomic via fsutil.AtomicWriteFile, but a crash between two successful writes still leaves one advanced. This residual risk is documented in library/phase6-write-ordering.md for the findings writeup. No transactionality mechanism is introduced. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/phase6_state.go | 21 +- internal/workload/sync/upload_failure_test.go | 313 ++++++++++++++++-- 2 files changed, 301 insertions(+), 33 deletions(-) diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index 897337760..487dcd5db 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -48,10 +48,19 @@ func phase6State(e *Engine) error { cfg.LastSyncedVersionID = &versionForState } - if err := wapi.SaveConfig(e.projectDir, cfg); err != nil { - return fmt.Errorf("save config: %w", err) - } - + // Build and write the manifest BEFORE writing config. The failure + // direction is what matters, not elegance: if SaveManifest fails, + // config has not been advanced yet, so the next sync sees the old + // version in config, detects drift, fetches AllFiles, and rebuilds + // BASE from real remote data — safe and self-healing. The converse + // (config advanced, manifest stale) silently poisons BASE: the next + // sync sees no drift, fast-paths, copies the stale BASE to REMOTE, + // and reports "Up to date." forever. + // + // This reorder is data-safe: nothing between the two writes reads + // config from disk. buildNewBaseManifest reads only e.remote, + // e.plan, and e.uploadOutcome (all in-memory). e.config and + // populateResult are touched only after both writes complete. manifest, err := buildNewBaseManifest(e, versionForState, now) if err != nil { return fmt.Errorf("build manifest: %w", err) @@ -61,6 +70,10 @@ func phase6State(e *Engine) error { return fmt.Errorf("save manifest: %w", err) } + if err := wapi.SaveConfig(e.projectDir, cfg); err != nil { + return fmt.Errorf("save config: %w", err) + } + if err := wapi.AppendHistory(e.projectDir, syncHistoryEntry(e, now)); err != nil { return fmt.Errorf("append history: %w", err) } diff --git a/internal/workload/sync/upload_failure_test.go b/internal/workload/sync/upload_failure_test.go index 28df26122..45bacccba 100644 --- a/internal/workload/sync/upload_failure_test.go +++ b/internal/workload/sync/upload_failure_test.go @@ -20,7 +20,9 @@ import ( "testing" "time" + "github.com/datarobot/cli/internal/drapi/filesapi" "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" "github.com/datarobot/cli/internal/workload/wapi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -165,16 +167,18 @@ func TestMissingSentEntry_BuildNewBaseManifestHardFails(t *testing.T) { // TestMissingSentEntry_Phase6DoesNotAdvanceManifest verifies that when // buildNewBaseManifest fails inside phase6State (because Sent is missing an -// entry), manifest.json is not written — it retains its exact pre-sync -// content. This is the on-disk consequence of the hard-fail. +// entry), neither manifest.json nor config.json is written — both retain +// their exact pre-sync content. This is the on-disk consequence of the +// hard-fail combined with the manifest-before-config write ordering. // -// NOTE: This test also exposes the SaveConfig-before-SaveManifest ordering -// hazard: SaveConfig runs before buildNewBaseManifest, so config.json IS -// advanced even though the manifest is not. The missing-Sent scenario is not -// reachable through normal operation (uploadFilesParallel returns an error -// if any upload fails, so a partial Sent never reaches Phase 6), but the -// hazard is real and documented in architecture.md section 3.2. See -// discoveredIssues in the handoff. +// The invariant: config never moves ahead of the manifest. Phase 6 now +// builds and writes the manifest first, then writes config. When +// buildNewBaseManifest fails, neither write has occurred, so config stays +// at the old version. The missing-Sent scenario is not reachable through +// normal operation (uploadFilesParallel returns an error if any upload +// fails, so a partial Sent never reaches Phase 6), but the ordering +// invariant is what protects the reachable hazard (SaveManifest I/O +// failure), so this test guards it directly. func TestMissingSentEntry_Phase6DoesNotAdvanceManifest(t *testing.T) { const ( catalogID = "cid-synced" @@ -229,32 +233,24 @@ func TestMissingSentEntry_Phase6DoesNotAdvanceManifest(t *testing.T) { assert.Equal(t, pre.manifestBytes, mBytes, "manifest.json must be byte-identical to its pre-sync content") - // config.json: SaveConfig runs BEFORE buildNewBaseManifest in phase6State. - // When buildNewBaseManifest fails, SaveConfig has already persisted the - // advanced config (LastSyncedVersionID = "ver-new"). This is the - // SaveConfig-before-SaveManifest ordering hazard documented in - // architecture.md section 3.2. The missing-Sent scenario is not reachable - // through normal operation (uploadFilesParallel returns an error on any - // upload failure, so a partial Sent never reaches Phase 6), so this - // hazard is theoretical. We assert the current behaviour and document - // the gap; see discoveredIssues in the handoff. + // config.json must NOT be advanced. Phase 6 now writes the manifest + // before config: buildNewBaseManifest runs first, and when it fails, + // neither SaveManifest nor SaveConfig has executed. Config stays at + // the old version, preserving the invariant that config never moves + // ahead of the manifest. cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) require.NoError(t, err) - // If the hazard were fixed (SaveConfig moved after buildNewBaseManifest, - // or made conditional on manifest success), this would assert equality. - // Today config IS advanced, so we assert the difference and name it. - assert.NotEqual(t, pre.configBytes, cBytes, - "config.json IS advanced by SaveConfig before buildNewBaseManifest fails (known hazard)") + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content — config never moves ahead of the manifest") - // The advanced config must point at the new version, proving the hazard - // is specifically the SaveConfig-before-SaveManifest ordering. - advancedCfg, err := wapi.LoadConfig(dir) + // Config must still point at the old version, proving the invariant. + unchangedCfg, err := wapi.LoadConfig(dir) require.NoError(t, err) - require.NotNil(t, advancedCfg.LastSyncedVersionID) - assert.Equal(t, "ver-new", *advancedCfg.LastSyncedVersionID, - "config LastSyncedVersionID must be advanced (the hazard)") + require.NotNil(t, unchangedCfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *unchangedCfg.LastSyncedVersionID, + "config LastSyncedVersionID must NOT be advanced when the manifest build fails") // The project file must be unchanged (Phase 6 does not modify the tree). content, err := os.ReadFile(filepath.Join(dir, "app.py")) @@ -291,6 +287,265 @@ func TestMissingSentEntry_DoesNotFallBackToPhase2Hash(t *testing.T) { "no manifest must be produced when Sent is missing — no fallback") } +// --- Phase 6 write-ordering invariants --- + +// TestSaveManifestFailure_DoesNotAdvanceConfig verifies that when SaveManifest +// fails inside phase6State (injected by making manifest.json a directory so +// the atomic rename fails), config.json is NOT advanced. Phase 6 now writes +// the manifest before config: SaveManifest runs first, and when it fails, +// SaveConfig has not executed. Config stays at the old version, preserving +// the invariant that config never moves ahead of the manifest. +// +// This is the reachable hazard (SaveManifest I/O failure) that the write +// reorder fixes. A stale config paired with an un-advanced manifest makes +// the next sync detect drift, fetch AllFiles, and rebuild BASE from real +// remote data — safe and self-healing. The converse (advanced config, +// stale manifest) silently poisons BASE. +func TestSaveManifestFailure_DoesNotAdvanceConfig(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Make manifest.json a directory so AtomicWriteFile's rename fails. + // SaveManifest creates a temp file in the parent dir, then renames it + // over the target — renaming a file over a directory fails with EISDIR. + mPath := filepath.Join(wapi.Dir(dir), "manifest.json") + + require.NoError(t, os.Remove(mPath)) + require.NoError(t, os.Mkdir(mPath, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(mPath) }) + + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + }, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: "ver-new", + Sent: map[string]FileEntry{ + "app.py": {Hash: sha256Hex([]byte("print('changed')\n")), Size: 14}, + }, + }, + newCatalogID: "cid-new", + newVersionID: "ver-new", + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when SaveManifest fails") + assert.Contains(t, err.Error(), "save manifest", + "error must come from SaveManifest, not SaveConfig") + + // config.json must NOT be advanced — SaveConfig runs after SaveManifest, + // which failed, so config was never written. + cBytes, err := os.ReadFile(wapi.ConfigPath(dir)) + require.NoError(t, err) + + assert.Equal(t, pre.configBytes, cBytes, + "config.json must be byte-identical to its pre-sync content — config never moves ahead of the manifest") + + unchangedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, unchangedCfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *unchangedCfg.LastSyncedVersionID, + "config LastSyncedVersionID must NOT be advanced when SaveManifest fails") +} + +// TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs verifies the safe +// failure direction: when SaveManifest succeeds but SaveConfig fails (injected +// by making config.json a directory), the manifest IS advanced (new version, +// new hashes) while config stays stale (old version). The next sync detects +// drift (config old != artifact new), fetches AllFiles rather than +// fast-pathing, and converges — config is updated to match the manifest. +// +// This is the self-healing property that makes manifest-before-config the +// safe ordering: an advanced manifest paired with a stale config makes the +// next run re-sync, whereas an advanced config paired with a stale manifest +// silently poisons BASE and reports "Up to date." forever. +func TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs(t *testing.T) { + const ( + catalogID = "cid-synced" + versionID = "ver-synced" + newVerID = "ver-new" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + pre := captureState(t, dir, []string{"app.py"}) + + // Make config.json a directory so SaveConfig's atomic rename fails. + // SaveManifest writes to manifest.json (a regular file), so it succeeds; + // SaveConfig writes to config.json (now a directory), so it fails. + cPath := wapi.ConfigPath(dir) + + require.NoError(t, os.Remove(cPath)) + require.NoError(t, os.Mkdir(cPath, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(cPath) }) + + streamedContent := "print('changed')\n" + streamedHash := sha256Hex([]byte(streamedContent)) + streamedSize := int64(len(streamedContent)) + + // Modify the disk file to match what was "uploaded" so that after + // SaveManifest writes the streamed hash, local == base and the next + // sync's plan is empty (the point is drift detection, not finding work). + modifyFile(t, dir, "app.py", streamedContent) + + // Include .drignore in remote so the manifest carries it through — + // buildNewBaseManifest seeds from remote, then overrides uploaded paths. + ignoreHash, ignoreSize, err := hashLocal(t, dir, ignore.FileName) + require.NoError(t, err) + + e := &Engine{ + projectDir: dir, + config: cfg, + plan: &SyncPlan{ + Uploads: []FileAction{ + {Path: "app.py", LocalHash: "phase2hash", LocalSize: 11}, + }, + }, + remote: RemoteManifest{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + ignore.FileName: {Hash: ignoreHash, Size: ignoreSize}, + }, + uploadOutcome: &UploadOutcome{ + CatalogID: "cid-new", + VersionID: newVerID, + Sent: map[string]FileEntry{ + "app.py": {Hash: streamedHash, Size: streamedSize}, + }, + }, + newCatalogID: "cid-new", + newVersionID: newVerID, + nowFn: time.Now, + } + + err = phase6State(e) + + require.Error(t, err, "phase6State must fail when SaveConfig fails") + assert.Contains(t, err.Error(), "save config", + "error must come from SaveConfig, not SaveManifest") + + // manifest.json IS advanced — SaveManifest ran before SaveConfig and + // succeeded. The manifest now carries the new version and the streamed hash. + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + assert.Equal(t, wapi.ManifestVersion, manifest.Version) + + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, newVerID, *manifest.SyncedVersionID, + "manifest syncedVersionId must be advanced") + + fm, ok := manifest.Files["app.py"] + require.True(t, ok) + + assert.Equal(t, streamedHash, fm.Hash, + "manifest must carry the streamed hash") + assert.Equal(t, streamedSize, fm.Size, + "manifest must carry the streamed size") + + // Restore config.json with the old content so the next sync can load it. + // This simulates the real-world state after a SaveConfig failure: the + // file retains its pre-sync content because the atomic write never landed. + require.NoError(t, os.RemoveAll(cPath)) + require.NoError(t, os.WriteFile(cPath, pre.configBytes, 0o644)) + + // Build the fake's server state from the manifest that SaveManifest wrote, + // so AllFiles returns exactly what BASE describes. This ensures the next + // sync sees base == remote and the plan is empty — the point is that the + // sync runs the full pipeline (drift detection, AllFiles fetch) rather + // than fast-pathing, not that it finds work to do. + serverFiles := make(map[string]filesapi.FileMeta, len(manifest.Files)) + + for path, fm := range manifest.Files { + serverFiles[path] = filesapi.FileMeta{Hash: fm.Hash, Size: fm.Size} + } + + // The next sync must detect drift (config old != artifact new), fetch + // AllFiles rather than fast-pathing, and converge. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-should-not-be-used", + versionID: newVerID, + }).withVersion(catalogID, newVerID, serverFiles) + + e2, err := newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVerID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + plan, err := e2.Plan() + require.NoError(t, err) + + // Drift was detected: AllFiles was called (not fast-pathed). + assert.Equal(t, 1, fake.AllFilesCalls(), + "drift must trigger an AllFiles round-trip, not the fast path") + + // The plan should be empty: manifest (advanced) == remote (AllFiles), + // and local == base (no disk changes since the failed sync). But the + // sync ran the full pipeline — it did not silently fast-path. + assert.True(t, plan.IsEmpty(), + "plan should be empty — manifest matches remote, no disk changes") + + // Execute to run Phase 6, which updates config to the new version. + result, err := e2.Execute(plan) + require.NoError(t, err) + + require.NotNil(t, result) + + // Config must now converge to the new version — self-healing. + convergedCfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + require.NotNil(t, convergedCfg.LastSyncedVersionID) + assert.Equal(t, newVerID, *convergedCfg.LastSyncedVersionID, + "config must converge to the new version after the self-healing sync") + + // Manifest and config must agree. + convergedManifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + require.NotNil(t, convergedManifest.SyncedVersionID) + assert.Equal(t, *convergedCfg.LastSyncedVersionID, *convergedManifest.SyncedVersionID, + "config and manifest must agree on the version after convergence") +} + // --- VAL-UPLOAD-011b / VAL-UPLOAD-013: Failure modes through the engine --- // TestPartialSent_WorkerError_FailsCleanly verifies that when some workers From a78ddccfd923e9ed2a0bcc141665251eb8521c61 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 14:45:25 -0700 Subject: [PATCH 06/20] [RAPTOR-19525] test(workload): lock in content-hash-only change detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add regression tests that would fail loudly if a size+mtime fast path (the mechanism RAPTOR-19525 alleged) were ever introduced: - TestSameSizeSameMtimeDetected: rewrites app.py to different bytes of identical length with mtime restored via os.Chtimes, asserts it appears in the upload plan — the codified refutation of the ticket's claim. - TestDetectionIsContentOnly: 2x2 matrix over {size matches manifest, size differs} x {content matches, content differs}, asserting detection fires iff content differs in every cell. - TestDetectionControls: true-positive controls (size+content change, 4 MiB large-file change) and false-positive controls (untouched file, mtime-only advance). - TestPlanRowSizesMatchDisk: each upload plan row's displayed byte size equals the real on-disk size at plan time. - TestClassifyTruthTable: comprehensive table over every (base, local, remote) hash triple covering all fourteen classifications and their mapped actions, transcribed from the current Classify implementation. - TestClassifyActionMappingIsExhaustive: every classification has a defined non-default action. Every detection test was proven to fail against a deliberately stubbed size+mtime fast path before being accepted. The false-positive controls were proven to fail against a mark-everything-dirty stub. The truth table was proven to catch a changed action mapping. No production code changes. Only *_test.go files added. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../sync/classify_truth_table_test.go | 269 +++++++++++++ .../sync/detection_regression_test.go | 361 ++++++++++++++++++ 2 files changed, 630 insertions(+) create mode 100644 internal/workload/sync/classify_truth_table_test.go create mode 100644 internal/workload/sync/detection_regression_test.go diff --git a/internal/workload/sync/classify_truth_table_test.go b/internal/workload/sync/classify_truth_table_test.go new file mode 100644 index 000000000..7bce46bb1 --- /dev/null +++ b/internal/workload/sync/classify_truth_table_test.go @@ -0,0 +1,269 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestClassifyTruthTable is a comprehensive table-driven test over every +// meaningful (base, local, remote) hash triple, covering all fourteen +// three-way classifications and their mapped actions. Each classification is +// exercised with multiple distinct hash values so the logic is proven +// value-agnostic, not merely correct for one chosen triple. +// +// An empty hash ("") means absent on that side. Non-empty hashes are compared +// by equality only; the actual hex content is irrelevant. The table is +// transcribed from the current Classify implementation (classify.go) and +// ActionFor, not from memory, so any later change to either function is +// caught. +// +// Fulfills VAL-REGRESSION-006. +func TestClassifyTruthTable(t *testing.T) { + // Four distinct non-empty hash values plus "" for absent. Using multiple + // values per branch proves the logic is value-agnostic. + const ( + A = "aaaa1111" + B = "bbbb2222" + C = "cccc3333" + D = "dddd4444" + ) + + cases := []struct { + name string + base string + local string + remote string + wantCls Classification + wantAct Action + }{ + // --- BASE ABSENT (classifyAbsentBase) --- + + // Both sides absent: nothing to do. + { + name: "absent_both_absent/empty_empty_empty", base: "", local: "", remote: "", + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + + // Local added, remote absent. + { + name: "absent_local_added/empty_A_empty", base: "", local: A, remote: "", + wantCls: ClsLocalAdded, wantAct: ActUploadAdd, + }, + { + name: "absent_local_added/empty_B_empty", base: "", local: B, remote: "", + wantCls: ClsLocalAdded, wantAct: ActUploadAdd, + }, + + // Remote added, local absent. + { + name: "absent_remote_added/empty_empty_A", base: "", local: "", remote: A, + wantCls: ClsRemoteAdded, wantAct: ActDownloadAdd, + }, + { + name: "absent_remote_added/empty_empty_B", base: "", local: "", remote: B, + wantCls: ClsRemoteAdded, wantAct: ActDownloadAdd, + }, + + // Both added the same content: no conflict, skip. + { + name: "absent_both_added_same/empty_A_A", base: "", local: A, remote: A, + wantCls: ClsBothAddedSame, wantAct: ActSkip, + }, + { + name: "absent_both_added_same/empty_B_B", base: "", local: B, remote: B, + wantCls: ClsBothAddedSame, wantAct: ActSkip, + }, + + // Both added different content: conflict. + { + name: "absent_add_conflict/empty_A_B", base: "", local: A, remote: B, + wantCls: ClsAddConflict, wantAct: ActConflictCopy, + }, + { + name: "absent_add_conflict/empty_C_D", base: "", local: C, remote: D, + wantCls: ClsAddConflict, wantAct: ActConflictCopy, + }, + + // --- BASE PRESENT, BOTH PRESENT (classifyBothPresentWithBase) --- + + // All three identical: unchanged. + { + name: "present_unchanged/A_A_A", base: A, local: A, remote: A, + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + { + name: "present_unchanged/B_B_B", base: B, local: B, remote: B, + wantCls: ClsUnchanged, wantAct: ActSkip, + }, + + // Local changed, remote unchanged: upload. + { + name: "present_local_modified/A_B_A", base: A, local: B, remote: A, + wantCls: ClsLocalModified, wantAct: ActUploadModify, + }, + { + name: "present_local_modified/A_C_A", base: A, local: C, remote: A, + wantCls: ClsLocalModified, wantAct: ActUploadModify, + }, + + // Remote changed, local unchanged: download. + { + name: "present_remote_modified/A_A_B", base: A, local: A, remote: B, + wantCls: ClsRemoteModified, wantAct: ActDownloadModify, + }, + { + name: "present_remote_modified/A_A_C", base: A, local: A, remote: C, + wantCls: ClsRemoteModified, wantAct: ActDownloadModify, + }, + + // Both changed to the same content: converged, skip. + { + name: "present_converged/A_B_B", base: A, local: B, remote: B, + wantCls: ClsConverged, wantAct: ActSkip, + }, + { + name: "present_converged/A_C_C", base: A, local: C, remote: C, + wantCls: ClsConverged, wantAct: ActSkip, + }, + + // Both changed differently: conflict. + { + name: "present_conflict/A_B_C", base: A, local: B, remote: C, + wantCls: ClsConflict, wantAct: ActConflictCopy, + }, + { + name: "present_conflict/B_C_D", base: B, local: C, remote: D, + wantCls: ClsConflict, wantAct: ActConflictCopy, + }, + + // --- BASE PRESENT, DELETION INVOLVED (classifyDeletionInvolvedWithBase) --- + + // Both deleted: skip. + { + name: "present_both_deleted/A_empty_empty", base: A, local: "", remote: "", + wantCls: ClsBothDeleted, wantAct: ActSkip, + }, + { + name: "present_both_deleted/B_empty_empty", base: B, local: "", remote: "", + wantCls: ClsBothDeleted, wantAct: ActSkip, + }, + + // Local deleted, remote unchanged from base: local delete. + { + name: "present_local_deleted/A_empty_A", base: A, local: "", remote: A, + wantCls: ClsLocalDeleted, wantAct: ActUploadDelete, + }, + { + name: "present_local_deleted/B_empty_B", base: B, local: "", remote: B, + wantCls: ClsLocalDeleted, wantAct: ActUploadDelete, + }, + + // Local deleted, remote changed from base: edit-del conflict. + // The user deleted the file, the teammate edited it. + { + name: "present_edit_del_conflict/A_empty_B", base: A, local: "", remote: B, + wantCls: ClsEditDelConflict, wantAct: ActDownloadOverDel, + }, + { + name: "present_edit_del_conflict/A_empty_C", base: A, local: "", remote: C, + wantCls: ClsEditDelConflict, wantAct: ActDownloadOverDel, + }, + + // Remote deleted, local unchanged from base: remote delete. + { + name: "present_remote_deleted/A_A_empty", base: A, local: A, remote: "", + wantCls: ClsRemoteDeleted, wantAct: ActDownloadDelete, + }, + { + name: "present_remote_deleted/B_B_empty", base: B, local: B, remote: "", + wantCls: ClsRemoteDeleted, wantAct: ActDownloadDelete, + }, + + // Remote deleted, local changed from base: del-edit conflict. + // The user edited the file, the teammate deleted it. + { + name: "present_del_edit_conflict/A_B_empty", base: A, local: B, remote: "", + wantCls: ClsDelEditConflict, wantAct: ActConflictCopy, + }, + { + name: "present_del_edit_conflict/A_C_empty", base: A, local: C, remote: "", + wantCls: ClsDelEditConflict, wantAct: ActConflictCopy, + }, + } + + // Verify we cover all fourteen classifications. + seenCls := make(map[Classification]struct{}) + for _, tc := range cases { + seenCls[tc.wantCls] = struct{}{} + } + + allCls := []Classification{ + ClsUnchanged, ClsLocalModified, ClsRemoteModified, ClsConverged, ClsConflict, + ClsLocalAdded, ClsRemoteAdded, ClsAddConflict, ClsLocalDeleted, ClsRemoteDeleted, + ClsBothDeleted, ClsDelEditConflict, ClsEditDelConflict, ClsBothAddedSame, + } + + for _, c := range allCls { + _, ok := seenCls[c] + assert.True(t, ok, "truth table must cover classification %s", c) + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + gotCls := Classify(tc.base, tc.local, tc.remote) + assert.Equal(t, tc.wantCls, gotCls, + "classification for (base=%q, local=%q, remote=%q)", tc.base, tc.local, tc.remote) + + gotAct := ActionFor(gotCls) + assert.Equal(t, tc.wantAct, gotAct, + "action for classification %s", gotCls) + }) + } +} + +// TestClassifyActionMappingIsExhaustive asserts that every classification has +// a defined action and that the action mapping is a total function: no +// classification maps to the zero-value ActSkip by falling through the switch +// default. This catches a refactor that adds a new classification without +// updating ActionFor. +func TestClassifyActionMappingIsExhaustive(t *testing.T) { + allCls := []Classification{ + ClsUnchanged, ClsLocalModified, ClsRemoteModified, ClsConverged, ClsConflict, + ClsLocalAdded, ClsRemoteAdded, ClsAddConflict, ClsLocalDeleted, ClsRemoteDeleted, + ClsBothDeleted, ClsDelEditConflict, ClsEditDelConflict, ClsBothAddedSame, + } + + // The skip classifications are the ones that legitimately map to ActSkip. + skipCls := map[Classification]bool{ + ClsUnchanged: true, + ClsConverged: true, + ClsBothDeleted: true, + ClsBothAddedSame: true, + } + + for _, c := range allCls { + act := ActionFor(c) + + if skipCls[c] { + assert.Equal(t, ActSkip, act, "%s must map to ActSkip", c) + } else { + assert.NotEqual(t, ActSkip, act, + "%s must map to a non-skip action; ActSkip here means it fell through the switch", c) + } + } +} diff --git a/internal/workload/sync/detection_regression_test.go b/internal/workload/sync/detection_regression_test.go new file mode 100644 index 000000000..39fea4adb --- /dev/null +++ b/internal/workload/sync/detection_regression_test.go @@ -0,0 +1,361 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests lock in the fact that change detection is purely content-hash +// based. The ticket (RAPTOR-19525) alleged a size+mtime fast path that silently +// skips files whose content changed but whose size and mtime are preserved. +// That mechanism does not exist in this codebase: Diff receives only hashes, +// and hashEntries rehashes every file every run. These tests would fail loudly +// if anyone ever introduced such a shortcut as a "performance improvement". + +// regressionEngine builds an engine over a syncedProject directory wired to a +// draft artifact whose codeRef matches the manifest's catalogID and versionID. +// The returned fakeFilesClient lets tests inspect call counters; the directory +// lets tests modify files on disk between Plan calls. +func regressionEngine(t *testing.T, files map[string]string) (*Engine, *fakeFilesClient, string) { + t.Helper() + + const ( + catalogID = "cid-regression" + versionID = "ver-regression" + ) + + dir := syncedProject(t, files, catalogID, versionID) + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-regression", + versionID: "ver-regression-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + return e, fake, dir +} + +// uploadPathSet returns the set of paths in the upload portion of the plan. +func uploadPathSet(plan *SyncPlan) map[string]struct{} { + out := make(map[string]struct{}, len(plan.Uploads)) + for _, fa := range plan.Uploads { + out[fa.Path] = struct{}{} + } + + return out +} + +// TestSameSizeSameMtimeDetected is the codified refutation of RAPTOR-19525's +// stated mechanism. A file is rewritten to different bytes of identical length, +// and its mtime is restored to the pre-change value via os.Chtimes. If change +// detection were size+mtime based (as the ticket alleged), this file would be +// invisible. Because detection is content-hash based, it appears in the plan +// as a modification to upload. +// +// Fulfills VAL-REGRESSION-001. +func TestSameSizeSameMtimeDetected(t *testing.T) { + const ( + original = "print('A')\n" // 12 bytes + modified = "print('B')\n" // 12 bytes, different content + ) + + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": original, + }) + + abs := filepath.Join(dir, "app.py") + + // Capture the original mtime before modification. + info, err := os.Stat(abs) + require.NoError(t, err) + + origMtime := info.ModTime() + + // Rewrite to different content of the same byte length. + require.NoError(t, os.WriteFile(abs, []byte(modified), 0o644)) + + // Restore the original mtime so both size and mtime match the pre-change + // state. This is the exact scenario the ticket alleged would be skipped. + require.NoError(t, os.Chtimes(abs, origMtime, origMtime)) + + // Confirm the mtime was actually restored (some filesystems have coarse + // mtime resolution, so verify rather than assume). + postInfo, err := os.Stat(abs) + require.NoError(t, err) + + assert.True(t, postInfo.ModTime().Equal(origMtime), + "mtime must be restored for the test to be meaningful") + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.True(t, present, + "same-size same-mtime content change must be detected as a modification; "+ + "if this fails, a size+mtime fast path was introduced") +} + +// TestDetectionIsContentOnly is a 2x2 matrix over {size matches manifest, size +// differs} x {content matches manifest, content differs}. Detection must fire +// if and only if content differs, in every cell. This proves detection is +// purely content-hash based and never consults size or mtime. +// +// The "size differs, content matches" cell is synthetic: the manifest's size +// field is manually corrupted while the hash stays correct. A size-based fast +// path would flag this file as changed; the hash-based mechanism correctly +// leaves it alone. +// +// Fulfills VAL-REGRESSION-005. +func TestDetectionIsContentOnly(t *testing.T) { + const ( + original = "print('A')\n" // 12 bytes + modified = "print('B')\n" // 12 bytes, same size, different content + grown = "print('BB')\n" // 13 bytes, different size and content + ) + + cases := []struct { + name string + setup func(t *testing.T, dir string) + wantDetected bool + }{ + { + name: "size_matches_content_matches", + // File unchanged: size and content both match the manifest. + // Detection must NOT fire. + setup: func(_ *testing.T, _ string) {}, + wantDetected: false, + }, + { + name: "size_matches_content_differs", + // Same-size content change: the ticket's exact scenario. + // Detection MUST fire. + setup: func(t *testing.T, dir string) { + t.Helper() + + modifyFile(t, dir, "app.py", modified) + }, + wantDetected: true, + }, + { + name: "size_differs_content_matches", + // Manifest's size field is corrupted while the hash stays correct. + // A size-based check would flag this; hash-based detection must NOT. + setup: func(t *testing.T, dir string) { + t.Helper() + + corruptManifestSize(t, dir, "app.py", 9999) + }, + wantDetected: false, + }, + { + name: "size_differs_content_differs", + // Normal modification: both size and content change. + // Detection MUST fire. + setup: func(t *testing.T, dir string) { + t.Helper() + + modifyFile(t, dir, "app.py", grown) + }, + wantDetected: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": original, + }) + + tc.setup(t, dir) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.Equal(t, tc.wantDetected, present, + "detection must fire iff content differs, regardless of size") + }) + } +} + +// TestDetectionControls provides true-positive and false-positive controls so +// the detection assertions above cannot be satisfied trivially. +// +// True positives: +// - size_and_content_change: both size and content change → detected. +// - large_file_content_change: a 4 MiB file whose content changes → detected, +// guarding against a future size-threshold shortcut for large files. +// +// False positives: +// - untouched_file: a file identical to its manifest entry → absent from plan. +// - mtime_only_advance: content unchanged, mtime advanced → not detected. +// +// Fulfills VAL-REGRESSION-003 and VAL-REGRESSION-004. +func TestDetectionControls(t *testing.T) { + t.Run("size_and_content_change", func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + modifyFile(t, dir, "app.py", "print('hello world')\n") + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.True(t, present, "a size-and-content change must be detected") + }) + + t.Run("large_file_content_change", func(t *testing.T) { + // A 4 MiB file guards against a future size-threshold shortcut that + // skips hashing for "large" files under the assumption they are + // unlikely to change without a size change. + const largeSize = 4 * 1024 * 1024 // 4 MiB + + original := strings.Repeat("A", largeSize) + modified := strings.Repeat("B", largeSize) + + e, _, dir := regressionEngine(t, map[string]string{ + "big.bin": original, + }) + + modifyFile(t, dir, "big.bin", modified) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["big.bin"] + assert.True(t, present, "a multi-megabyte content change must be detected") + }) + + t.Run("untouched_file", func(t *testing.T) { + e, _, _ := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + // No modifications: the file is identical to its manifest entry. + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.False(t, present, + "an untouched file must not appear in the upload plan") + }) + + t.Run("mtime_only_advance", func(t *testing.T) { + e, _, dir := regressionEngine(t, map[string]string{ + "app.py": "print('A')\n", + }) + + abs := filepath.Join(dir, "app.py") + + // Advance the mtime without changing the content. + future := time.Now().Add(2 * time.Hour) + require.NoError(t, os.Chtimes(abs, future, future)) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathSet(plan) + _, present := paths["app.py"] + assert.False(t, present, + "an mtime-only advance with identical content must not be detected") + }) +} + +// TestPlanRowSizesMatchDisk asserts that each upload plan row's displayed byte +// size (fa.LocalSize) equals the real on-disk file size at plan time. This +// ensures the sizes a validator reads off the plan can be trusted as evidence. +// +// Fulfills VAL-REGRESSION-014. +func TestPlanRowSizesMatchDisk(t *testing.T) { + files := map[string]string{ + "app.py": "print('A')\n", + "config.yaml": "name: test\n", + "data.bin": "ABCDEF", + } + + e, _, dir := regressionEngine(t, files) + + // Modify files to different sizes so the plan is non-empty and the + // sizes on disk differ from the manifest's recorded sizes. + modifyFile(t, dir, "app.py", "print('hello world')\n") + modifyFile(t, dir, "data.bin", "ABCDEFGHIJKL") + + plan, err := e.Plan() + require.NoError(t, err) + + require.NotEmpty(t, plan.Uploads, "plan must have uploads for the size assertion to be meaningful") + + for _, fa := range plan.Uploads { + abs := filepath.Join(dir, filepath.FromSlash(fa.Path)) + info, err := os.Stat(abs) + require.NoError(t, err, "stat %s", fa.Path) + + assert.Equal(t, info.Size(), fa.LocalSize, + "upload plan row for %s: displayed size must equal real on-disk size", fa.Path) + } +} + +// corruptManifestSize loads the manifest, changes the size field for the given +// path to a wrong value while keeping the hash correct, and re-saves it. This +// creates a synthetic "size differs, content matches" state for the detection +// matrix: the file on disk is unchanged (same hash), but the manifest's size +// field no longer matches the real size. A size-based fast path would flag +// this file as changed; the hash-based mechanism correctly leaves it alone. +func corruptManifestSize(t *testing.T, dir, rel string, wrongSize int64) { + t.Helper() + + manifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + fm, ok := manifest.Files[rel] + require.True(t, ok, "manifest must contain %s", rel) + + // Keep the hash, corrupt only the size. + fm.Size = wrongSize + manifest.Files[rel] = fm + + require.NoError(t, wapi.SaveManifest(dir, manifest)) +} From e34db052bc54d60884b8cbb22263bb8e28679ce5 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Wed, 26 Aug 2026 15:02:04 -0700 Subject: [PATCH 07/20] [RAPTOR-19525] test(workload): guard sync surroundings against regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add regression coverage for the behaviour immediately surrounding the upload-integrity fix so it cannot regress silently. Covers ignore rules (.drignore exclusion, .wapiignore legacy shadow warning, system excludes, case-folding), case-collision detection before any upload, path normalization (forward slashes, NFC, no leading ./ or trailing slash) in both manifest build and base/remote comparison, plan action mapping (local-deleted → upload-delete, remote-modified → download, both-sides → conflict, remote-wins resolution), dr workload up's code-change measurement (modified tree count, unchanged tree zero, first-deploy flag for unlinked project), the sync lock (second concurrent sync rejected, stale-rollback recovery, forward state migration). 22 new test cases across two files. Every representative test was proven to fail against a deliberately broken production line before being accepted. Full task test green; task lint clean on linux, darwin, and windows. No production files touched. Fulfills VAL-REGRESSION-007, VAL-REGRESSION-008, VAL-REGRESSION-009. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../sync/surroundings_regression_test.go | 968 ++++++++++++++++++ .../workload/up/codechange_regression_test.go | 140 +++ 2 files changed, 1108 insertions(+) create mode 100644 internal/workload/sync/surroundings_regression_test.go create mode 100644 internal/workload/up/codechange_regression_test.go diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go new file mode 100644 index 000000000..aab52aa00 --- /dev/null +++ b/internal/workload/sync/surroundings_regression_test.go @@ -0,0 +1,968 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/fileops" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests guard the behaviour immediately surrounding the upload-integrity +// fix so it cannot regress silently. They cover ignore rules, case-collision +// detection, path normalization, plan action mapping, the sync lock, +// stale-rollback recovery, and state migration — all through the engine's +// Plan/Execute seam so a regression in any of them is caught at the level +// that matters. +// +// Fulfills VAL-REGRESSION-007 and VAL-REGRESSION-009. + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(a): Ignore rules +// --------------------------------------------------------------------------- + +// TestDrignoreExcludesFromPlan verifies that files matching .drignore patterns +// are absent from the upload plan. A .drignore pattern that excludes *.tmp must +// keep scratch.tmp out of the uploads while letting app.py through. +func TestDrignoreExcludesFromPlan(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "scratch.tmp": "temporary\n", + }) + + // Overwrite the .drignore template with a pattern that excludes *.tmp. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), []byte("*.tmp\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, "app.py", "non-ignored file must be in the plan") + assert.NotContains(t, paths, "scratch.tmp", ".drignore-excluded file must be absent from the plan") +} + +// TestWapiignoreShadowWarning_LegacyOnly verifies that the deprecation notice +// fires when only the legacy .wapiignore filename is present (no .drignore). +// The notice tells the user the old name is deprecated and to rename it. +func TestWapiignoreShadowWarning_LegacyOnly(t *testing.T) { + dir := legacyProject(t, map[string]string{ + ignore.LegacyFileName: "*.tmp\n", + "scratch.tmp": "x", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + _, err := e.Plan() + require.NoError(t, err) + + notice := e.IgnoreFileNotice() + assert.Contains(t, notice, ignore.LegacyFileName, "notice must name the legacy file") + assert.Contains(t, notice, ignore.FileName, "notice must name the current file to rename to") +} + +// TestWapiignoreShadowWarning_BothPresent verifies that the shadow warning +// fires when both .drignore and .wapiignore are present. The current name wins +// and the legacy patterns are inert, which the warning must say. +func TestWapiignoreShadowWarning_BothPresent(t *testing.T) { + dir := initProject(t, map[string]string{ + ignore.FileName: "*.new\n", + ignore.LegacyFileName: "*.old\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + _, err := e.Plan() + require.NoError(t, err) + + // The shadow warning goes to log.Warn, not to IgnoreFileNotice. The + // notice is empty because the current name is in effect. + assert.Empty(t, e.IgnoreFileNotice(), "current name in effect has no deprecation notice") +} + +// TestSystemExcludes_StillExcluded verifies that built-in system-excluded paths +// (.datarobot/workload, .wapi, .git, .gitignore, .datarobot.yaml) are absent +// from the upload plan even when no .drignore file exists. +func TestSystemExcludes_StillExcluded(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Create system-excluded paths inside the project. Use a file name that + // does not overwrite the real config.json created by initProject. + require.NoError(t, os.WriteFile(filepath.Join(dir, ".datarobot", "workload", "extra.json"), []byte("{}"), 0o644)) + + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("*.tmp\n"), 0o644)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, ".datarobot.yaml"), []byte("name: test\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, "app.py", "regular file must be in the plan") + + for _, excluded := range []string{ + ".datarobot/workload/extra.json", + ".git/HEAD", + ".gitignore", + ".datarobot.yaml", + } { + assert.NotContains(t, paths, excluded, "system-excluded path must be absent from the plan: %s", excluded) + } +} + +// TestCaseFoldedSystemExcludes verifies that system excludes fold case: a +// .Datarobot/workload path is excluded just like .datarobot/workload. +func TestCaseFoldedSystemExcludes(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Create a differently-cased state directory. + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".Datarobot", "workload"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".Datarobot", "workload", "secret.json"), []byte("{}"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + assert.NotContains(t, paths, ".Datarobot/workload/secret.json", + "case-folded system exclude must still be excluded") +} + +// TestCaseFoldedIgnorePatterns verifies that case-folding of ignore patterns +// still matches paths differing only in case. A .drignore pattern "BUILD/" +// must exclude a directory named "build/" (case-folded match via system +// excludes is already tested above; this tests user-pattern case-folding +// through the gitignore library). +func TestCaseFoldedIgnorePatterns(t *testing.T) { + // The gitignore library is case-sensitive on Unix but the system excludes + // fold case. Here we test that a user pattern in .drignore matches a + // file of the same name with different case, which the system-exclude + // case-folding handles for system paths. For user patterns, the + // gitignore library is case-sensitive, so a pattern "*.TMP" does NOT + // match "scratch.tmp". We verify the actual behaviour: user patterns are + // case-sensitive (as documented), while system excludes fold case. + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "scratch.tmp": "x", + }) + + // Pattern with uppercase extension — gitignore is case-sensitive, + // so *.TMP does NOT match scratch.tmp. + require.NoError(t, os.WriteFile(filepath.Join(dir, ignore.FileName), []byte("*.TMP\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + paths := uploadPathsOf(plan) + + // User patterns are case-sensitive, so scratch.tmp is NOT excluded by *.TMP. + assert.Contains(t, paths, "scratch.tmp", + "user patterns are case-sensitive: *.TMP does not match scratch.tmp") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(b): Case-collision hard error before any upload +// --------------------------------------------------------------------------- + +// TestCaseCollision_FailsBeforeUpload verifies that two paths differing only +// in case cause the sync to fail with a case-collision error. On a +// case-insensitive filesystem (macOS, Windows) the walker cannot produce two +// such entries, so this test calls caseCollisionsFromManifest directly with a +// crafted manifest — the same function phase2Manifests calls after hashing. +func TestCaseCollision_FailsBeforeUpload(t *testing.T) { + // Craft a local manifest with two paths differing only in case. + local := LocalManifest{ + "Config.yaml": {Hash: "aaa", Size: 10}, + "config.yaml": {Hash: "bbb", Size: 20}, + } + + collisions := caseCollisionsFromManifest(local) + + require.Len(t, collisions, 1, "one case-collision group expected") + assert.Equal(t, "config.yaml", collisions[0].Lowered) + assert.ElementsMatch(t, []string{"Config.yaml", "config.yaml"}, collisions[0].Paths) + + msg := fileops.FormatCaseCollisions(collisions) + assert.Contains(t, msg, "case-only path collisions") + assert.Contains(t, msg, "Config.yaml vs config.yaml") +} + +// TestCaseCollision_ErrorIsBeforeUpload verifies that the case-collision check +// runs in Phase 2 (manifests), before Phase 5 (execute) where uploads happen. +// We cannot create case-colliding files on macOS, so we verify the check +// position by confirming that a Plan with no case collisions succeeds and +// the fake's upload counters are zero (Plan does not execute). +func TestCaseCollision_ErrorIsBeforeUpload(t *testing.T) { + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, "cid-cc", "ver-cc") + + fake := &fakeFilesClient{ + catalogID: "cid-cc", + stageID: "stage-cc", + versionID: "ver-cc-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "cid-cc", "ver-cc"), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + require.True(t, plan.IsEmpty(), "synced project with no changes has an empty plan") + + // No case collision, no uploads — Plan does not call Execute. + assert.Equal(t, 0, fake.UploadToStageCalls(), "Plan must not upload") + assert.Equal(t, 0, fake.ApplyStageCalls(), "Plan must not apply") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(c): Path normalization +// --------------------------------------------------------------------------- + +// TestPathNormalization_ForwardSlash verifies that relative paths in the plan +// use forward slashes, not backslashes. On Unix this is trivially true; on +// Windows the walker must convert OS-native backslashes to forward slashes. +func TestPathNormalization_ForwardSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "src/utils/helper.py": "def help(): pass\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.NotContains(t, fa.Path, "\\", "path must use forward slashes: %s", fa.Path) + } +} + +// TestPathNormalization_NoLeadingDotSlash verifies that relative paths in the +// plan have no leading "./" prefix. +func TestPathNormalization_NoLeadingDotSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.False(t, strings.HasPrefix(fa.Path, "./"), + "path must not have leading ./: %s", fa.Path) + } +} + +// TestPathNormalization_NoTrailingSlash verifies that relative paths in the +// plan have no trailing slash. +func TestPathNormalization_NoTrailingSlash(t *testing.T) { + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + "src/utils/helper.py": "def help(): pass\n", + }) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + for _, fa := range plan.Uploads { + assert.False(t, strings.HasSuffix(fa.Path, "/"), + "path must not have trailing slash: %s", fa.Path) + } +} + +// TestPathNormalization_NFC verifies that relative paths in the plan are +// NFC-normalized. On macOS the filesystem stores filenames in NFD, so a file +// named café.py (NFD: cafe + combining accent) must appear as café.py (NFC) +// in the plan. +func TestPathNormalization_NFC(t *testing.T) { + // NFC form of "café": precomposed é (U+00E9). + nfc := "café.py" + + // NFD form: e + combining acute (U+0301). macOS stores this on disk. + nfd := "cafe\u0301.py" + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Write the file with the NFD name — the filesystem will store it as-is. + require.NoError(t, os.WriteFile(filepath.Join(dir, nfd), []byte("x"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err) + + // The plan must contain the NFC form, not the NFD form. + paths := uploadPathsOf(plan) + + assert.Contains(t, paths, nfc, "path must be NFC-normalized in the plan") + assert.NotContains(t, paths, nfd, "NFD form must not appear in the plan") + + // Verify the manifest key is also NFC by checking that NormalizePath + // produces NFC from the NFD input. + assert.Equal(t, nfc, fileops.NormalizePath(nfd)) +} + +// TestPathNormalization_ManifestAndComparison verifies that path normalization +// is consistent between the local manifest build and the base/remote +// comparison. When a synced project has a file with a non-ASCII name, the +// manifest key (base) and the plan path (local) must both be NFC-normalized +// so the diff does not falsely report a change. +func TestPathNormalization_ManifestAndComparison(t *testing.T) { + nfc := "café.py" + + nfd := "cafe\u0301.py" + + dir := initProject(t, map[string]string{ + "app.py": "print('hi')\n", + }) + + // Write the file with the NFD name. + require.NoError(t, os.WriteFile(filepath.Join(dir, nfd), []byte("x"), 0o644)) + + // syncedProject builds the manifest from the file hashes, but we need + // to ensure the manifest key is NFC. Let's build it manually. + catalogID := "cid-nfc" + versionID := "ver-nfc" + + cfg, err := wapi.LoadConfig(dir) + require.NoError(t, err) + + cfg.CatalogID = &catalogID + cfg.LastSyncedVersionID = &versionID + require.NoError(t, wapi.SaveConfig(dir, cfg)) + + // Build manifest with NFC keys (what a correct sync would produce). + manifestFiles := make(map[string]wapi.FileMeta) + + for _, rel := range []string{"app.py", nfc, ignore.FileName} { + hash, size, err := hashLocal(t, dir, rel) + if err != nil { + // The file might be stored under NFD on disk; try the NFD name. + hash, size, err = hashLocal(t, dir, nfd) + require.NoError(t, err) + } + + manifestFiles[rel] = wapi.FileMeta{Hash: hash, Size: size} + } + + syncedAt := time.Now().UTC() + manifest := wapi.Manifest{ + Version: wapi.ManifestVersion, + SyncedAt: &syncedAt, + SyncedVersionID: &versionID, + Files: manifestFiles, + } + require.NoError(t, wapi.SaveManifest(dir, manifest)) + + // Now Plan: the local walk produces NFC paths, the base has NFC keys, + // so the diff should find no changes for café.py. + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-nfc", + versionID: "ver-nfc-next", + } + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // The plan must be empty: local == base (both NFC), and the fast path + // copies base to remote, so no diff. + assert.True(t, plan.IsEmpty(), + "NFC-normalized paths must match between local and base: plan should be empty, got uploads=%v deletes=%v", + uploadPathsOf(plan), deletePathsOf(plan)) +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-007(d): Plan action mapping +// --------------------------------------------------------------------------- + +// TestPlanAction_LocalDeleted verifies that a file deleted locally (present in +// base and remote, absent from local) produces an upload-delete action in the +// plan's Deletes list. +func TestPlanAction_LocalDeleted(t *testing.T) { + const ( + catalogID = "cid-del" + versionID = "ver-del" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "to-delete.py": "x\n", + }, catalogID, versionID) + + // Delete the file locally. + require.NoError(t, os.Remove(filepath.Join(dir, "to-delete.py"))) + + // Pre-populate the fake's server state with the synced version. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-del", + versionID: "ver-del-next", + }).withVersion(catalogID, versionID, map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + "to-delete.py": {Hash: sha256Hex([]byte("x\n")), Size: 2}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + // The artifact must point at the synced version so the engine detects drift. + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // The deleted file must appear in the Deletes list with ActUploadDelete. + var deleteActions []FileAction + + for _, fa := range plan.Deletes { + if fa.Path == "to-delete.py" { + deleteActions = append(deleteActions, fa) + } + } + + require.Len(t, deleteActions, 1, "to-delete.py must be in the Deletes list") + assert.Equal(t, ActUploadDelete, deleteActions[0].Action, + "locally-deleted file must map to ActUploadDelete") + assert.Equal(t, ClsLocalDeleted, deleteActions[0].Classification) +} + +// TestPlanAction_RemoteModified verifies that a file modified on the remote +// only (base == local, remote differs) produces a download action. +func TestPlanAction_RemoteModified(t *testing.T) { + const ( + catalogID = "cid-rm" + versionID = "ver-rm" + remoteVerID = "ver-rm-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // The server has a different version of app.py. The artifact's codeRef + // points at remoteVerID (different from the synced versionID) so the + // engine detects drift and fetches AllFiles for the remote version. + remoteHash := sha256Hex([]byte("print('changed')\n")) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-rm", + versionID: "ver-rm-next", + }).withVersion(catalogID, remoteVerID, map[string]filesapi.FileMeta{ + "app.py": {Hash: remoteHash, Size: 16}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // app.py must appear in the Downloads list. + var downloads []FileAction + + for _, fa := range plan.Downloads { + if fa.Path == "app.py" { + downloads = append(downloads, fa) + } + } + + require.Len(t, downloads, 1, "app.py must be in the Downloads list") + assert.Equal(t, ActDownloadModify, downloads[0].Action, + "remote-only modification must map to ActDownloadModify") + assert.Equal(t, ClsRemoteModified, downloads[0].Classification) +} + +// TestPlanAction_BothSidesDifferent verifies that a file modified on both sides +// differently (local != base, remote != base, local != remote) produces a +// conflict. +func TestPlanAction_BothSidesDifferent(t *testing.T) { + const ( + catalogID = "cid-conf" + versionID = "ver-conf" + remoteVerID = "ver-conf-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Modify the file locally. + modifyFile(t, dir, "app.py", "print('local-change')\n") + + // The server has a different version of app.py. The artifact's codeRef + // points at remoteVerID (different from the synced versionID) so the + // engine detects drift and fetches AllFiles for the remote version. + remoteHash := sha256Hex([]byte("print('remote-change')\n")) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-conf", + versionID: "ver-conf-next", + }).withVersion(catalogID, remoteVerID, map[string]filesapi.FileMeta{ + "app.py": {Hash: remoteHash, Size: 22}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // app.py must appear in the Conflicts list. + var conflicts []FileAction + + for _, fa := range plan.Conflicts { + if fa.Path == "app.py" { + conflicts = append(conflicts, fa) + } + } + + require.Len(t, conflicts, 1, "app.py must be in the Conflicts list") + assert.Equal(t, ActConflictCopy, conflicts[0].Action, + "both-sides-different must map to ActConflictCopy") + assert.Equal(t, ClsConflict, conflicts[0].Classification) +} + +// TestPlanAction_RemoteWinsResolution verifies that the remote-wins resolution +// path downloads the remote version. When a conflict is resolved (remote +// wins), the conflict path's manifest entry has the remote hash, not a +// streamed upload hash. This is tested through Execute with the fake. +func TestPlanAction_RemoteWinsResolution(t *testing.T) { + const ( + catalogID = "cid-rw" + versionID = "ver-rw" + remoteVerID = "ver-rw-remote" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + }, catalogID, versionID) + + // Modify the file locally. + modifyFile(t, dir, "app.py", "print('local-change')\n") + + // The server has a different version. The artifact's codeRef points at + // remoteVerID (different from the synced versionID) so the engine detects + // drift and fetches AllFiles for the remote version. + remoteContent := "print('remote-change')\n" + remoteHash := sha256Hex([]byte(remoteContent)) + + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-rw", + versionID: "ver-rw-next", + }).withVersion(catalogID, remoteVerID, map[string]filesapi.FileMeta{ + "app.py": {Hash: remoteHash, Size: int64(len(remoteContent))}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + // The fake's DownloadFile returns an error, so we cannot Execute a + // conflict resolution through the fake. Instead, verify the plan + // structure: the conflict is in the Conflicts list, and the conflict + // path's RemoteHash is the server's hash. Phase 6 (buildNewBaseManifest) + // uses fa.RemoteHash for conflict entries, which is the remote-wins + // resolution. + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVerID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + require.True(t, plan.HasConflicts(), "plan must have conflicts") + + var conflict FileAction + + for _, fa := range plan.Conflicts { + if fa.Path == "app.py" { + conflict = fa + } + } + + require.Equal(t, "app.py", conflict.Path) + assert.Equal(t, remoteHash, conflict.RemoteHash, + "conflict's RemoteHash must be the server's hash (remote-wins resolution uses this)") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-009: Sync lock, stale-rollback recovery, state migration +// --------------------------------------------------------------------------- + +// TestSyncLock_SecondConcurrentSyncRejected verifies that a second concurrent +// sync against the same project is rejected by the sync lock. On Unix, the +// second Plan() must fail with a lock error; on Windows the lock is a no-op +// (tracked in RAPTOR-16928) so the test skips. +func TestSyncLock_SecondConcurrentSyncRejected(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("v1 sync lock is a no-op on windows; tracked in RAPTOR-16928") + } + + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // First engine acquires the lock. + e1, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + plan, err := e1.Plan() + require.NoError(t, err) + require.False(t, plan.IsEmpty(), "fixture must produce a non-empty plan") + + t.Cleanup(func() { _ = e1.Close() }) + + // Second engine on the same project must fail to acquire the lock. + e2, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + _, err = e2.Plan() + require.Error(t, err, "second concurrent sync must be rejected by the lock") + assert.Contains(t, err.Error(), "another sync is already running") +} + +// TestSyncLock_StaleRollbackRecovery verifies that a stale rollback from a +// crashed prior run is restored so a new sync proceeds. Phase 0 calls +// RestoreStaleIfPresent before acquiring the lock, so a stale rollback +// directory must be cleaned up and the engine's StaleRollbackRestored() must +// report true. +func TestSyncLock_StaleRollbackRecovery(t *testing.T) { + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // Simulate a crashed prior run: create a stale rollback directory with + // a backup of app.py. + rollDir := wapi.RollbackDir(dir) + require.NoError(t, os.MkdirAll(rollDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(rollDir, "app.py"), []byte("intact\n"), 0o644)) + + // Clobber the working tree to simulate the crash mid-sync. + require.NoError(t, os.WriteFile(filepath.Join(dir, "app.py"), []byte("BROKEN\n"), 0o644)) + + e := lockfileEngine(t, dir, noLockfileRunner) + + plan, err := e.Plan() + require.NoError(t, err, "stale rollback must be recovered so Plan proceeds") + require.NotNil(t, plan) + + assert.True(t, e.StaleRollbackRestored(), "engine must report that a stale rollback was restored") + + // The file must be restored to its pre-crash content. + got, err := os.ReadFile(filepath.Join(dir, "app.py")) + require.NoError(t, err) + assert.Equal(t, "intact\n", string(got), "stale rollback must restore the original file") + + // The rollback directory must be cleaned up. + _, err = os.Stat(rollDir) + assert.ErrorIs(t, err, os.ErrNotExist, "stale rollback directory must be removed after recovery") +} + +// TestStateMigration_ForwardMigration verifies that an older on-disk state +// format (legacy .wapi/ directory) is migrated forward to .datarobot/workload/ +// without error. Phase 0 calls EnsureMigrated, which moves the legacy +// directory. The engine's StateMigrationNotice() must report the move. +func TestStateMigration_ForwardMigration(t *testing.T) { + // Initialize a proper project (creates .datarobot/workload/ with a valid + // config.json that has createdAt and cliVersion), then move the state + // directory to the legacy .wapi/ location to simulate an older on-disk + // state format. + dir := initProject(t, map[string]string{"app.py": "print('hi')\n"}) + + // Move the current state directory to the legacy location. + currentDir := filepath.Join(dir, wapi.RootDirName, wapi.StateDirName) + legacyDir := filepath.Join(dir, wapi.LegacyDirName) + require.NoError(t, os.Rename(currentDir, legacyDir)) + + // Remove the now-empty .datarobot/ parent so EnsureMigrated sees only the + // legacy directory. + require.NoError(t, os.RemoveAll(filepath.Join(dir, wapi.RootDirName))) + + // Verify the legacy directory exists before migration. + assert.DirExists(t, legacyDir) + + e, err := newWithDeps(dir, Options{}, Deps{ + Files: &fakeFilesClient{}, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, "", ""), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + _, err = e.Plan() + require.NoError(t, err, "migration must succeed so Plan proceeds") + + // The legacy directory must be gone. + assert.NoDirExists(t, legacyDir, "legacy state directory must be moved") + + // The current state directory must exist. + assert.DirExists(t, currentDir, "current state directory must exist after migration") + + // The engine must report the migration. + notice := e.StateMigrationNotice() + assert.Contains(t, notice, wapi.LegacyDirName, "notice must name the legacy directory") +} + +// --------------------------------------------------------------------------- +// VAL-REGRESSION-008: dr workload up code-change measurement +// --------------------------------------------------------------------------- + +// TestWorkloadUp_CodeChangeCount_ModifiedTree verifies that the sync engine's +// Plan (which `dr workload up`'s defaultCodeChange builds as a dry-run) reports +// the correct upload + delete count for a modified tree. defaultCodeChange +// computes `len(plan.Uploads) + len(plan.Deletes)`, so we verify that count +// matches the real number of changed files. +func TestWorkloadUp_CodeChangeCount_ModifiedTree(t *testing.T) { + const ( + catalogID = "cid-wup" + versionID = "ver-wup" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "utils.py": "def util(): pass\n", + "to-delete.py": "x\n", + }, catalogID, versionID) + + // Modify one file, add one new file, delete one file. + modifyFile(t, dir, "app.py", "print('changed')\n") + require.NoError(t, os.WriteFile(filepath.Join(dir, "new.py"), []byte("new\n"), 0o644)) + require.NoError(t, os.Remove(filepath.Join(dir, "to-delete.py"))) + + // Pre-populate the fake's server state with the synced version. + fake := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-wup", + versionID: "ver-wup-next", + }).withVersion(catalogID, versionID, map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("print('hi')\n")), Size: 11}, + "utils.py": {Hash: sha256Hex([]byte("def util(): pass\n")), Size: 17}, + "to-delete.py": {Hash: sha256Hex([]byte("x\n")), Size: 2}, + ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, + }) + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + // defaultCodeChange reports len(plan.Uploads) + len(plan.Deletes). + // Modified app.py + new new.py = 2 uploads. + // Deleted to-delete.py = 1 delete. + // Total = 3 changed files. + count := len(plan.Uploads) + len(plan.Deletes) + assert.Equal(t, 3, count, + "modified tree must report 3 changed files (2 uploads + 1 delete), got %d (uploads=%d, deletes=%d)", + count, len(plan.Uploads), len(plan.Deletes)) +} + +// TestWorkloadUp_CodeChangeCount_UnchangedTree verifies that the sync engine's +// Plan reports zero changed files for an unchanged tree (synced project with no +// modifications). defaultCodeChange reports `len(plan.Uploads) + len(plan.Deletes)`, +// which must be zero. +func TestWorkloadUp_CodeChangeCount_UnchangedTree(t *testing.T) { + const ( + catalogID = "cid-wup-uc" + versionID = "ver-wup-uc" + ) + + dir := syncedProject(t, map[string]string{ + "app.py": "print('hi')\n", + "utils.py": "def util(): pass\n", + }, catalogID, versionID) + + // No modifications — the tree is unchanged. + + fake := &fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-wup-uc", + versionID: "ver-wup-uc-next", + } + + e, err := newWithDeps(dir, Options{DryRun: true, Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, versionID), nil + }, + }, + Now: time.Now, + Lockfile: noLockfileRunner, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + plan, err := e.Plan() + require.NoError(t, err) + + count := len(plan.Uploads) + len(plan.Deletes) + assert.Equal(t, 0, count, + "unchanged tree must report 0 changed files, got %d (uploads=%d, deletes=%d)", + count, len(plan.Uploads), len(plan.Deletes)) + assert.True(t, plan.IsEmpty(), "unchanged tree must produce an empty plan") +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// deletePathsOf returns the paths in the delete portion of the plan. +func deletePathsOf(plan *SyncPlan) []string { + paths := make([]string, 0, len(plan.Deletes)) + for _, fa := range plan.Deletes { + paths = append(paths, fa.Path) + } + + return paths +} diff --git a/internal/workload/up/codechange_regression_test.go b/internal/workload/up/codechange_regression_test.go new file mode 100644 index 000000000..daa737697 --- /dev/null +++ b/internal/workload/up/codechange_regression_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package up + +import ( + "testing" + + "github.com/datarobot/cli/internal/workload/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests guard `dr workload up`'s code-change measurement so the +// upload-integrity fix cannot regress it silently. defaultCodeChange builds +// a dry-run sync engine and counts plan upload + delete rows; the first-deploy +// flag is set when the project is not linked. +// +// The modified-tree and unchanged-tree count tests live in the sync package +// (TestWorkloadUp_CodeChangeCount_*) because defaultCodeChange uses sync.New +// with production deps for linked projects, which cannot be tested without +// staging. The sync engine's Plan() is the measurement's core, and testing it +// with injected fakes verifies the same logic. +// +// Fulfills VAL-REGRESSION-008. + +// dockerfileManifestYAML is a minimal manifest with a Dockerfile build mode, +// which is what defaultCodeChange requires to measure code changes. +const dockerfileManifestYAML = `name: my-app +artifact: + name: my-app-artifact + spec: + type: service + containerGroups: + - name: default + containers: + - name: primary + primary: true + port: 8080 + imageBuildConfig: + dockerfile: + source: provided +runtime: + containerGroups: + - name: default + replicaCount: 1 + containers: + - name: primary + resources: + cpu: 0.5 + memory: 1Gi +` + +// TestDefaultCodeChange_FirstDeploy verifies that defaultCodeChange reports +// the first-deploy flag for an unlinked project. When wapi.Exists returns +// false (no .datarobot/workload or .wapi directory), the measurement returns +// CodeChange{Applies: true, FirstDeploy: true} without calling the sync +// engine — there is nothing to compare against, so every file is new. +func TestDefaultCodeChange_FirstDeploy(t *testing.T) { + dir := t.TempDir() + + // The project is not linked: no .datarobot/workload or .wapi directory. + m, err := manifest.Parse([]byte(dockerfileManifestYAML), "") + require.NoError(t, err) + + require.Equal(t, manifest.BuildModeDockerfile, m.BuildMode(), + "fixture must have a Dockerfile build mode for code change to apply") + + loaded := Loaded{ + ProjectDir: dir, + Manifest: m, + } + + change, err := defaultCodeChange(loaded, Live{}) + require.NoError(t, err) + + assert.True(t, change.Applies, "Dockerfile build mode means code change applies") + assert.True(t, change.FirstDeploy, + "unlinked project must report FirstDeploy=true (nothing to compare against)") + assert.Equal(t, 0, change.Files, + "first deploy has no diff count (every file is new, not changed)") +} + +// TestDefaultCodeChange_ImageModeDoesNotApply verifies that a manifest with +// an image build mode (not Dockerfile or Generated) does not report a code +// change. A published image has no code to sync. +func TestDefaultCodeChange_ImageModeDoesNotApply(t *testing.T) { + dir := t.TempDir() + + const imageManifestYAML = `name: my-app +artifact: + name: my-app-artifact + spec: + type: service + containerGroups: + - name: default + containers: + - name: primary + primary: true + port: 8080 + imageUri: registry.example.com/my-app:v1 +runtime: + containerGroups: + - name: default + replicaCount: 1 + containers: + - name: primary + resources: + cpu: 0.5 + memory: 1Gi +` + + m, err := manifest.Parse([]byte(imageManifestYAML), "") + require.NoError(t, err) + + assert.Equal(t, manifest.BuildModeImage, m.BuildMode(), + "fixture must have an image build mode") + + loaded := Loaded{ + ProjectDir: dir, + Manifest: m, + } + + change, err := defaultCodeChange(loaded, Live{}) + require.NoError(t, err) + + assert.False(t, change.Applies, "image build mode means no code change to measure") + assert.False(t, change.FirstDeploy, "non-Dockerfile mode must not set FirstDeploy") +} From 23b135c8d11786ddeded0a7597d7387cb13fe17a Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Thu, 27 Aug 2026 14:50:53 -0700 Subject: [PATCH 08/20] [RAPTOR-19525] fix(filesapi): send zip-path overwrite as a multipart form field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zip-path incremental sync (UploadFromZipExisting) sent overwrite=REPLACE as a URL query parameter, but the server's /files//fromFile/ route binds its validator fields from the multipart form body only and silently defaults to RENAME when the form field is absent. Every re-uploaded path was therefore stored as "name (2).ext" while the original path kept its stale bytes, and the manifest recorded the new streamed hash at the original path — so every later sync falsely reported "Up to date." while the server served old content. Move overwrite into the multipart form body. newStreamingMultipartRequest now frames extra form fields as complete parts BEFORE the file part (fields-first lets streaming parsers collect parameters without buffering the file), with field names sorted for deterministic framing; Content-Length stays exact because the fields fold into the prologue (verified byte-for-byte by test). useArchiveContents stays in the query string: the monorepo validator (MLData/files/public_api/files/validators.py, AddFilesIntoContainerBaseValidator) declares use_archive_contents as a form field whose default is 'True' — the server defaults to archive extraction and never reads request.args, so the query flag is ignored and extraction happens regardless. Moving it would change nothing observable; the test pins that decision. UploadFromZipNew is unchanged apart from the shared framing helper: a first sync has no pre-existing paths, so the server's RENAME default is correct and it sends no overwrite field. Verified end to end against staging: 25-file first sync (zip path), 22-file incremental re-sync (zip path) — all 22 re-uploaded paths hold the new bytes (disk shasum == manifest hash == server fileChecksum), zero "name (2).ext" entries, unmodified controls intact, and a following plain --dry-run truthfully prints "Up to date.". --- internal/drapi/filesapi/client_test.go | 25 ++- internal/drapi/filesapi/fromfile.go | 21 ++- internal/drapi/filesapi/fromfile_test.go | 203 +++++++++++++++++++++++ internal/drapi/filesapi/multipart.go | 42 ++++- internal/drapi/filesapi/stage.go | 2 +- 5 files changed, 276 insertions(+), 17 deletions(-) create mode 100644 internal/drapi/filesapi/fromfile_test.go diff --git a/internal/drapi/filesapi/client_test.go b/internal/drapi/filesapi/client_test.go index 151a8a497..11aca826f 100644 --- a/internal/drapi/filesapi/client_test.go +++ b/internal/drapi/filesapi/client_test.go @@ -364,25 +364,38 @@ func TestPollStatus_CompletedRedirect(t *testing.T) { assert.True(t, IsTerminalStatus(resp.Status)) } +// TestUploadFromZipExisting proves the REPLACE request rides in the +// multipart form body. The server's /files//fromFile/ route binds its +// validator fields from the parsed form only and silently defaults to +// RENAME when the overwrite field is absent — a query parameter is never +// read, and RENAME would store re-uploaded paths as "name (2).ext" while +// the original path keeps its stale bytes. Form fields must also precede +// the file part: streaming parsers collect fields as they arrive, before +// committing to a potentially huge file stream. func TestUploadFromZipExisting(t *testing.T) { startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/v2/files/cid-1/fromFile/", r.URL.Path) assert.Equal(t, "true", r.URL.Query().Get("useArchiveContents")) - assert.Equal(t, "REPLACE", r.URL.Query().Get("overwrite")) + assert.Empty(t, r.URL.Query().Get("overwrite"), + "overwrite in the URL query is silently ignored by the server") assert.Contains(t, r.Header.Get("Content-Type"), "multipart/form-data") - mr, err := r.MultipartReader() + parts, err := readMultipartParts(r) if !assert.NoError(t, err) { return } - part, err := mr.NextPart() - if !assert.NoError(t, err) { + if !assert.Len(t, parts, 2) { return } - assert.Equal(t, "file", part.FormName()) - assert.Equal(t, "changes.zip", part.FileName()) + assert.Equal(t, "overwrite", parts[0].Name, "form fields must precede the file part") + assert.Equal(t, "REPLACE", string(parts[0].Content)) + assert.Empty(t, parts[0].FileName) + + assert.Equal(t, "file", parts[1].Name) + assert.Equal(t, "changes.zip", parts[1].FileName) + assert.Equal(t, "PK\x03\x04fake-zip", string(parts[1].Content)) w.WriteHeader(http.StatusAccepted) _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) diff --git a/internal/drapi/filesapi/fromfile.go b/internal/drapi/filesapi/fromfile.go index e2f0d9a57..f2e040a93 100644 --- a/internal/drapi/filesapi/fromfile.go +++ b/internal/drapi/filesapi/fromfile.go @@ -33,7 +33,7 @@ func (c *httpClient) UploadFromZipNew(name string, size int64, body io.Reader) ( return nil, fmt.Errorf("build files url: %w", err) } - return uploadZipMultipart(requestURL, name, size, body) + return uploadZipMultipart(requestURL, nil, name, size, body) } func (c *httpClient) UploadFromZipExisting(catalogID, name, overwrite string, size int64, body io.Reader) (*FromFileResp, error) { @@ -43,18 +43,29 @@ func (c *httpClient) UploadFromZipExisting(catalogID, name, overwrite string, si q := url.Values{} q.Set("useArchiveContents", "true") - q.Set("overwrite", overwrite) requestURL, err := drapi.EndpointURL("/files/"+url.PathEscape(catalogID)+"/fromFile/", q) if err != nil { return nil, fmt.Errorf("build fromFile url: %w", err) } - return uploadZipMultipart(requestURL, name, size, body) + // overwrite must ride in the multipart form body: the server's + // /files//fromFile/ route binds its validator fields from the + // parsed form only, never from the query string, and silently defaults + // to RENAME when the field is absent. RENAME stores a re-uploaded path + // as "name (2).ext" while the original path keeps its stale bytes. + // useArchiveContents stays in the query: the server also ignores it + // there, but its declared form default is 'True' (archive extraction), + // so extraction happens either way and the request is unchanged + // apart from the overwrite fix. + fields := url.Values{} + fields.Set("overwrite", overwrite) + + return uploadZipMultipart(requestURL, fields, name, size, body) } -func uploadZipMultipart(requestURL, name string, size int64, body io.Reader) (*FromFileResp, error) { - req, err := newStreamingMultipartRequest(requestURL, nil, name, size, body) +func uploadZipMultipart(requestURL string, fields url.Values, name string, size int64, body io.Reader) (*FromFileResp, error) { + req, err := newStreamingMultipartRequest(requestURL, nil, fields, name, size, body) if err != nil { return nil, err } diff --git a/internal/drapi/filesapi/fromfile_test.go b/internal/drapi/filesapi/fromfile_test.go new file mode 100644 index 000000000..a284cad28 --- /dev/null +++ b/internal/drapi/filesapi/fromfile_test.go @@ -0,0 +1,203 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesapi + +import ( + "bytes" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// multipartPart is one decoded part of a multipart request body, in wire +// order. FileName is non-empty for file parts. +type multipartPart struct { + Name string + FileName string + Content []byte +} + +// decodeMultipart walks a multipart reader and returns its parts in wire +// order. Form fields written before the file part appear before it here, +// which is how tests pin the fields-first convention. +func decodeMultipart(mr *multipart.Reader) ([]multipartPart, error) { + var parts []multipartPart + + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + return parts, nil + } + + if err != nil { + return nil, err + } + + content, err := io.ReadAll(part) + if err != nil { + return nil, err + } + + parts = append(parts, multipartPart{ + Name: part.FormName(), + FileName: part.FileName(), + Content: content, + }) + } +} + +// readMultipartParts parses an UNCONSUMED multipart request body with +// mime/multipart and returns its parts in wire order. Tests assert on +// these decoded parts rather than raw bytes so the framing stays free to +// evolve. Reading r.Body first (e.g. for a Content-Length check) exhausts +// the stream — parse the buffered copy via multipart.NewReader instead. +func readMultipartParts(r *http.Request) ([]multipartPart, error) { + mr, err := r.MultipartReader() + if err != nil { + return nil, err + } + + return decodeMultipart(mr) +} + +// TestUploadFromZipExisting_UseArchiveContentsStaysInQuery pins the +// placement of useArchiveContents. The monorepo fromFile validator +// declares it as a multipart form field whose default is 'True' (archive +// extraction), and the route binds validator fields from the parsed form +// only — request.args is never consulted. Extraction therefore happens via +// the server default regardless of what the CLI sends, so leaving the +// flag in the query string (where it is ignored) changes nothing +// observable. This test documents that decision so any future move into +// the form is a conscious one. +func TestUploadFromZipExisting_UseArchiveContentsStaysInQuery(t *testing.T) { + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "true", r.URL.Query().Get("useArchiveContents")) + + parts, err := readMultipartParts(r) + if !assert.NoError(t, err) { + return + } + + for _, part := range parts { + assert.NotEqual(t, "useArchiveContents", part.Name, + "useArchiveContents is intentionally left out of the form body") + } + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte("PK\x03\x04fake-zip")) + _, err := c.UploadFromZipExisting("cid-1", "changes.zip", OverwriteReplace, int64(body.Len()), body) + require.NoError(t, err) +} + +// TestUploadFromZipNew_NoOverwriteField guards the shared-framing parity: +// a first sync has no pre-existing paths, so it must send no overwrite +// form field and no overwrite query parameter — the server's RENAME +// default is correct there and the request should stay minimal. +func TestUploadFromZipNew_NoOverwriteField(t *testing.T) { + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.URL.Query().Get("overwrite")) + + parts, err := readMultipartParts(r) + if !assert.NoError(t, err) { + return + } + + for _, part := range parts { + assert.NotEqual(t, "overwrite", part.Name) + } + + if !assert.Len(t, parts, 1) { + return + } + + assert.Equal(t, "file", parts[0].Name) + assert.Equal(t, "wapi-sync.zip", parts[0].FileName) + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"new-cid","catalogVersionId":"new-ver","statusId":"sid-new"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte("PK\x03\x04fake-zip")) + resp, err := c.UploadFromZipNew("wapi-sync.zip", int64(body.Len()), body) + require.NoError(t, err) + assert.Equal(t, "new-ver", resp.CatalogVersionID) +} + +// TestUploadFromZipExisting_ContentLengthMatchesBodyWithFormFields +// verifies the Content-Length accounting survives folding form fields +// into the prologue: the advertised length must equal the received body +// exactly, and the streamed file must still decode intact. An off-by-N +// here surfaces as a transport error or a truncated multipart stream, +// never as a silent pass. +func TestUploadFromZipExisting_ContentLengthMatchesBodyWithFormFields(t *testing.T) { + // Long enough to cross io.Copy's internal buffer more than once. + payload := strings.Repeat("0123456789abcdef", 4096) + + startServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, err := io.ReadAll(r.Body) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, r.ContentLength, int64(len(raw)), + "advertised Content-Length must match the received body byte-for-byte") + + // The body is consumed by the ReadAll above, so parse the buffered + // copy rather than asking the request for a fresh MultipartReader. + _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if !assert.NoError(t, err) { + return + } + + parts, err := decodeMultipart(multipart.NewReader(bytes.NewReader(raw), params["boundary"])) + if !assert.NoError(t, err) { + return + } + + if !assert.Len(t, parts, 2) { + return + } + + assert.Equal(t, "REPLACE", string(parts[0].Content)) + assert.Equal(t, payload, string(parts[1].Content)) + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"catalogId":"cid-1","catalogVersionId":"v9","statusId":"sid-9"}`)) + })) + + c := New() + + body := bytes.NewReader([]byte(payload)) + _, err := c.UploadFromZipExisting("cid-1", "changes.zip", OverwriteReplace, int64(body.Len()), body) + require.NoError(t, err) +} + +// The package's mime/multipart types stay referenced even if helper +// implementations drift; mirrors the guard at the bottom of client_test.go. +var _ = multipart.ErrMessageTooLarge diff --git a/internal/drapi/filesapi/multipart.go b/internal/drapi/filesapi/multipart.go index 904a1b098..b9b27a10f 100644 --- a/internal/drapi/filesapi/multipart.go +++ b/internal/drapi/filesapi/multipart.go @@ -22,6 +22,7 @@ import ( "net/http" "net/textproto" "net/url" + "sort" "github.com/datarobot/cli/internal/drapi" ) @@ -34,6 +35,12 @@ const multipartFormField = "file" // by the pipe (one chunk in flight) plus the small envelope, regardless // of file size — important because the engine may upload multi-GiB zips. // +// Fields ride in the multipart body, not the URL query: some server +// routes (fromFile) bind their validator fields from the parsed form +// only and silently ignore query parameters. Fields are framed as +// complete parts BEFORE the file part so a streaming parser collects +// them without buffering the file. +// // Trade-off: the request has no GetBody, so http.Transport cannot // transparently retry the body on connection reset. Callers needing // retry must redo the call from scratch (re-opening the source if it @@ -41,6 +48,7 @@ const multipartFormField = "file" func newStreamingMultipartRequest( requestURL string, query url.Values, + fields url.Values, filename string, size int64, body io.Reader, @@ -49,7 +57,7 @@ func newStreamingMultipartRequest( requestURL += "?" + query.Encode() } - contentType, prologue, epilogue, err := multipartFraming(filename) + contentType, prologue, epilogue, err := multipartFraming(fields, filename) if err != nil { return nil, err } @@ -65,6 +73,9 @@ func newStreamingMultipartRequest( return nil, fmt.Errorf("build multipart request: %w", err) } + // ContentLength stays exact because the form fields are folded into + // the prologue; the file bytes still contribute exactly size, and the + // epilogue is unchanged. if size >= 0 { req.ContentLength = int64(len(prologue)) + size + int64(len(epilogue)) } @@ -80,14 +91,35 @@ func newStreamingMultipartRequest( return req, nil } -// multipartFraming returns the prologue and epilogue around a single -// file part. Going through multipart.Writer keeps the framing -// RFC-2046-correct even though we stream the body separately. -func multipartFraming(filename string) (string, []byte, []byte, error) { +// multipartFraming returns the prologue and epilogue around the streamed +// file part, with any extra form fields framed as complete parts first. +// Fields must precede the file part: streaming parsers read form fields +// as they arrive, so a server can collect its parameters before +// committing to an arbitrarily large file stream. Field names are sorted +// so the framing is deterministic. Going through multipart.Writer keeps +// the framing RFC-2046-correct even though we stream the file content +// separately. +func multipartFraming(fields url.Values, filename string) (string, []byte, []byte, error) { var head bytes.Buffer w := multipart.NewWriter(&head) + names := make([]string, 0, len(fields)) + + for name := range fields { + names = append(names, name) + } + + sort.Strings(names) + + for _, name := range names { + for _, value := range fields[name] { + if err := w.WriteField(name, value); err != nil { + return "", nil, nil, fmt.Errorf("write multipart field %s: %w", name, err) + } + } + } + hdr := make(textproto.MIMEHeader) hdr.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, multipartFormField, filename)) hdr.Set("Content-Type", "application/octet-stream") diff --git a/internal/drapi/filesapi/stage.go b/internal/drapi/filesapi/stage.go index 677abe05c..3096f68e9 100644 --- a/internal/drapi/filesapi/stage.go +++ b/internal/drapi/filesapi/stage.go @@ -60,7 +60,7 @@ func (c *httpClient) UploadToStage(catalogID, stageID, name string, size int64, return fmt.Errorf("build upload url: %w", err) } - req, err := newStreamingMultipartRequest(requestURL, nil, name, size, body) + req, err := newStreamingMultipartRequest(requestURL, nil, nil, name, size, body) if err != nil { return err } From c5ce5ec097721983f91ec2c6fe8acd391627f4d9 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Fri, 28 Aug 2026 23:07:59 -0700 Subject: [PATCH 09/20] [RAPTOR-19525] fix(workload): repair the mangled license headers on three regression test files The Copyrights CI check flagged surroundings_regression_test.go, upload_failure_test.go, and codechange_regression_test.go. Each carried a header that was silently corrupted during an edit: two had the license URL truncated to licenses/2.0 and one had dropped "in" from "except in compliance with the License". All three now byte-match the canonical header used by their sibling test files, and license-eye reports 0 invalid files. --- internal/workload/sync/surroundings_regression_test.go | 2 +- internal/workload/sync/upload_failure_test.go | 2 +- internal/workload/up/codechange_regression_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go index aab52aa00..19e109b90 100644 --- a/internal/workload/sync/surroundings_regression_test.go +++ b/internal/workload/sync/surroundings_regression_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/internal/workload/sync/upload_failure_test.go b/internal/workload/sync/upload_failure_test.go index 45bacccba..b9d5eec8d 100644 --- a/internal/workload/sync/upload_failure_test.go +++ b/internal/workload/sync/upload_failure_test.go @@ -1,7 +1,7 @@ // Copyright 2026 DataRobot, Inc. and its affiliates. // // Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except compliance with the License. +// you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 diff --git a/internal/workload/up/codechange_regression_test.go b/internal/workload/up/codechange_regression_test.go index daa737697..5d9663e18 100644 --- a/internal/workload/up/codechange_regression_test.go +++ b/internal/workload/up/codechange_regression_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, From 05c31434b08dc9c6f9a4857b6feb7268466e9a6f Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Fri, 28 Aug 2026 23:17:42 -0700 Subject: [PATCH 10/20] [RAPTOR-19525] fix(workload): discard the sync rollback at Phase 6 entry, before any state write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit phase6State assigned its rollback.Discard() to the very bottom, after SaveManifest and SaveConfig. A SaveConfig failure returned early and stranded the rollback dir holding pre-sync bytes. The next run's Phase 0 (RestoreStaleIfPresent) blindly copied those bytes back into the working tree; against the manifest the failed run had already advanced, the diff classified them as local edits, and the sync silently re-uploaded them over the remote — a self-perpetuating corruption chain with no prompt. Discarding at Phase 6 entry is safe: e.rollback is only assigned after executePlan succeeds (a Phase 5 failure restores and returns), Phase 6 never restores because the remote has already advanced, and discarding unconditionally makes cleanup independent of state-write success. Regression tests cover all three Phase 6 outcomes (clean success, SaveManifest failure, SaveConfig failure) plus the two-run corruption chain: fault-injected SaveConfig failure leaves no rollback dir, and the next run performs no stale restore and produces zero false LOCAL_MODIFIED uploads for rollback-covered paths. The mid-Phase-5 interruption behavior is unchanged (rollback dir still strands and the next run still restores). The tests were demonstrated to fail against the pre-fix ordering. The fake files client gains opt-in downloadable content so engine tests can exercise a plan with downloads, which is what puts real pre-sync bytes into the rollback tree. --- .../workload/sync/fake_files_client_test.go | 44 ++- internal/workload/sync/phase5_execute.go | 9 +- internal/workload/sync/phase6_state.go | 64 ++- .../workload/sync/rollback_discard_test.go | 364 ++++++++++++++++++ 4 files changed, 457 insertions(+), 24 deletions(-) create mode 100644 internal/workload/sync/rollback_discard_test.go diff --git a/internal/workload/sync/fake_files_client_test.go b/internal/workload/sync/fake_files_client_test.go index 565476228..e4a0b78f8 100644 --- a/internal/workload/sync/fake_files_client_test.go +++ b/internal/workload/sync/fake_files_client_test.go @@ -83,6 +83,11 @@ type fakeFilesClient struct { // follows next links, mirroring the real client's pagination behavior. allFilesPageSize int + // Downloadable content: versionID → (path → bytes served by DownloadFile). + // Opt-in: a test that never registers content gets the loud "not + // expected" failure, so unexpected download calls stay visible. + downloadContent map[string]map[string][]byte + // Fault-injection hooks (all opt-in; zero values are no-ops). dropPathFromApply string // omit this path from the resulting version after apply wrongChecksumPath string // return wrong checksum for this path in AllFiles @@ -114,6 +119,23 @@ func (f *fakeFilesClient) withVersion(catalogID, versionID string, files map[str return f } +// withDownloadable registers the content bytes DownloadFile serves for a +// version. The registered content must hash to the FileMeta the same test +// put in the version state — the download path verifies the streamed bytes +// against the plan's RemoteHash, exactly like the real server relationship. +func (f *fakeFilesClient) withDownloadable(versionID string, files map[string][]byte) *fakeFilesClient { + f.mu.Lock() + defer f.mu.Unlock() + + if f.downloadContent == nil { + f.downloadContent = make(map[string]map[string][]byte) + } + + f.downloadContent[versionID] = files + + return f +} + // withPageSize sets the AllFiles page size for pagination simulation. func (f *fakeFilesClient) withPageSize(n int) *fakeFilesClient { f.mu.Lock() @@ -534,8 +556,26 @@ func (f *fakeFilesClient) AllFiles(_, versionID string) (map[string]filesapi.Fil return result, nil } -func (f *fakeFilesClient) DownloadFile(_, _, _ string, _ io.Writer) (string, int64, error) { - return "", 0, errors.New("fakeFilesClient: DownloadFile not expected") +// DownloadFile serves content registered via withDownloadable. Without +// registered content it fails loudly, so a download the test did not plan +// for surfaces as an error instead of a silent empty file. +func (f *fakeFilesClient) DownloadFile(_, versionID, path string, w io.Writer) (string, int64, error) { + f.mu.Lock() + defer f.mu.Unlock() + + files, ok := f.downloadContent[versionID] + if !ok { + return "", 0, fmt.Errorf("fakeFilesClient: no downloadable content registered for version %s", versionID) + } + + data, ok := files[path] + if !ok { + return "", 0, fmt.Errorf("fakeFilesClient: no content registered for %s in version %s", path, versionID) + } + + n, err := w.Write(data) + + return path, int64(n), err } func (f *fakeFilesClient) DeleteFiles(catalogID string, paths []string) (*filesapi.DeleteFilesResp, error) { diff --git a/internal/workload/sync/phase5_execute.go b/internal/workload/sync/phase5_execute.go index b535c4076..667bd8a5c 100644 --- a/internal/workload/sync/phase5_execute.go +++ b/internal/workload/sync/phase5_execute.go @@ -68,8 +68,13 @@ func phase5Execute(e *Engine) error { return err } - // Phase 6 discards the rollback only after SaveConfig + SaveManifest - // succeed. + // Hand the rollback to Phase 6, which discards it at entry — before its + // first state write, not after its last. Once the plan has executed + // successfully the backup tree protects nothing, and discarding + // unconditionally keeps a Phase 6 write failure from stranding the dir + // for the next run's stale-restore to resurrect pre-sync bytes from. + // A Phase 5 failure above restores and returns, leaving the dir in place + // for exactly that stale-restore recovery. e.rollback = rb return nil diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index 487dcd5db..d5ca5cab7 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -21,10 +21,27 @@ import ( "github.com/datarobot/cli/internal/workload/wapi" ) -// phase6State writes the new BASE manifest, config, history entry, and -// discards the rollback. Failures here do NOT roll back Phase 5 since -// the remote has already advanced; the next sync will reconcile. +// phase6State writes the new BASE manifest, config, and history entry, and +// discards the rollback at entry. Failures here do NOT roll back Phase 5 +// since the remote has already advanced; the next sync will reconcile. func phase6State(e *Engine) error { + // Discard the rollback BEFORE any state write, unconditionally. When this + // runs, Phase 5 executed the whole plan successfully (e.rollback is only + // assigned after executePlan returns nil; a Phase 5 failure restores and + // returns without reaching here), so the backup tree has no remaining + // purpose — and Phase 6 never restores on failure because the remote has + // already advanced. Discarding first makes the cleanup independent of + // write success: if SaveManifest or SaveConfig below fails, the early + // return must not strand the rollback dir, because the next run's + // stale-rollback recovery would blindly copy the pre-sync bytes back + // into the working tree. Against the manifest this run just wrote, those + // resurrected bytes look like local edits and are silently re-uploaded + // over the remote. + if e.rollback != nil { + _ = e.rollback.Discard() + e.rollback = nil + } + if e.plan == nil { return nil } @@ -48,19 +65,31 @@ func phase6State(e *Engine) error { cfg.LastSyncedVersionID = &versionForState } - // Build and write the manifest BEFORE writing config. The failure - // direction is what matters, not elegance: if SaveManifest fails, - // config has not been advanced yet, so the next sync sees the old - // version in config, detects drift, fetches AllFiles, and rebuilds - // BASE from real remote data — safe and self-healing. The converse - // (config advanced, manifest stale) silently poisons BASE: the next - // sync sees no drift, fast-paths, copies the stale BASE to REMOTE, - // and reports "Up to date." forever. + // Build and write the manifest BEFORE writing config. Both orders leave + // a one-file window on failure, and the safe direction is the one where + // the next sync detects drift and rebuilds from real remote data: + // + // - SaveManifest fails: config has not been advanced yet, so the next + // sync sees the old version in config, detects drift, fetches + // AllFiles, and rebuilds BASE from the remote — safe and + // self-healing. The manifest write is retried by that same sync. + // + // - SaveConfig fails: the manifest is already advanced while config + // still names the old version. The version mismatch makes the next + // sync detect drift and fetch the remote; BASE (the advanced + // manifest) truthfully describes that remote, so the plan is empty + // and the run merely converges config. The rollback dir is already + // gone by then — discarded at entry above — so no stale-restore can + // resurrect pre-sync bytes as false local edits. // - // This reorder is data-safe: nothing between the two writes reads - // config from disk. buildNewBaseManifest reads only e.remote, - // e.plan, and e.uploadOutcome (all in-memory). e.config and - // populateResult are touched only after both writes complete. + // The converse (config advanced, manifest stale) is the poisonous + // direction: the next sync sees no drift, fast-paths, copies the stale + // BASE to REMOTE, and reports "Up to date." forever. + // + // This is data-safe because nothing between the two writes reads config + // from disk. buildNewBaseManifest reads only e.remote, e.plan, and + // e.uploadOutcome (all in-memory). e.config and populateResult are + // touched only after both writes complete. manifest, err := buildNewBaseManifest(e, versionForState, now) if err != nil { return fmt.Errorf("build manifest: %w", err) @@ -78,11 +107,6 @@ func phase6State(e *Engine) error { return fmt.Errorf("append history: %w", err) } - if e.rollback != nil { - _ = e.rollback.Discard() - e.rollback = nil - } - e.config = cfg e.populateResult(versionForState) diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go new file mode 100644 index 000000000..5228009b1 --- /dev/null +++ b/internal/workload/sync/rollback_discard_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sync + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/workload" + "github.com/datarobot/cli/internal/workload/ignore" + "github.com/datarobot/cli/internal/workload/wapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The tests in this file pin the Phase 6 discard-first ordering: the rollback +// directory must be discarded at Phase 6 ENTRY, before any state write, not +// after the last one succeeds. +// +// The hazard this guards: e.rollback is assigned only after Phase 5 executed +// the whole plan successfully, but a Phase 6 failure (SaveManifest or +// SaveConfig) returns early. If the discard sits below the state writes, a +// write failure strands the rollback dir. The next run's Phase 0 +// (RestoreStaleIfPresent) then blindly copies the pre-sync bytes back into +// the working tree, the advanced manifest makes the diff classify them as +// local edits, and the sync re-uploads the stale bytes over the remote — +// a self-perpetuating corruption chain with no prompt and no error. + +// rollbackDiscardScenario builds a synced project whose next sync carries one +// upload (a.py, locally modified) and one download (b.py, remotely modified). +// The download is the point: downloads are backed up into the rollback tree, +// so the rollback covers b.py's pre-sync bytes and a stranded rollback dir +// has real bytes to resurrect. An uploads-only plan would strand an empty +// rollback dir and could not demonstrate the corruption. +// +// Server timeline: ver-synced (what the project was synced at) → ver-remote +// (b.py changed remotely, drift) → ver-new (created by this sync's apply, +// which uploads the local a.py on top of ver-remote). +type rollbackDiscardScenario struct { + dir string + fake *fakeFilesClient + engine *Engine + plan *SyncPlan + origA string // a.py content at sync time + newA string // a.py content after the local edit + origB string // b.py content at sync time + remoteB string // b.py content after the remote edit + rollDir string + cfgBytes []byte // config.json bytes captured before the run +} + +// setupRollbackDiscardScenario wires the scenario and returns it after Plan(), +// so the caller can inject a Phase 6 fault and then Execute. +func setupRollbackDiscardScenario(t *testing.T) *rollbackDiscardScenario { + t.Helper() + + const ( + catalogID = "cid-synced" + oldVersion = "ver-synced" + remoteVer = "ver-remote" + newVersion = "ver-new" + ) + + s := &rollbackDiscardScenario{ + origA: "aaa\n", + newA: "AAAA\n", + origB: "bbb\n", + remoteB: "BBBB-remote\n", + } + + s.dir = syncedProject(t, map[string]string{ + "a.py": s.origA, + "b.py": s.origB, + }, catalogID, oldVersion) + + // Pending local change: a.py becomes an upload. + modifyFile(t, s.dir, "a.py", s.newA) + + // The remote moved ahead: ver-remote holds the edited b.py. ApplyStage + // merges the staged a.py into this version, producing ver-new. + drHash, drSize, err := hashLocal(t, s.dir, ignore.FileName) + require.NoError(t, err) + + s.fake = (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-1", + versionID: newVersion, + }).withVersion(catalogID, remoteVer, map[string]filesapi.FileMeta{ + "a.py": {Hash: sha256Hex([]byte(s.origA)), Size: int64(len(s.origA))}, + "b.py": {Hash: sha256Hex([]byte(s.remoteB)), Size: int64(len(s.remoteB))}, + ignore.FileName: {Hash: drHash, Size: drSize}, + }).withDownloadable(remoteVer, map[string][]byte{ + "a.py": []byte(s.origA), + "b.py": []byte(s.remoteB), + }) + + // The artifact's codeRef still points at ver-remote, so Phase 1 detects + // drift against the stale config and fetches real remote state. + e, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: s.fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, remoteVer), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e.Close() }) + + s.engine = e + s.rollDir = wapi.RollbackDir(s.dir) + + cfg, err := os.ReadFile(wapi.ConfigPath(s.dir)) + require.NoError(t, err) + + s.cfgBytes = cfg + + s.plan, err = e.Plan() + require.NoError(t, err) + + require.Len(t, s.plan.Uploads, 1, "a.py must be the one upload") + require.Equal(t, "a.py", s.plan.Uploads[0].Path) + require.Len(t, s.plan.Downloads, 1, "b.py must be the one download") + require.Equal(t, "b.py", s.plan.Downloads[0].Path) + + return s +} + +// replaceFileWithDir removes a state file and recreates it as a directory, so +// AtomicWriteFile's rename fails when the engine next writes it. Phase 1 has +// already loaded the real file, so the fault hits only Phase 6. +func replaceFileWithDir(t *testing.T, path string) { + t.Helper() + + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o755)) + + t.Cleanup(func() { _ = os.RemoveAll(path) }) +} + +// assertNoRollbackDir asserts the rollback location is empty — the property +// a Phase 6 outcome must never violate. Non-fatal so a red run reports the +// full downstream corruption (stale restore, false uploads) alongside it. +func assertNoRollbackDir(t *testing.T, rollDir string) { + t.Helper() + + _, err := os.Stat(rollDir) + assert.True(t, os.IsNotExist(err), + "rollback directory must not survive the Phase 6 outcome at %s", rollDir) +} + +// TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads is the +// main regression: SaveConfig fails AFTER SaveManifest advanced the manifest. +// The rollback dir must already be gone (discarded at Phase 6 entry), and the +// next run must reconcile from the remote without resurrecting b.py's +// pre-sync bytes as a false LOCAL_MODIFIED upload. +// +// Fulfills VAL-ROLLBACK-001 (SaveConfig-failure leg) and VAL-ROLLBACK-002. +func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testing.T) { + const ( + catalogID = "cid-synced" + oldVersion = "ver-synced" + newVersion = "ver-new" + ) + + s := setupRollbackDiscardScenario(t) + + // Fault: SaveConfig's atomic write fails (config.json is now a + // directory). SaveManifest has already run by the time the write is + // attempted, so the manifest advances while config stays stale. + replaceFileWithDir(t, wapi.ConfigPath(s.dir)) + + _, err := s.engine.Execute(s.plan) + + require.Error(t, err, "phase 6 must fail when SaveConfig fails") + assert.Contains(t, err.Error(), "save config", + "error must come from SaveConfig, not an earlier step") + + // The manifest DID advance (SaveManifest ran first): it records ver-new + // with the streamed upload hash for a.py and the remote hash for b.py. + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + require.NotNil(t, manifest.SyncedVersionID) + assert.Equal(t, newVersion, *manifest.SyncedVersionID, + "manifest must be advanced past the failed SaveConfig") + assert.Equal(t, sha256Hex([]byte(s.newA)), manifest.Files["a.py"].Hash, + "manifest must record the uploaded a.py hash") + assert.Equal(t, sha256Hex([]byte(s.remoteB)), manifest.Files["b.py"].Hash, + "manifest must record the downloaded b.py hash") + + // The rollback dir must be GONE. Discarding only after the state writes + // strands it here, and the stranded dir is what resurrects stale bytes + // on the next run. + assertNoRollbackDir(t, s.rollDir) + + // The working tree keeps the synced bytes: a.py holds the local edit, + // b.py holds the downloaded remote content. Nothing may restore over it. + bOnDisk, err := os.ReadFile(filepath.Join(s.dir, "b.py")) + require.NoError(t, err) + + assert.Equal(t, s.remoteB, string(bOnDisk), + "b.py must keep the downloaded remote content — no pre-sync restore") + + // Restore config.json with its pre-sync bytes: in the real world the + // atomic write never landed, so the file keeps its old content. + cPath := wapi.ConfigPath(s.dir) + + require.NoError(t, os.RemoveAll(cPath)) + + //nolint:gosec // G703: cPath is the project's own config.json under t.TempDir(); the bytes were read from that same file before the fault + require.NoError(t, os.WriteFile(cPath, s.cfgBytes, 0o644)) + + staleCfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, staleCfg.LastSyncedVersionID) + assert.Equal(t, oldVersion, *staleCfg.LastSyncedVersionID, + "config.json must still hold the pre-sync version — the drift the next run must detect") + + // --- Next run: must reconcile from the remote, not from stale bytes --- + + all, err := s.fake.AllFiles(catalogID, newVersion) + require.NoError(t, err) + + fake2 := (&fakeFilesClient{ + catalogID: catalogID, + stageID: "stage-2", + versionID: newVersion, + }).withVersion(catalogID, newVersion, all) + + // The codeRef now points at ver-new: run 1's PatchCodeRef succeeded + // before Phase 6 failed, so the artifact genuinely moved ahead. + e2, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: fake2, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVersion), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e2.Close() }) + + plan2, err := e2.Plan() + require.NoError(t, err) + + // No stale restore: with the rollback discarded at entry, Phase 0 has + // nothing to resurrect. Pre-fix, this is true AND b.py has been rolled + // back to its pre-sync bytes. + assert.False(t, e2.StaleRollbackRestored(), + "the next run must not restore a stale rollback — the rollback dir was discarded at Phase 6 entry") + + // The plan must not contain false LOCAL_MODIFIED / LOCAL_ADDED uploads + // for the rollback-covered paths. b.py is the covered path here: its + // manifest entry (advanced) matches the remote, so there is nothing to + // upload. Pre-fix, the resurrected pre-sync bytes are classified as + // LOCAL_MODIFIED and silently re-uploaded over the remote. + assert.Empty(t, plan2.Uploads, + "the next run must not upload anything for rollback-covered paths — got %v", plan2.Uploads) + assert.True(t, plan2.IsEmpty(), + "the next run's plan must reconcile from the remote and be empty") + + // Executing the (empty) plan runs Phase 6, which converges config to the + // version the manifest already records — the self-healing direction. + _, err = e2.Execute(plan2) + require.NoError(t, err) + + convergedCfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, convergedCfg.LastSyncedVersionID) + assert.Equal(t, newVersion, *convergedCfg.LastSyncedVersionID, + "config must converge to the manifest's version on the next run") + + // Three-way consistency after convergence: manifest hash == server + // checksum for every file. + convergedManifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + serverFiles, err := fake2.AllFiles(catalogID, newVersion) + require.NoError(t, err) + + for path, fm := range convergedManifest.Files { + serverFM, ok := serverFiles[path] + require.True(t, ok, "server must hold %s after convergence", path) + + assert.Equal(t, serverFM.Hash, fm.Hash, + "manifest hash for %s must match the server after convergence", path) + } +} + +// TestPhase6SaveManifestFailure_DiscardsRollback pins the other write-failure +// leg of VAL-ROLLBACK-001: SaveManifest itself fails. The discard-first +// ordering has already removed the rollback dir by then, so no Phase 6 +// outcome — not even the earliest write failing — strands it. +func TestPhase6SaveManifestFailure_DiscardsRollback(t *testing.T) { + s := setupRollbackDiscardScenario(t) + + // Fault: SaveManifest's atomic write fails (manifest.json is now a + // directory). Neither state file advances past this point. + replaceFileWithDir(t, filepath.Join(wapi.Dir(s.dir), "manifest.json")) + + _, err := s.engine.Execute(s.plan) + + require.Error(t, err, "phase 6 must fail when SaveManifest fails") + assert.Contains(t, err.Error(), "save manifest", + "error must come from SaveManifest, not an earlier step") + + // Discard-first means the rollback dir is already gone even though the + // very first state write failed. + assertNoRollbackDir(t, s.rollDir) + + // config.json must not have advanced — SaveConfig runs after the failed + // SaveManifest. + cfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *cfg.LastSyncedVersionID, + "config must not advance when SaveManifest fails") +} + +// TestPhase6CleanSuccessDiscardsRollback pins the no-failure leg of +// VAL-ROLLBACK-001 for completeness: a fully successful Phase 6 also leaves +// no rollback dir behind. The interruption tests already assert this for an +// uploads-only plan; this variant covers a plan that also downloads. +func TestPhase6CleanSuccessDiscardsRollback(t *testing.T) { + s := setupRollbackDiscardScenario(t) + + result, err := s.engine.Execute(s.plan) + require.NoError(t, err) + require.NotNil(t, result) + + assertNoRollbackDir(t, s.rollDir) + + // The synced state must reflect the applied plan. + manifest, err := wapi.LoadManifest(s.dir) + require.NoError(t, err) + + assert.Equal(t, sha256Hex([]byte(s.newA)), manifest.Files["a.py"].Hash) + assert.Equal(t, sha256Hex([]byte(s.remoteB)), manifest.Files["b.py"].Hash) +} From 8d45e294578c49b821a9dccda0f8527aed77ee66 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Fri, 28 Aug 2026 23:19:31 -0700 Subject: [PATCH 11/20] [RAPTOR-19525] docs(test): correct the DownloadFile fake comment to name the specific error --- internal/workload/sync/surroundings_regression_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go index 19e109b90..cd46bb41d 100644 --- a/internal/workload/sync/surroundings_regression_test.go +++ b/internal/workload/sync/surroundings_regression_test.go @@ -663,7 +663,9 @@ func TestPlanAction_RemoteWinsResolution(t *testing.T) { ".drignore": {Hash: sha256Hex([]byte("")), Size: 0}, }) - // The fake's DownloadFile returns an error, so we cannot Execute a + // The fake's DownloadFile fails with a specific + // "no downloadable content registered" error unless the test registers + // downloadable content via withDownloadable, so we cannot Execute a // conflict resolution through the fake. Instead, verify the plan // structure: the conflict is in the Conflicts list, and the conflict // path's RemoteHash is the server's hash. Phase 6 (buildNewBaseManifest) From 341a4272ea3cb916aa7e6d8b044eb7a90ba4fb04 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Sat, 29 Aug 2026 00:34:58 -0700 Subject: [PATCH 12/20] [RAPTOR-19525] fix(workload): abort Phase 6 when the rollback Discard fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 6 discarded the rollback at entry but swallowed the Discard error (`_ = e.rollback.Discard()`). If RemoveAll itself fails, the rollback dir survives next to ADVANCED state, and the next run's stale-rollback recovery resurrects pre-sync bytes that the advanced manifest misclassifies as local edits and silently re-uploads over the remote — the strand/stale-restore/ false-upload chain, reopened through the Discard-failure path. Surface the error instead. At Phase 6 entry nothing has been persisted, so a wrapped error is the safe direction: the next run keeps the rollback dir AND un-advanced state, which is the recoverable mid-Phase-5 outcome — the stale restore puts back bytes the un-advanced manifest still matches, and the next diff schedules downloads, not false uploads. The discard is extracted into discardRollback so the phase's own failure contract stays explicit. Fault-injection test: a hook on Phase 5's final step (PatchCodeRef) strips write permission from the rollback dir between rollback creation and Phase 6, proving a Discard failure aborts with the wrapped error before SaveManifest — manifest byte-identical, config un-advanced, rollback dir still present. Also drop the now-unused //nolint:gosec on the config-restore write: gosec's taint analysis no longer flags that site once the package gains the new test, and nolintlint rejects directives that suppress nothing. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/phase6_state.go | 33 ++++++- .../workload/sync/rollback_discard_test.go | 89 ++++++++++++++++++- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index d5ca5cab7..cd6fb325f 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -23,7 +23,10 @@ import ( // phase6State writes the new BASE manifest, config, and history entry, and // discards the rollback at entry. Failures here do NOT roll back Phase 5 -// since the remote has already advanced; the next sync will reconcile. +// since the remote has already advanced; the next sync will reconcile. A +// Discard failure is the one entry failure that leaves the rollback dir +// behind, and it aborts before any state write so that leftover dir pairs +// with un-advanced state — the safe mid-Phase-5 shape (see below). func phase6State(e *Engine) error { // Discard the rollback BEFORE any state write, unconditionally. When this // runs, Phase 5 executed the whole plan successfully (e.rollback is only @@ -37,9 +40,8 @@ func phase6State(e *Engine) error { // into the working tree. Against the manifest this run just wrote, those // resurrected bytes look like local edits and are silently re-uploaded // over the remote. - if e.rollback != nil { - _ = e.rollback.Discard() - e.rollback = nil + if err := discardRollback(e); err != nil { + return err } if e.plan == nil { @@ -113,6 +115,29 @@ func phase6State(e *Engine) error { return nil } +// discardRollback removes the rollback tree at Phase 6 entry. A Discard +// failure must abort the phase here, before any state write: nothing is +// persisted yet, so returning an error leaves the next run with the rollback +// dir AND un-advanced state — the recoverable mid-Phase-5 outcome. The stale +// restore puts back bytes the un-advanced manifest still matches, and the +// next diff schedules downloads, not false uploads. Swallowing the error +// instead strands the rollback dir next to advanced state, where the same +// stale restore resurrects pre-sync bytes as phantom local edits which the +// next sync silently re-uploads over the remote. +func discardRollback(e *Engine) error { + if e.rollback == nil { + return nil + } + + if err := e.rollback.Discard(); err != nil { + return fmt.Errorf("discard rollback: %w", err) + } + + e.rollback = nil + + return nil +} + // buildNewBaseManifest computes NEW_BASE = REMOTE + uploads (streamed hashes) // - deletes, with conflicts resolved as remote-wins. Each uploaded path's // hash and size come from the UploadOutcome recorded in Phase 5 — the bytes diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go index 5228009b1..399cef83f 100644 --- a/internal/workload/sync/rollback_discard_test.go +++ b/internal/workload/sync/rollback_discard_test.go @@ -15,8 +15,10 @@ package sync import ( + "fmt" "os" "path/filepath" + "runtime" "testing" "time" @@ -67,6 +69,15 @@ type rollbackDiscardScenario struct { // setupRollbackDiscardScenario wires the scenario and returns it after Plan(), // so the caller can inject a Phase 6 fault and then Execute. func setupRollbackDiscardScenario(t *testing.T) *rollbackDiscardScenario { + return setupRollbackDiscardScenarioWithPatchHook(t, nil) +} + +// setupRollbackDiscardScenarioWithPatchHook is the hook-aware variant. The +// hook, when non-nil, replaces the no-op PatchCodeRef fake: it runs at the +// very end of Phase 5 — after every backup, download, delete, and upload, +// with the rollback dir already on disk but Phase 6 not yet entered — which +// is exactly the window a Phase 6-entry fault must land in. +func setupRollbackDiscardScenarioWithPatchHook(t *testing.T, patchHook func(artifactID, catalogID, catalogVersionID string) error) *rollbackDiscardScenario { t.Helper() const ( @@ -117,7 +128,8 @@ func setupRollbackDiscardScenario(t *testing.T) *rollbackDiscardScenario { GetFn: func(id string) (*workload.Artifact, error) { return draftArtifact(id, catalogID, remoteVer), nil }, - PatchFn: func(_, _, _ string) error { return nil }, + // nil PatchFn is safe: the fake no-ops when the hook is unset. + PatchFn: patchHook, }, Now: time.Now, }) @@ -226,7 +238,9 @@ func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testi require.NoError(t, os.RemoveAll(cPath)) - //nolint:gosec // G703: cPath is the project's own config.json under t.TempDir(); the bytes were read from that same file before the fault + // gosec's taint analysis does not flag this write today, so no nolint + // directive is needed here; if a future gosec flags it again, add one + // back with the G703 rationale. require.NoError(t, os.WriteFile(cPath, s.cfgBytes, 0o644)) staleCfg, err := wapi.LoadConfig(s.dir) @@ -362,3 +376,74 @@ func TestPhase6CleanSuccessDiscardsRollback(t *testing.T) { assert.Equal(t, sha256Hex([]byte(s.newA)), manifest.Files["a.py"].Hash) assert.Equal(t, sha256Hex([]byte(s.remoteB)), manifest.Files["b.py"].Hash) } + +// TestPhase6DiscardFailure_AbortsBeforeStateWrites covers the one Phase 6 +// failure mode the write-failure tests above cannot reach: Discard itself +// failing. The fault rides Phase 5's final step (PatchCodeRef), which fires +// after the rollback dir exists but before Phase 6 runs, and strips write +// permission from the rollback dir so Discard's os.RemoveAll fails. +// +// Phase 6 must abort with a wrapped error BEFORE SaveManifest/SaveConfig. +// At entry nothing has been persisted, so un-advanced state plus the +// surviving rollback dir is exactly the recoverable mid-Phase-5 outcome: +// the next run's stale-rollback restore puts back pre-sync bytes that the +// un-advanced manifest still matches, and the next diff schedules downloads, +// not false uploads. Swallowing the error instead (the pre-fix behavior) +// strands the rollback dir next to ADVANCED state, and that same stale +// restore resurrects pre-sync bytes as phantom local edits which the next +// sync silently re-uploads over the remote. +func TestPhase6DiscardFailure_AbortsBeforeStateWrites(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fault injection relies on POSIX directory permissions; windows ignores them") + } + + // Declare first so the hook closure can read s.rollDir: the hook fires + // during Execute, long after the assignment completes. + var s *rollbackDiscardScenario + + s = setupRollbackDiscardScenarioWithPatchHook(t, func(_, _, _ string) error { + // Phase 5's last step: make the rollback dir unremovable so the + // Phase 6 entry Discard fails. The hook itself must succeed so + // Phase 5 completes and Phase 6 is genuinely reached. + if err := os.Chmod(s.rollDir, 0o555); err != nil { + return fmt.Errorf("fault: chmod rollback dir: %w", err) + } + + return nil + }) + + // Restore permissions before t.TempDir cleanup so the read-only dir can + // be removed; cleanup runs LIFO, so this lands ahead of the TempDir one. + t.Cleanup(func() { _ = os.Chmod(s.rollDir, 0o755) }) + + preManifest, err := os.ReadFile(filepath.Join(wapi.Dir(s.dir), "manifest.json")) + require.NoError(t, err) + + _, err = s.engine.Execute(s.plan) + + require.Error(t, err, "Phase 6 must abort when the rollback dir cannot be discarded") + assert.Contains(t, err.Error(), "discard rollback", + "error must be the wrapped Discard failure, not a later write error") + + // No state write may have happened: the manifest stays byte-identical to + // its pre-sync content, and config does not advance. + postManifest, err := os.ReadFile(filepath.Join(wapi.Dir(s.dir), "manifest.json")) + require.NoError(t, err) + + assert.Equal(t, preManifest, postManifest, + "manifest.json must be untouched when Discard fails at Phase 6 entry") + + cfg, err := wapi.LoadConfig(s.dir) + require.NoError(t, err) + + require.NotNil(t, cfg.LastSyncedVersionID) + assert.Equal(t, "ver-synced", *cfg.LastSyncedVersionID, + "config.json must not advance when Discard fails at Phase 6 entry") + + // The rollback dir survives the failed Discard — that is the fault. Its + // presence is safe only because no state advanced (asserted above): the + // next run's stale-rollback recovery restores it against un-advanced + // state, which is the recoverable mid-Phase-5 outcome. + _, statErr := os.Stat(s.rollDir) + assert.NoError(t, statErr, "the unremovable rollback dir must still be present after the abort") +} From 649446984781cce7c09e3c9894bb25f088258364 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Sat, 29 Aug 2026 00:58:37 -0700 Subject: [PATCH 13/20] test(workload): nil-guard the rollback-dir chmod cleanup against failed setup The discard-failure test's t.Cleanup closure dereferences s.rollDir to restore the read-only permissions. If scenario setup itself fails, the scenario (and its rollback dir) may never have been assigned; guard the closure so the cleanup cannot nil-deref during teardown of an already failing test. --- internal/workload/sync/rollback_discard_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go index 399cef83f..8f6003294 100644 --- a/internal/workload/sync/rollback_discard_test.go +++ b/internal/workload/sync/rollback_discard_test.go @@ -414,7 +414,15 @@ func TestPhase6DiscardFailure_AbortsBeforeStateWrites(t *testing.T) { // Restore permissions before t.TempDir cleanup so the read-only dir can // be removed; cleanup runs LIFO, so this lands ahead of the TempDir one. - t.Cleanup(func() { _ = os.Chmod(s.rollDir, 0o755) }) + // Nil-guarded: if scenario setup itself fails, s (or s.rollDir) may never + // have been assigned and the cleanup must not dereference it. + t.Cleanup(func() { + if s == nil || s.rollDir == "" { + return + } + + _ = os.Chmod(s.rollDir, 0o755) + }) preManifest, err := os.ReadFile(filepath.Join(wapi.Dir(s.dir), "manifest.json")) require.NoError(t, err) From bb0f29a7ecff3ddf9415f62128783b52dfa372ce Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 13:15:34 -0700 Subject: [PATCH 14/20] [RAPTOR-19525] test(testutil): add SkipIfRoot/SkipIfWindows helpers and adopt at chmod-fault sites Root containers make chmod faults inert: the fault never fires and the test fails on its own setup instead of the behaviour it pins. The discard-failure test had only the Windows guard, so it failed instead of skipping as root. Factor the six-line windows+root skip idiom into internal/testutil: SkipIfWindows(t, reason) and SkipIfRoot(t) (real guard build-tagged !windows, no-op on windows where os.Geteuid does not compile and the Windows guard has already skipped). Adopt both at all four chmod-fault sites, keeping the per-site skip messages. Test-only; no production code touched. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- cmd/workload/del/cmd_test.go | 10 ++---- internal/fsutil/fsutil_test.go | 9 ++--- internal/testutil/skip.go | 33 +++++++++++++++++ internal/testutil/skip_unix.go | 36 +++++++++++++++++++ internal/testutil/skip_windows.go | 29 +++++++++++++++ .../workload/sync/rollback_discard_test.go | 11 +++--- internal/workload/wapi/migrate_test.go | 10 ++---- 7 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 internal/testutil/skip.go create mode 100644 internal/testutil/skip_unix.go create mode 100644 internal/testutil/skip_windows.go diff --git a/cmd/workload/del/cmd_test.go b/cmd/workload/del/cmd_test.go index e4d2fd19e..f3edbce48 100644 --- a/cmd/workload/del/cmd_test.go +++ b/cmd/workload/del/cmd_test.go @@ -18,11 +18,11 @@ import ( "bytes" "os" "path/filepath" - "runtime" "testing" "github.com/datarobot/cli/cmd/workload/internal/idargs" "github.com/datarobot/cli/internal/misc/reader" + "github.com/datarobot/cli/internal/testutil" "github.com/datarobot/cli/internal/workload/manifest" "github.com/datarobot/cli/internal/workload/wapi" "github.com/stretchr/testify/assert" @@ -248,13 +248,9 @@ func TestClearStaleBinding_FindsAManifestUnderDir(t *testing.T) { // fails must not fail the command. It does have to say so, because the user is // the one who has to finish the job. func TestClearStaleBinding_WarnsWhenTheManifestCannotBeWritten(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory permission bits do not block rename the same way on Windows") - } + testutil.SkipIfWindows(t, "directory permission bits do not block rename the same way on Windows") - if os.Geteuid() == 0 { - t.Skip("root ignores directory permission bits") - } + testutil.SkipIfRoot(t) dir := t.TempDir() path := writeManifest(t, dir, boundManifest) diff --git a/internal/fsutil/fsutil_test.go b/internal/fsutil/fsutil_test.go index 280fb2baf..3b59a40e8 100644 --- a/internal/fsutil/fsutil_test.go +++ b/internal/fsutil/fsutil_test.go @@ -20,6 +20,7 @@ import ( "runtime" "testing" + "github.com/datarobot/cli/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -75,13 +76,9 @@ func TestExists_ParentIsAFile(t *testing.T) { // An unreadable parent directory stats as EACCES, the other error class that // returns no FileInfo. func TestExists_UnreadableParent(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory permission bits do not gate stat the same way on Windows") - } + testutil.SkipIfWindows(t, "directory permission bits do not gate stat the same way on Windows") - if os.Geteuid() == 0 { - t.Skip("root ignores directory permission bits") - } + testutil.SkipIfRoot(t) tmp := t.TempDir() diff --git a/internal/testutil/skip.go b/internal/testutil/skip.go new file mode 100644 index 000000000..94490e122 --- /dev/null +++ b/internal/testutil/skip.go @@ -0,0 +1,33 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "runtime" + "testing" +) + +// SkipIfWindows skips the test when it runs on Windows. The reason must say +// what the test needs that Windows does not provide: the Windows CI leg is +// continue-on-error, so an unexplained skip is invisible in a CI summary and +// looks no different from a silent pass. A visible reason keeps the skipped +// run diagnosable rather than doubly invisible. +func SkipIfWindows(t *testing.T, reason string) { + t.Helper() + + if runtime.GOOS == "windows" { + t.Skip(reason) + } +} diff --git a/internal/testutil/skip_unix.go b/internal/testutil/skip_unix.go new file mode 100644 index 000000000..06d56a587 --- /dev/null +++ b/internal/testutil/skip_unix.go @@ -0,0 +1,36 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package testutil + +import ( + "os" + "testing" +) + +// SkipIfRoot skips the test when it runs with effective UID 0. Root bypasses +// file and directory permission bits, so a test whose fault is injected via +// chmod (e.g. a read-only directory that must refuse a write or a remove) is +// inert as root: the fault never fires and the test fails on its own setup +// instead of on the behaviour under test. Container CI images commonly run +// as root, which is where this bites. +func SkipIfRoot(t *testing.T) { + t.Helper() + + if os.Geteuid() == 0 { + t.Skip("root ignores directory permission bits") + } +} diff --git a/internal/testutil/skip_windows.go b/internal/testutil/skip_windows.go new file mode 100644 index 000000000..cede765d3 --- /dev/null +++ b/internal/testutil/skip_windows.go @@ -0,0 +1,29 @@ +// Copyright 2026 DataRobot, Inc. and its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package testutil + +import "testing" + +// SkipIfRoot is a no-op on Windows. os.Geteuid does not exist there, so the +// real guard cannot compile; every call site reaches this helper only after +// SkipIfWindows has already skipped, because a chmod fault is a POSIX-only +// mechanism. A no-op therefore loses no coverage, and keeping the signature +// lets call sites use both helpers unconditionally without their own +// platform branching. +func SkipIfRoot(t *testing.T) { + t.Helper() +} diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go index 8f6003294..bcfb916d4 100644 --- a/internal/workload/sync/rollback_discard_test.go +++ b/internal/workload/sync/rollback_discard_test.go @@ -18,11 +18,11 @@ import ( "fmt" "os" "path/filepath" - "runtime" "testing" "time" "github.com/datarobot/cli/internal/drapi/filesapi" + "github.com/datarobot/cli/internal/testutil" "github.com/datarobot/cli/internal/workload" "github.com/datarobot/cli/internal/workload/ignore" "github.com/datarobot/cli/internal/workload/wapi" @@ -393,9 +393,12 @@ func TestPhase6CleanSuccessDiscardsRollback(t *testing.T) { // restore resurrects pre-sync bytes as phantom local edits which the next // sync silently re-uploads over the remote. func TestPhase6DiscardFailure_AbortsBeforeStateWrites(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("fault injection relies on POSIX directory permissions; windows ignores them") - } + testutil.SkipIfWindows(t, "fault injection relies on POSIX directory permissions; windows ignores them") + + // The fault is a chmod, which root bypasses: without this guard the + // Discard succeeds, no error is returned, and the test fails on its own + // setup rather than on the ordering it pins. + testutil.SkipIfRoot(t) // Declare first so the hook closure can read s.rollDir: the hook fires // during Execute, long after the assignment completes. diff --git a/internal/workload/wapi/migrate_test.go b/internal/workload/wapi/migrate_test.go index fad7fef1d..c07d8d133 100644 --- a/internal/workload/wapi/migrate_test.go +++ b/internal/workload/wapi/migrate_test.go @@ -18,9 +18,9 @@ import ( "os" "path" "path/filepath" - "runtime" "testing" + "github.com/datarobot/cli/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -100,13 +100,9 @@ func TestEnsureMigrated_BothPresentKeepsCurrent(t *testing.T) { // A move that cannot happen must not fail the command: the legacy directory // stays readable and every path helper keeps resolving to it. func TestEnsureMigrated_UnmovableFallsBackToLegacy(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("directory permission bits do not block rename the same way on Windows") - } + testutil.SkipIfWindows(t, "directory permission bits do not block rename the same way on Windows") - if os.Geteuid() == 0 { - t.Skip("root ignores directory permission bits") - } + testutil.SkipIfRoot(t) tmp := t.TempDir() legacy := seedLegacyDir(t, tmp) From 9cc35465a86197614d45229182fedc270f544395 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 13:24:23 -0700 Subject: [PATCH 15/20] [RAPTOR-19525] test(workload): assert real post-SaveConfig-failure convergence through Run, not direct Execute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old test asserted convergence after a SaveConfig failure but then forced it: right after asserting plan.IsEmpty() it called e2.Execute(plan) directly, which no production caller ever does with an empty plan. Run short-circuits empty plans before Execute, so in production the post-failure sync detects drift, fetches AllFiles, finds an empty plan, and returns WITHOUT running Phase 6 — config.json keeps the old version until some later sync has real work. Rename to TestSaveConfigFailure_ManifestAdvanced_ConfigConvergesOnNextRealSync and drive the post-failure sync through Run(). Assert the honest sequence: drift detected (one AllFiles call, not the fast path), empty plan, zero upload-side calls, no new version, config still old while the manifest stays advanced — the safe asymmetry, since manifest-ahead-of-config self-heals via drift detection on every later run while the reverse silently poisons BASE forever. Then introduce a real change and assert a third Run converges config to the manifest's version through Phase 6. The direct Execute call is gone. Teeth proven both ways: replacing Run with the old Plan+Execute pattern fails the config-unchanged, no-new-version, and AllFiles-count assertions, and temporarily removing production's empty-plan short-circuit in Run fails the same three. Test-only; no production code touched. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/upload_failure_test.go | 169 ++++++++++++++---- 1 file changed, 136 insertions(+), 33 deletions(-) diff --git a/internal/workload/sync/upload_failure_test.go b/internal/workload/sync/upload_failure_test.go index b9d5eec8d..92b6937bb 100644 --- a/internal/workload/sync/upload_failure_test.go +++ b/internal/workload/sync/upload_failure_test.go @@ -371,18 +371,31 @@ func TestSaveManifestFailure_DoesNotAdvanceConfig(t *testing.T) { "config LastSyncedVersionID must NOT be advanced when SaveManifest fails") } -// TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs verifies the safe -// failure direction: when SaveManifest succeeds but SaveConfig fails (injected -// by making config.json a directory), the manifest IS advanced (new version, -// new hashes) while config stays stale (old version). The next sync detects -// drift (config old != artifact new), fetches AllFiles rather than -// fast-pathing, and converges — config is updated to match the manifest. +// TestSaveConfigFailure_ManifestAdvanced_ConfigConvergesOnNextRealSync +// verifies the safe failure direction and the honest recovery path: when +// SaveManifest succeeds but SaveConfig fails (injected by making config.json +// a directory so the atomic rename fails), the manifest IS advanced (new +// version, streamed hashes) while config stays stale (old version). // -// This is the self-healing property that makes manifest-before-config the -// safe ordering: an advanced manifest paired with a stale config makes the -// next run re-sync, whereas an advanced config paired with a stale manifest -// silently poisons BASE and reports "Up to date." forever. -func TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs(t *testing.T) { +// What production then does — and all this test asserts — is asymmetric on +// purpose. The next sync detects drift (config's old version vs the +// artifact's new one), fetches AllFiles rather than fast-pathing, computes an +// EMPTY plan (the advanced manifest matches the remote, and the disk matches +// both), and returns WITHOUT executing: Run short-circuits empty plans before +// Execute, so Phase 6 never runs and config.json keeps the old version. No +// production caller reaches Execute with an empty plan, so a test that forces +// one there would assert a convergence path that cannot happen. Config is +// converged only by the next sync that has real work to do — which is exactly +// what the second half of this test drives and asserts. +// +// The asymmetry — manifest advanced past a stale config — is the safe +// direction, and the reason manifest-before-config is the correct write +// order: the version mismatch makes EVERY later sync detect drift and fetch +// the real remote, so the window self-heals at the first sync with actual +// work. The reverse — config advanced past a stale manifest — is silent, +// permanent BASE poisoning: no drift is ever detected, the fast path copies +// the stale BASE to REMOTE, and the CLI reports "Up to date." forever. +func TestSaveConfigFailure_ManifestAdvanced_ConfigConvergesOnNextRealSync(t *testing.T) { const ( catalogID = "cid-synced" versionID = "ver-synced" @@ -488,62 +501,152 @@ func TestSaveConfigFailure_ManifestAdvanced_NextSyncResyncs(t *testing.T) { serverFiles[path] = filesapi.FileMeta{Hash: fm.Hash, Size: fm.Size} } - // The next sync must detect drift (config old != artifact new), fetch - // AllFiles rather than fast-pathing, and converge. + // The next sync goes through Run() — the production entry point, where + // the plan and the execute decision both live. The fake's server state + // is built from the manifest SaveManifest wrote, so AllFiles serves + // exactly what BASE describes: base == remote, and the disk matches + // both, so the honest plan is empty. What must still happen is the + // round-trip — drift was detected, the remote was fetched, no silent + // fast path. What must NOT happen is Execute: no production caller + // executes an empty plan. fake := (&fakeFilesClient{ catalogID: catalogID, stageID: "stage-should-not-be-used", versionID: newVerID, }).withVersion(catalogID, newVerID, serverFiles) - e2, err := newWithDeps(dir, Options{Yes: true}, Deps{ - Files: fake, - Artifacts: &fakeArtifactStore{ - GetFn: func(id string) (*workload.Artifact, error) { - return draftArtifact(id, catalogID, newVerID), nil + // One engine per sync invocation, matching how the command layer + // constructs a fresh engine for every run. + newEngine := func() (*Engine, error) { + return newWithDeps(dir, Options{Yes: true}, Deps{ + Files: fake, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVerID), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, }, - PatchFn: func(_, _, _ string) error { return nil }, - }, - Now: time.Now, - }) + Now: time.Now, + }) + } + + e2, err := newEngine() require.NoError(t, err) t.Cleanup(func() { _ = e2.Close() }) - plan, err := e2.Plan() + result, err := e2.Run() require.NoError(t, err) + require.NotNil(t, result) // Drift was detected: AllFiles was called (not fast-pathed). assert.Equal(t, 1, fake.AllFilesCalls(), "drift must trigger an AllFiles round-trip, not the fast path") - // The plan should be empty: manifest (advanced) == remote (AllFiles), - // and local == base (no disk changes since the failed sync). But the - // sync ran the full pipeline — it did not silently fast-path. - assert.True(t, plan.IsEmpty(), + // The plan Run computed is empty: manifest (advanced) == remote + // (AllFiles), and local == base (no disk changes since the failed + // sync). The sync ran the full pipeline — it did not silently + // fast-path — it just found nothing to do. + assert.True(t, e2.plan.IsEmpty(), "plan should be empty — manifest matches remote, no disk changes") - // Execute to run Phase 6, which updates config to the new version. - result, err := e2.Execute(plan) + // Run returned WITHOUT executing. The empty-plan short-circuit fires + // before Execute, so no upload-side call was issued at all. + assert.Equal(t, 0, fake.CreateStageCalls(), + "the empty-plan sync must not stage anything — Run returns before Execute") + assert.Equal(t, 0, fake.UploadToStageCalls(), + "the empty-plan sync must not upload anything — Run returns before Execute") + assert.Equal(t, 0, fake.ApplyStageCalls(), + "the empty-plan sync must not apply anything — Run returns before Execute") + + assert.Empty(t, result.NewVersion, + "the empty-plan run creates no new version — Phase 6 never ran") + + // Phase 6 never ran, so config.json still holds the OLD version while + // manifest.json holds the new one. That asymmetry is the safe one (see + // the function comment): the version mismatch makes every later sync + // detect drift and fetch the real remote, so the window self-heals at + // the first sync with actual work. The reverse asymmetry — config + // advanced past a stale manifest — would never be detected and would + // silently poison BASE forever. + staleCfg, err := wapi.LoadConfig(dir) require.NoError(t, err) - require.NotNil(t, result) + require.NotNil(t, staleCfg.LastSyncedVersionID) + assert.Equal(t, versionID, *staleCfg.LastSyncedVersionID, + "config must still hold the old version — Run short-circuits the empty plan, so Phase 6 never runs") + + advManifest, err := wapi.LoadManifest(dir) + require.NoError(t, err) + + require.NotNil(t, advManifest.SyncedVersionID) + assert.Equal(t, newVerID, *advManifest.SyncedVersionID, + "manifest stays advanced — SaveManifest ran before the SaveConfig failure") + + // --- Convergence needs a sync with real work --- + + // Only a plan with actual work makes Run reach Execute, and only the + // Phase 6 reached that way converges config. Introduce a real change + // so the next run has something to upload. + finalContent := "print('real work')\n" - // Config must now converge to the new version — self-healing. + modifyFile(t, dir, "app.py", finalContent) + + e3, err := newEngine() + require.NoError(t, err) + + t.Cleanup(func() { _ = e3.Close() }) + + result3, err := e3.Run() + require.NoError(t, err) + require.NotNil(t, result3) + + require.Len(t, e3.plan.Uploads, 1, + "the modified file must be the one real upload") + assert.Equal(t, "app.py", e3.plan.Uploads[0].Path) + assert.Equal(t, 1, result3.UploadedCount, + "the real-work sync must report the upload it performed") + + // Config still lagged the artifact, so this run detected drift too and + // fetched AllFiles a second time. + assert.Equal(t, 2, fake.AllFilesCalls(), + "the still-drifted run must fetch AllFiles again") + + assert.Equal(t, 1, fake.ApplyStageCalls(), + "the real-work sync must execute its plan") + + // Config must now converge to the manifest's version — the honest + // self-healing path: a sync with real work, never a forced Execute. convergedCfg, err := wapi.LoadConfig(dir) require.NoError(t, err) require.NotNil(t, convergedCfg.LastSyncedVersionID) assert.Equal(t, newVerID, *convergedCfg.LastSyncedVersionID, - "config must converge to the new version after the self-healing sync") + "config must converge to the manifest's version once a sync has real work") - // Manifest and config must agree. convergedManifest, err := wapi.LoadManifest(dir) require.NoError(t, err) require.NotNil(t, convergedManifest.SyncedVersionID) assert.Equal(t, *convergedCfg.LastSyncedVersionID, *convergedManifest.SyncedVersionID, "config and manifest must agree on the version after convergence") + + // The converged manifest records the bytes this sync actually + // streamed, and the server holds exactly those bytes. + convergedFM, ok := convergedManifest.Files["app.py"] + require.True(t, ok) + + assert.Equal(t, sha256Hex([]byte(finalContent)), convergedFM.Hash, + "manifest must record the hash streamed by the converging sync") + + serverAll, err := fake.AllFiles(catalogID, newVerID) + require.NoError(t, err) + + serverFM, ok := serverAll["app.py"] + require.True(t, ok) + + assert.Equal(t, convergedFM.Hash, serverFM.Hash, + "the server must hold the bytes the manifest records after convergence") } // --- VAL-UPLOAD-011b / VAL-UPLOAD-013: Failure modes through the engine --- From 4b27b8fdcff9030ea4b0a5a2bfe5221791fdaa42 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 13:43:39 -0700 Subject: [PATCH 16/20] [RAPTOR-19525] test(workload): pin the case-collision check at its phase2 call site on any host The only end-to-end case-collision test (TestCaseCollision_ErrorIsBeforeUpload) asserted an error it never produced and counted upload calls that Plan never makes, so deleting the caseCollisionsFromManifest call in phase2Manifests left the whole package green on macOS. Replace it with TestCaseCollision_ Phase2StopsBeforeRemoteLoad, which drives phase2Manifests directly with a seam-injected colliding local manifest and asserts the collision error, so the check is pinned at its call site on case-insensitive hosts too. The hashEntriesFn seam (diskspace.go availableBytesFn pattern) is the only production change and is behavior-neutral: hashEntries remains the default. Mutation-verified on macOS: deleting the call site makes the new test fail ("An error is expected but got nil"); restoring it makes the test pass. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/phase2_manifests.go | 9 ++- .../sync/surroundings_regression_test.go | 63 ++++++++++--------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/internal/workload/sync/phase2_manifests.go b/internal/workload/sync/phase2_manifests.go index 339d3d875..fc458bf99 100644 --- a/internal/workload/sync/phase2_manifests.go +++ b/internal/workload/sync/phase2_manifests.go @@ -22,6 +22,13 @@ import ( "github.com/datarobot/cli/internal/workload/ignore" ) +// hashEntriesFn is a test seam in the style of availableBytesFn +// (diskspace.go): tests swap it to inject a local manifest that a real +// filesystem walk can never produce. Case-insensitive hosts (macOS, Windows) +// collapse case-colliding paths, so the collision check below can only be +// exercised on every host by injecting the manifest rather than walking one. +var hashEntriesFn = hashEntries + // phase2Manifests builds the LOCAL manifest by walking + hashing the // project, and either fetches REMOTE from FilesAPI (when drifted) or // copies it from BASE (the solo-developer fast path). @@ -55,7 +62,7 @@ func phase2Manifests(e *Engine) error { return fmt.Errorf("walk project directory: %w", err) } - local, err := hashEntries(entries) + local, err := hashEntriesFn(entries) if err != nil { return err } diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go index cd46bb41d..cd74fdf2c 100644 --- a/internal/workload/sync/surroundings_regression_test.go +++ b/internal/workload/sync/surroundings_regression_test.go @@ -226,43 +226,46 @@ func TestCaseCollision_FailsBeforeUpload(t *testing.T) { assert.Contains(t, msg, "Config.yaml vs config.yaml") } -// TestCaseCollision_ErrorIsBeforeUpload verifies that the case-collision check -// runs in Phase 2 (manifests), before Phase 5 (execute) where uploads happen. -// We cannot create case-colliding files on macOS, so we verify the check -// position by confirming that a Plan with no case collisions succeeds and -// the fake's upload counters are zero (Plan does not execute). -func TestCaseCollision_ErrorIsBeforeUpload(t *testing.T) { - dir := syncedProject(t, map[string]string{ +// TestCaseCollision_Phase2StopsBeforeRemoteLoad pins the case-collision check +// at its real call site inside phase2Manifests: a colliding local manifest +// must make Phase 2 fail before the remote load, on any host. +// TestCaseCollision_FailsBeforeUpload pins caseCollisionsFromManifest itself, +// but nothing before this test proved phase2Manifests still calls it — a +// real-FS walk can never produce a colliding pair on a case-insensitive host +// (macOS, Windows collapse such paths), so the local manifest is injected +// via the hashEntriesFn seam instead of walked off disk. Deleting the call +// site makes this test fail; there is no skip and no case-sensitivity probe. +func TestCaseCollision_Phase2StopsBeforeRemoteLoad(t *testing.T) { + dir := initProject(t, map[string]string{ "app.py": "print('hi')\n", - }, "cid-cc", "ver-cc") + }) - fake := &fakeFilesClient{ - catalogID: "cid-cc", - stageID: "stage-cc", - versionID: "ver-cc-next", + e := lockfileEngine(t, dir, noLockfileRunner) + + // Swap the hashing seam for a manifest with a case-only collision. The + // package uses no t.Parallel, so the package-level swap is safe under + // -race -shuffle; t.Cleanup restores the real implementation even when + // an assertion fails mid-test. + origHashEntries := hashEntriesFn + + hashEntriesFn = func(_ []fileops.Entry) (LocalManifest, error) { + return LocalManifest{ + "Greeting.txt": {Hash: "h1", Size: 6}, + "greeting.txt": {Hash: "h2", Size: 3}, + }, nil } - e, err := newWithDeps(dir, Options{}, Deps{ - Files: fake, - Artifacts: &fakeArtifactStore{ - GetFn: func(id string) (*workload.Artifact, error) { - return draftArtifact(id, "cid-cc", "ver-cc"), nil - }, - }, - Now: time.Now, - Lockfile: noLockfileRunner, - }) - require.NoError(t, err) + t.Cleanup(func() { hashEntriesFn = origHashEntries }) - t.Cleanup(func() { _ = e.Close() }) + err := phase2Manifests(e) - plan, err := e.Plan() - require.NoError(t, err) - require.True(t, plan.IsEmpty(), "synced project with no changes has an empty plan") + // With e.drifted unset (false), a phase2Manifests missing the collision + // check would copy BASE and return nil here — that is the mutation this + // test exists to catch. + require.Error(t, err, "phase2Manifests must fail on a case-only collision") - // No case collision, no uploads — Plan does not call Execute. - assert.Equal(t, 0, fake.UploadToStageCalls(), "Plan must not upload") - assert.Equal(t, 0, fake.ApplyStageCalls(), "Plan must not apply") + assert.Contains(t, err.Error(), "case-only path collisions") + assert.Contains(t, err.Error(), "Greeting.txt vs greeting.txt") } // --------------------------------------------------------------------------- From f453cbe18a40b028e0aaef249e50531a121c7ab6 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 13:50:32 -0700 Subject: [PATCH 17/20] [RAPTOR-19525] test(workload): fix case-fold header and pin manifest/config schemas against added fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review-867 fixes, both test/comment-only: - TestCaseFoldedIgnorePatterns: the doc comment claimed a .drignore pattern "BUILD/" must exclude "build/" (case-folded matching), while the body pins the opposite — user patterns are case-sensitive via the gitignore library (*.TMP does not match scratch.tmp), and only system excludes fold case. The header now states exactly what the assertions guarantee. - TestManifestSchemaUnchanged: the comments claimed "standard fields only" / "no new fields" but assert.Contains on the raw JSON stays green when a field is ADDED. Replaced with an exact top-level key-set comparison (map[string]any against an allow-list transcribed from the wapi.Manifest and wapi.Config json tags: manifest = version/syncedAt/syncedVersionId/ files, config = artifactId/catalogId/lastSyncedVersionId/createdAt/ cliVersion). Mutation-verified: adding a junk field to either writer struct fails the test naming the unexpected key. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/phase6_state_test.go | 65 +++++++++++++++---- .../sync/surroundings_regression_test.go | 21 +++--- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/internal/workload/sync/phase6_state_test.go b/internal/workload/sync/phase6_state_test.go index c08bdd156..44cbf84b3 100644 --- a/internal/workload/sync/phase6_state_test.go +++ b/internal/workload/sync/phase6_state_test.go @@ -15,6 +15,7 @@ package sync import ( + "encoding/json" "os" "testing" "time" @@ -228,7 +229,9 @@ func TestPhase6EmptyPlanStillWritesManifest(t *testing.T) { } // TestManifestSchemaUnchanged verifies that the written manifest carries -// "version": 1 and that no new fields are added to manifest.json or config.json. +// "version": 1 and that manifest.json and config.json carry exactly their +// known top-level key sets — no field added, none removed. The version-1 +// schema is frozen: any new field must fail this test, not slip past it. func TestManifestSchemaUnchanged(t *testing.T) { const ( catalogID = "cid-synced" @@ -272,23 +275,25 @@ func TestManifestSchemaUnchanged(t *testing.T) { assert.Equal(t, 1, manifest.Version, "manifest version must stay 1") - // Verify the manifest has exactly the expected top-level keys by reading - // the raw JSON and checking no unexpected fields were added. + // Pin the manifest schema by exact key set. Substring-presence checks on + // the raw JSON stay green when a field is ADDED — the very drift "no new + // fields" must catch — so compare the parsed object's top-level keys + // against the allow-list transcribed from wapi.Manifest's json tags. rawManifest, err := os.ReadFile(dir + "/.datarobot/workload/manifest.json") require.NoError(t, err) - // The manifest must contain "version": 1 and the standard fields only. - assert.Contains(t, string(rawManifest), `"version": 1`) - assert.Contains(t, string(rawManifest), `"files"`) - assert.Contains(t, string(rawManifest), `"syncedVersionId"`) + assert.Contains(t, string(rawManifest), `"version": 1`, + "the serialized manifest must carry version 1 as a JSON number") - // Verify config.json has no new fields. + assertTopLevelKeys(t, rawManifest, "manifest.json", + []string{"version", "syncedAt", "syncedVersionId", "files"}) + + // Same pin for config.json, transcribed from wapi.Config's json tags. rawConfig, err := os.ReadFile(dir + "/.datarobot/workload/config.json") require.NoError(t, err) - assert.Contains(t, string(rawConfig), `"artifactId"`) - assert.Contains(t, string(rawConfig), `"catalogId"`) - assert.Contains(t, string(rawConfig), `"lastSyncedVersionId"`) + assertTopLevelKeys(t, rawConfig, "config.json", + []string{"artifactId", "catalogId", "lastSyncedVersionId", "createdAt", "cliVersion"}) // syncedVersionId in manifest must equal the version in the result. require.NotNil(t, manifest.SyncedVersionID) @@ -303,3 +308,41 @@ func TestManifestSchemaUnchanged(t *testing.T) { assert.Equal(t, *manifest.SyncedVersionID, *cfg.LastSyncedVersionID, "config LastSyncedVersionID must equal manifest syncedVersionId") } + +// assertTopLevelKeys pins the exact set of top-level keys of an on-disk JSON +// state file. Substring-presence assertions cannot fail when a field is +// ADDED, which is precisely the schema drift this guards against — the +// key-set comparison fails on any added or removed key and names the +// offending key in the failure message. +func assertTopLevelKeys(t *testing.T, data []byte, label string, want []string) { + t.Helper() + + var m map[string]any + + require.NoError(t, json.Unmarshal(data, &m), + "%s must parse as a flat JSON object", label) + + wantSet := make(map[string]bool, len(want)) + + for _, k := range want { + wantSet[k] = true + } + + // A key present in the file but absent from the allow-list is a field + // ADDED to the on-disk schema. This is the direction substring checks + // are blind to, and the one that turns a schema change silent: the + // file's keys must each be contained in the allow-list, so an unknown + // key fails the assertion naming itself. + for k := range m { + assert.Contains(t, wantSet, k, + "%s has unexpected top-level key %q — a field was added to the on-disk schema (allow-list: %v)", label, k, want) + } + + // The other direction: an allow-list key missing from the file means a + // field was removed or renamed. Either way the schema cannot drift + // silently. + for _, k := range want { + assert.Contains(t, m, k, + "%s is missing top-level key %q (allow-list: %v)", label, k, want) + } +} diff --git a/internal/workload/sync/surroundings_regression_test.go b/internal/workload/sync/surroundings_regression_test.go index cd74fdf2c..962d691c5 100644 --- a/internal/workload/sync/surroundings_regression_test.go +++ b/internal/workload/sync/surroundings_regression_test.go @@ -165,19 +165,16 @@ func TestCaseFoldedSystemExcludes(t *testing.T) { "case-folded system exclude must still be excluded") } -// TestCaseFoldedIgnorePatterns verifies that case-folding of ignore patterns -// still matches paths differing only in case. A .drignore pattern "BUILD/" -// must exclude a directory named "build/" (case-folded match via system -// excludes is already tested above; this tests user-pattern case-folding -// through the gitignore library). +// TestCaseFoldedIgnorePatterns pins the case-sensitivity split between the +// two ignore layers. USER patterns from .drignore go through the gitignore +// library, which matches case-sensitively: *.TMP does NOT match scratch.tmp, +// so the file stays in the upload plan. SYSTEM excludes fold case instead +// (a differently-cased .Datarobot state directory is still excluded), which +// the system-exclude test above covers separately. func TestCaseFoldedIgnorePatterns(t *testing.T) { - // The gitignore library is case-sensitive on Unix but the system excludes - // fold case. Here we test that a user pattern in .drignore matches a - // file of the same name with different case, which the system-exclude - // case-folding handles for system paths. For user patterns, the - // gitignore library is case-sensitive, so a pattern "*.TMP" does NOT - // match "scratch.tmp". We verify the actual behaviour: user patterns are - // case-sensitive (as documented), while system excludes fold case. + // A user pattern in .drignore is matched case-sensitively by the + // gitignore library: *.TMP must NOT exclude scratch.tmp. System excludes + // fold case, but they are a separate layer pinned by the test above. dir := initProject(t, map[string]string{ "app.py": "print('hi')\n", "scratch.tmp": "x", From 714cfbe83c9cf78e53920fee8a1aebf08784694b Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 13:55:23 -0700 Subject: [PATCH 18/20] [RAPTOR-19525] test(workload): fail loudly when AllFiles is called with an unregistered version fakeFilesClient.AllFiles returned an empty map with a nil error for a version ID that was never registered, so a typo'd fixture version produced a plausible-looking plan (every base entry REMOTE_DELETED, every local file LOCAL_ADDED) instead of a test failure. Return an error naming the version instead, matching DownloadFile's loud-failure contract for unregistered versions and the real client, whose AllFiles surfaces the server's non-200 as an error. The allFilesCalls counter still increments on the error path: it counts calls received, and a failed lookup is still a call. Pinned by the new TestFakeAllFiles_UnknownVersionErrors self-test. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../sync/fake_files_client_self_test.go | 41 +++++++++++++++++++ .../workload/sync/fake_files_client_test.go | 13 +++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/internal/workload/sync/fake_files_client_self_test.go b/internal/workload/sync/fake_files_client_self_test.go index 022e14e3e..7d146ac3e 100644 --- a/internal/workload/sync/fake_files_client_self_test.go +++ b/internal/workload/sync/fake_files_client_self_test.go @@ -333,6 +333,47 @@ func TestFakeCallCounters(t *testing.T) { assert.Equal(t, 2, fake.AllFilesCalls()) } +// TestFakeAllFiles_UnknownVersionErrors verifies that AllFiles fails loudly +// with an error naming the version ID when asked for a version that was +// never registered in the fake's server state. Returning an empty map here +// would turn a typo'd fixture version into a plausible-looking plan (every +// base entry REMOTE_DELETED, every local file LOCAL_ADDED) instead of a +// test failure — the same loud-failure contract DownloadFile already +// upholds for unregistered versions. +func TestFakeAllFiles_UnknownVersionErrors(t *testing.T) { + registered := map[string]filesapi.FileMeta{ + "app.py": {Hash: sha256Hex([]byte("app")), Size: 3}, + } + + fake := (&fakeFilesClient{ + catalogID: "cid-1", + stageID: "stage-1", + versionID: "ver-1", + }).withVersion("cid-1", "ver-1", registered) + + // Control: a registered version still serves its files, so the error + // below is specific to the unregistered ID rather than a blanket failure. + all, err := fake.AllFiles(fake.catalogID, "ver-1") + require.NoError(t, err) + + assert.Len(t, all, 1) + assert.Equal(t, registered["app.py"], all["app.py"]) + assert.Equal(t, 1, fake.AllFilesCalls()) + + // The unknown version must error and name the version, never return an + // empty map that would read as "the remote has no files". + _, err = fake.AllFiles(fake.catalogID, "ver-typo") + require.Error(t, err) + + assert.Contains(t, err.Error(), "fakeFilesClient.AllFiles", "error must identify the failing client method") + assert.Contains(t, err.Error(), "ver-typo", "error must name the unregistered version ID") + + // Counter semantics are "calls received": the failed lookup still + // counts, so a test's call-count assertions are not diluted because a + // call happened to hit the error path. + assert.Equal(t, 2, fake.AllFilesCalls()) +} + // TestFakeFaultInjection_DropPath verifies that the dropPathFromApply hook // omits a path from the resulting version. Passes with the fault off, fails // with it on (the path is missing from AllFiles). diff --git a/internal/workload/sync/fake_files_client_test.go b/internal/workload/sync/fake_files_client_test.go index e4a0b78f8..1f625ebd0 100644 --- a/internal/workload/sync/fake_files_client_test.go +++ b/internal/workload/sync/fake_files_client_test.go @@ -531,12 +531,21 @@ func (f *fakeFilesClient) AllFiles(_, versionID string) (map[string]filesapi.Fil f.mu.Lock() defer f.mu.Unlock() + // The counter means "calls received", so it increments even when the + // lookup below fails: a test asserting a call count does not want the + // total silently lowered by error-path bookkeeping, and the call did + // reach the client either way. f.allFilesCalls++ - // Get the files for the requested version from server state. + // Get the files for the requested version from server state. An + // unregistered version ID is a fixture mistake (usually a typo'd + // version string), not an empty remote: the real client surfaces the + // server's non-200 as an error here, and returning an empty map would + // instead fabricate a plausible-looking plan (every base entry + // REMOTE_DELETED, every local file LOCAL_ADDED) that hides the typo. files, ok := f.versions[versionID] if !ok { - return make(map[string]filesapi.FileMeta), nil + return nil, fmt.Errorf("fakeFilesClient.AllFiles: version %s not found", versionID) } // Copy so the caller cannot mutate our state. From 70bb928991a8c3d9662171589da5570fdbec5622 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 14:01:10 -0700 Subject: [PATCH 19/20] [RAPTOR-19525] test(workload): stop claiming empty-plan Execute convergence in the rollback-discard test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tail of TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads asserted plan2.IsEmpty() and then called e2.Execute(plan2) directly, with a comment claiming that executing the empty plan runs Phase 6 and converges config — the self-healing direction. No production caller reaches Execute with an empty plan: Run() short-circuits empty plans before Execute, and the cmd.go preview paths skip Execute entirely, so Phase 6 never runs on an empty-plan sync and config never converges that way. The test was asserting an unreachable path as if it were the self-healing mechanism. Rewrite the next-run half to drive every leg through Run(), the production entry point, mirroring the honest shape of TestSaveConfigFailure_ManifestAdvanced_ConfigConvergesOnNextRealSync (upload_failure_test.go): - The drifted empty-plan run must fetch AllFiles (no fast path), compute an empty plan with no false uploads for the rollback-covered path, and return WITHOUT executing: zero stage/upload/apply calls, no new version, and config still holding the pre-sync version. - Convergence is then asserted through the only production path that reaches Phase 6: a subsequent sync with real work (a.py modified). b.py, the rollback-covered path, is deliberately left untouched so the real-work leg also proves it produces no false upload alongside genuine work. The core purpose is unchanged: rollback discarded at Phase 6 entry, no false uploads on the next run. Mutation-verified both directions: temporarily removing the empty-plan short-circuit in Run() fails the "config must still hold the pre-sync version" leg (the exact unreachable path the old test forced), and making Run() skip Execute for all plans fails the real-work convergence leg. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .../workload/sync/rollback_discard_test.go | 114 +++++++++++++++--- 1 file changed, 100 insertions(+), 14 deletions(-) diff --git a/internal/workload/sync/rollback_discard_test.go b/internal/workload/sync/rollback_discard_test.go index bcfb916d4..92e9b6114 100644 --- a/internal/workload/sync/rollback_discard_test.go +++ b/internal/workload/sync/rollback_discard_test.go @@ -183,7 +183,11 @@ func assertNoRollbackDir(t *testing.T, rollDir string) { // main regression: SaveConfig fails AFTER SaveManifest advanced the manifest. // The rollback dir must already be gone (discarded at Phase 6 entry), and the // next run must reconcile from the remote without resurrecting b.py's -// pre-sync bytes as a false LOCAL_MODIFIED upload. +// pre-sync bytes as a false LOCAL_MODIFIED upload. Every leg of the next-run +// half drives the production entry point Run(); config convergence — the +// self-healing of the manifest-ahead-of-config asymmetry — is asserted only +// through the sync after that, which has real work to do, because Run +// short-circuits empty plans before Execute and Phase 6 never runs on them. // // Fulfills VAL-ROLLBACK-001 (SaveConfig-failure leg) and VAL-ROLLBACK-002. func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testing.T) { @@ -277,8 +281,22 @@ func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testi t.Cleanup(func() { _ = e2.Close() }) - plan2, err := e2.Plan() + // The next sync goes through Run() — the production entry point, where + // the plan and the execute decision both live. The fake's server state + // is AllFiles on the version this sync's apply created, so BASE (the + // advanced manifest) == REMOTE and the disk matches both: the honest + // plan is empty. What must still happen is the round-trip — config + // still names the old version, so drift was detected and the real + // remote fetched, no silent fast path. What must NOT happen is a false + // upload for the rollback-covered path, or Execute: Run short-circuits + // empty plans before Execute, so Phase 6 never runs on this sync. No + // production caller reaches Execute with an empty plan, so a test that + // forced one there would assert a convergence path that cannot happen; + // convergence is asserted below through the only production path that + // reaches it — a subsequent sync with real work. + result2, err := e2.Run() require.NoError(t, err) + require.NotNil(t, result2) // No stale restore: with the rollback discarded at entry, Phase 0 has // nothing to resurrect. Pre-fix, this is true AND b.py has been rolled @@ -286,27 +304,95 @@ func TestPhase6SaveConfigFailure_DiscardsRollback_NextRunNoFalseUploads(t *testi assert.False(t, e2.StaleRollbackRestored(), "the next run must not restore a stale rollback — the rollback dir was discarded at Phase 6 entry") - // The plan must not contain false LOCAL_MODIFIED / LOCAL_ADDED uploads - // for the rollback-covered paths. b.py is the covered path here: its - // manifest entry (advanced) matches the remote, so there is nothing to - // upload. Pre-fix, the resurrected pre-sync bytes are classified as - // LOCAL_MODIFIED and silently re-uploaded over the remote. - assert.Empty(t, plan2.Uploads, - "the next run must not upload anything for rollback-covered paths — got %v", plan2.Uploads) - assert.True(t, plan2.IsEmpty(), + // Drift was detected (config names the old version, the artifact the + // new one): AllFiles was fetched, not fast-pathed. + assert.Equal(t, 1, fake2.AllFilesCalls(), + "drift must trigger an AllFiles round-trip, not the fast path") + + // The plan Run computed must not contain false LOCAL_MODIFIED / + // LOCAL_ADDED uploads for the rollback-covered paths. b.py is the + // covered path here: its manifest entry (advanced) matches the remote, + // so there is nothing to upload. Pre-fix, the resurrected pre-sync bytes + // are classified as LOCAL_MODIFIED and silently re-uploaded over the + // remote. + assert.Empty(t, e2.plan.Uploads, + "the next run must not upload anything for rollback-covered paths — got %v", e2.plan.Uploads) + assert.True(t, e2.plan.IsEmpty(), "the next run's plan must reconcile from the remote and be empty") - // Executing the (empty) plan runs Phase 6, which converges config to the - // version the manifest already records — the self-healing direction. - _, err = e2.Execute(plan2) + // Run returned WITHOUT executing. The empty-plan short-circuit fires + // before Execute, so no upload-side call was issued at all. + assert.Equal(t, 0, fake2.CreateStageCalls(), + "the empty-plan sync must not stage anything — Run returns before Execute") + assert.Equal(t, 0, fake2.UploadToStageCalls(), + "the empty-plan sync must not upload anything — Run returns before Execute") + assert.Equal(t, 0, fake2.ApplyStageCalls(), + "the empty-plan sync must not apply anything — Run returns before Execute") + + assert.Empty(t, result2.NewVersion, + "the empty-plan run creates no new version — Phase 6 never ran") + + // Phase 6 never ran, so config.json still holds the pre-sync version + // while the manifest holds the advanced one. That asymmetry is the safe + // one: the version mismatch makes every later sync detect drift and + // fetch the real remote, so the window self-heals at the first sync + // with actual work — which is what the leg below drives. + stillStaleCfg, err := wapi.LoadConfig(s.dir) require.NoError(t, err) + require.NotNil(t, stillStaleCfg.LastSyncedVersionID) + assert.Equal(t, oldVersion, *stillStaleCfg.LastSyncedVersionID, + "config must still hold the pre-sync version — Run short-circuits the empty plan, so Phase 6 never runs") + + // --- Convergence needs a sync with real work --- + + // Only a plan with actual work makes Run reach Execute, and only the + // Phase 6 reached that way converges config. Modify a.py — NOT b.py: + // b.py is the rollback-covered path whose absence from every upload plan + // is this test's core claim, so the real-work leg must leave it + // untouched and prove it is still not uploaded alongside genuine work. + realWorkA := "AAAA-real-work\n" + + modifyFile(t, s.dir, "a.py", realWorkA) + + e3, err := newWithDeps(s.dir, Options{Yes: true}, Deps{ + Files: fake2, + Artifacts: &fakeArtifactStore{ + GetFn: func(id string) (*workload.Artifact, error) { + return draftArtifact(id, catalogID, newVersion), nil + }, + PatchFn: func(_, _, _ string) error { return nil }, + }, + Now: time.Now, + }) + require.NoError(t, err) + + t.Cleanup(func() { _ = e3.Close() }) + + result3, err := e3.Run() + require.NoError(t, err) + require.NotNil(t, result3) + + // a.py is the one real upload; b.py must STILL be absent — the + // rollback-covered path produces no false upload even in a sync that + // genuinely has work to do. + require.Len(t, e3.plan.Uploads, 1, + "the modified a.py must be the one real upload — b.py must not appear") + assert.Equal(t, "a.py", e3.plan.Uploads[0].Path) + assert.Equal(t, 1, result3.UploadedCount, + "the real-work sync must report the upload it performed") + assert.Equal(t, 1, fake2.ApplyStageCalls(), + "the real-work sync must execute its plan") + + // Config must now converge to the version the manifest already records — + // the honest self-healing path: a sync with real work through Run(), + // never a forced Execute on an empty plan. convergedCfg, err := wapi.LoadConfig(s.dir) require.NoError(t, err) require.NotNil(t, convergedCfg.LastSyncedVersionID) assert.Equal(t, newVersion, *convergedCfg.LastSyncedVersionID, - "config must converge to the manifest's version on the next run") + "config must converge to the manifest's version once a sync has real work") // Three-way consistency after convergence: manifest hash == server // checksum for every file. From c6ca6369bc76a0a1571d4875878653bf18f728a8 Mon Sep 17 00:00:00 2001 From: AJ Alon Date: Mon, 31 Aug 2026 14:07:45 -0700 Subject: [PATCH 20/20] [RAPTOR-19525] docs(workload): state the real config-convergence path in the Phase 6 SaveConfig comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SaveConfig-failure bullet claimed the next sync's plan is empty and "the run merely converges config". That is not what production does: Run() short-circuits empty plans before Execute, so Phase 6 never runs on that next sync and config.json keeps the stale version. Config converges only when a later sync has real work. Reword the bullet to state the verified sequence: drift re-detected on every later sync, empty plan computed against the truthful advanced manifest, Run returning before Execute, Phase 6 skipped, config stale until a sync with real work executes — and why the asymmetry is safe (drift detection re-fires every run, so the window self-heals; the reverse direction is silent, permanent poisoning). Comment-only change; the behavior is already pinned by TestSaveConfigFailure_ManifestAdvanced_ConfigConvergesOnNextRealSync and the rollback-discard tests, which are the evidence the new wording is true. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- internal/workload/sync/phase6_state.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/workload/sync/phase6_state.go b/internal/workload/sync/phase6_state.go index cd6fb325f..3e82d5320 100644 --- a/internal/workload/sync/phase6_state.go +++ b/internal/workload/sync/phase6_state.go @@ -77,12 +77,19 @@ func phase6State(e *Engine) error { // self-healing. The manifest write is retried by that same sync. // // - SaveConfig fails: the manifest is already advanced while config - // still names the old version. The version mismatch makes the next - // sync detect drift and fetch the remote; BASE (the advanced - // manifest) truthfully describes that remote, so the plan is empty - // and the run merely converges config. The rollback dir is already - // gone by then — discarded at entry above — so no stale-restore can - // resurrect pre-sync bytes as false local edits. + // still names the old version. The version mismatch makes every + // later sync detect drift and fetch the real remote; BASE (the + // advanced manifest) truthfully describes that remote, so those + // syncs compute an empty plan. An empty plan never reaches this + // phase — Run returns before Execute on empty plans — so Phase 6 + // is skipped and config.json stays stale on each of those runs, + // converging only when a later sync has real work to execute. + // The rollback dir is already gone by then — discarded at entry + // above — so no stale-restore can resurrect pre-sync bytes as + // false local edits. The asymmetry is safe because it is loud: a + // manifest ahead of config re-triggers drift detection on every + // sync, so the window self-heals at the first sync with actual + // work; the reverse direction below is silent and never heals. // // The converse (config advanced, manifest stale) is the poisonous // direction: the next sync sees no drift, fast-paths, copies the stale