diff --git a/core/bazel/bazel.go b/core/bazel/bazel.go index 9180fa8e..7391331c 100644 --- a/core/bazel/bazel.go +++ b/core/bazel/bazel.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "time" buildpb "github.com/bazelbuild/buildtools/build_proto" @@ -182,3 +183,23 @@ func ensureBazelisk(ctx context.Context) (_ string, retErr error) { } return dest, nil } + +// OutputBase resolves the bazel executable and runs `bazel info output_base`, +// returning the absolute path to the output base directory. +func OutputBase(ctx context.Context, workspacePath, bazelCommand string) (string, error) { + resolved, err := detectBazelExecutable(ctx, bazelCommand) + if err != nil { + return "", fmt.Errorf("detect bazel: %w", err) + } + cmd := execcmd.CommandContext(ctx, resolved, "info", "output_base") + cmd.Dir = workspacePath + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("bazel info output_base: %w", err) + } + result := strings.TrimSpace(string(out)) + if result == "" { + return "", fmt.Errorf("bazel info output_base returned empty path") + } + return result, nil +} diff --git a/core/targethasher/BUILD.bazel b/core/targethasher/BUILD.bazel index ea505f89..7b304093 100644 --- a/core/targethasher/BUILD.bazel +++ b/core/targethasher/BUILD.bazel @@ -4,11 +4,13 @@ go_library( name = "targethasher", srcs = [ "graph.go", + "markers.go", "sourcehasher.go", ], importpath = "github.com/uber/tango/core/targethasher", visibility = ["//visibility:public"], deps = [ + "//core/bazel", "@com_github_bazelbuild_buildtools//build_proto", "@com_github_bazelbuild_buildtools//labels", "@com_github_deckarep_golang_set_v2//:golang-set", diff --git a/core/targethasher/graph.go b/core/targethasher/graph.go index 2610ad1f..f37f4330 100644 --- a/core/targethasher/graph.go +++ b/core/targethasher/graph.go @@ -78,6 +78,13 @@ type HashConfig struct { // AllTargetsFiles lists repo-relative paths whose hashes should be // extracted from KnownSourceHashes into Result.AllTargetsFileHashes. AllTargetsFiles []string + // RepoMarkerHashes maps canonical bzlmod repo names to repo rule input + // hashes read from Bazel's marker files ($(output_base)/external/@repo.marker). + // Used by HashExternalTargetsBzlmod to pre-hash external source files + // without reading their content from disk. The marker hash changes on any + // dependency upgrade, so repos whose canonical name stays the same across + // versions (e.g. "protobuf+") are still correctly detected as changed. + RepoMarkerHashes map[string][]byte } // Target contains information about the hash for a single target @@ -155,7 +162,7 @@ func FromProto(ctx context.Context, r *buildpb.QueryResult, workspaceroot string result, err := fromProto(ctx, r, &diskHashHelper{ workspaceroot: workspaceroot, knownFileHashes: hashConfig.KnownSourceHashes, - }, workspaceroot, fullHashRepos, set.NewSet(hashConfig.SequentialHashTargets...), excludedRegex, hashConfig.UseBzlmod) + }, workspaceroot, fullHashRepos, set.NewSet(hashConfig.SequentialHashTargets...), excludedRegex, hashConfig.UseBzlmod, hashConfig.RepoMarkerHashes) if err != nil { return result, err } @@ -176,7 +183,7 @@ func FromProto(ctx context.Context, r *buildpb.QueryResult, workspaceroot string // FromProtoNoHash calculates a DAG graph based on a query result. It does not calculate hashes for targets. func FromProtoNoHash(ctx context.Context, r *buildpb.QueryResult) (Result, error) { - return fromProto(ctx, r, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false) + return fromProto(ctx, r, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false, nil) } // for external targets, url and urls attributes could cause non-deterministic hash values, @@ -386,6 +393,88 @@ func HashExternalTargets(ctx context.Context, r *buildpb.QueryResult, targets ma return nil } +// bzlmodRepoName extracts the canonical bzlmod repo name from a target label. +// For "@@rules_python++pip+foo//pkg:target" it returns "rules_python++pip+foo". +// Returns "" for non-bzlmod targets. +func bzlmodRepoName(targetName string) string { + if !strings.HasPrefix(targetName, "@@") { + return "" + } + // Strip leading "@@", then find "//" separator. + rest := targetName[2:] + idx := strings.Index(rest, "//") + if idx <= 0 { + return "" + } + return rest[:idx] +} + +// shouldCollapseToBzlmodRepo reports whether a bzlmod external target should +// be pre-hashed using the repo's marker file hash instead of hashing its +// file content during the DFS. Only source and generated files are collapsed; +// rule targets are left alone so dependency edges are preserved. +func shouldCollapseToBzlmodRepo(target *Target, repo string, fullHashRepos set.Set[string], excludedRegex []*regexp.Regexp) bool { + if target == nil { + return false + } + if repo == "" || fullHashRepos.Contains(repo) { + return false + } + if target.RuleType != SourceFileType && target.RuleType != GeneratedFileType { + return false + } + if target.Hash != nil { + return false + } + if isExcluded(target.Name, excludedRegex) { + return false + } + return true +} + +// HashExternalTargetsBzlmod pre-hashes bzlmod external source and +// generated file targets using hashes derived from Bazel's marker files. +// This is the bzlmod equivalent of legacy WORKSPACE HashExternalTargets. +// +// Marker files track both the repo rule's declarative inputs (version, +// URL, integrity) and per-file SHA-256 content hashes for local patches +// applied via single_version_override. readMarkerHash hashes all stable +// lines (skipping ENV) so the collapsed hash changes on dependency +// upgrades AND patch content modifications. +// +// Every external repo referenced in the query result should have a +// marker file after bazel query completes. Returns an error if a +// collapsible repo is missing its marker. +func HashExternalTargetsBzlmod(targets map[string]*Target, fullHashRepos set.Set[string], excludedRegex []*regexp.Regexp, repoMarkerHashes map[string][]byte) error { + if len(repoMarkerHashes) == 0 { + return nil + } + + repoHashes := make(map[string][]byte) + for _, target := range targets { + repo := bzlmodRepoName(target.Name) + if !shouldCollapseToBzlmodRepo(target, repo, fullHashRepos, excludedRegex) { + continue + } + + h, ok := repoHashes[repo] + if !ok { + markerHash, hasMarker := repoMarkerHashes[repo] + if !hasMarker || len(markerHash) == 0 { + return fmt.Errorf("bzlmod repo %q has targets in query but no marker file", repo) + } + rh := newHash() + rh.Write(markerHash) + h = rh.Sum(nil) + repoHashes[repo] = h + } + + target.Hash = h + target.HashWithoutDeps = h + } + return nil +} + // GetTopologicalRootsAndIdentifyBuildableRoots returns a list of topological roots and marks buildable roots in the target graph func GetTopologicalRootsAndIdentifyBuildableRoots(targets map[string]*Target) []string { // get targets that cannot be root, i.e. dependencies of some other targets @@ -419,7 +508,7 @@ func GetTopologicalRootsAndIdentifyBuildableRoots(targets map[string]*Target) [] return roots } -func fromProto(ctx context.Context, r *buildpb.QueryResult, hasher SourceHasher, workspaceroot string, fullHashRepos set.Set[string], sequentialHashTargets set.Set[string], excludedRegex []*regexp.Regexp, useBzlmod bool) (Result, error) { +func fromProto(ctx context.Context, r *buildpb.QueryResult, hasher SourceHasher, workspaceroot string, fullHashRepos set.Set[string], sequentialHashTargets set.Set[string], excludedRegex []*regexp.Regexp, useBzlmod bool, repoMarkerHashes map[string][]byte) (Result, error) { warns := make(map[string]error) // Build target graph with dependencies, but without hash and root information. targets, err := GetInternalTargetsWithoutHashAndRootInfo(ctx, r) @@ -427,13 +516,22 @@ func fromProto(ctx context.Context, r *buildpb.QueryResult, hasher SourceHasher, return EmptyResult(), err } - // add external rule targets (//external:*) to the same map and hash them - // no need for bzlmod because there's no //external:* rules, we will hash external source as is if !useBzlmod { + // Legacy WORKSPACE: add external rule targets (//external:*) to the + // map and hash them. No //external:* rules exist under bzlmod. if err := HashExternalTargets(ctx, r, targets, hasher, workspaceroot, fullHashRepos, warns, useBzlmod); err != nil { return EmptyResult(), err } + } else { + // Bzlmod: collapse external source/generated file targets using + // Bazel marker file hashes. Avoids visiting millions of individual + // pip-wheel files during the DFS — the bzlmod equivalent of legacy + // WORKSPACE //external:repo collapsing. + if err := HashExternalTargetsBzlmod(targets, fullHashRepos, excludedRegex, repoMarkerHashes); err != nil { + return EmptyResult(), err + } } + // get topological roots and update buildable roots info roots := GetTopologicalRootsAndIdentifyBuildableRoots(targets) diff --git a/core/targethasher/graph_test.go b/core/targethasher/graph_test.go index f3cebd5c..94ecd897 100644 --- a/core/targethasher/graph_test.go +++ b/core/targethasher/graph_test.go @@ -79,7 +79,7 @@ func TestContextCancellation(t *testing.T) { qr := &buildpb.QueryResult{ Target: []*buildpb.Target{&buildpb.Target{}}, } - result, err := fromProto(ctx, qr, nil, "", set.NewSet[string](), set.NewSet[string](), nil, false) + result, err := fromProto(ctx, qr, nil, "", set.NewSet[string](), set.NewSet[string](), nil, false, nil) assert.Equal(t, EmptyResult(), result) assert.ErrorIs(t, err, context.Canceled) @@ -107,7 +107,7 @@ func TestFromProtoSimpleRule(t *testing.T) { }, } - result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false) + result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false, nil) require.NoError(t, err) assert.Len(t, result.Targets, 1) @@ -138,7 +138,7 @@ func TestFromProtoWithDependencies(t *testing.T) { }, } - result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false) + result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false, nil) require.NoError(t, err) assert.Len(t, result.Targets, 2) @@ -176,7 +176,7 @@ func TestFromProtoWithExcludedRegex(t *testing.T) { // Exclude targets matching "//vendor:.*" excludedRegex := []*regexp.Regexp{regexp.MustCompile("//vendor:.*")} - result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), excludedRegex, false) + result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), excludedRegex, false, nil) require.NoError(t, err) assert.Len(t, result.Targets, 2) @@ -291,7 +291,7 @@ func TestFromProtoWithGeneratedFile(t *testing.T) { }, } - result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false) + result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), nil, false, nil) require.NoError(t, err) assert.Len(t, result.Targets, 2) @@ -382,6 +382,157 @@ func assertEqualTargetHash(t *testing.T, expected, actual Target) { assert.True(t, cmp.Equal(expected, actual, opt, ignore), cmp.Diff(expected, actual, opt)) } +func TestBzlmodRepoName(t *testing.T) { + tests := []struct { + name string + target string + want string + }{ + {"bzlmod source file", "@@rules_python++pip+foo//pkg:file.py", "rules_python++pip+foo"}, + {"bzlmod rule", "@@io_bazel_rules_go//go:def.bzl", "io_bazel_rules_go"}, + {"internal target", "//src/pkg:lib", ""}, + {"single @", "@repo//pkg:target", ""}, + {"no double slash", "@@repo", ""}, + {"empty repo name", "@@//pkg:target", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, bzlmodRepoName(tt.target)) + }) + } +} + +func TestHashExternalTargetsBzlmod(t *testing.T) { + markerA := []byte{0xaa, 0xbb, 0xcc} + markerB := []byte{0xdd, 0xee, 0xff} + markers := map[string][]byte{ + "repo_a": markerA, + "repo_b": markerB, + "rules_python++pip+foo": []byte{0x11, 0x22}, + } + + t.Run("collapses with marker hash", func(t *testing.T) { + targets := map[string]*Target{ + "@@repo_a//pkg:file1.py": {Name: "@@repo_a//pkg:file1.py", RuleType: SourceFileType, External: true}, + "@@repo_a//pkg:file2.py": {Name: "@@repo_a//pkg:file2.py", RuleType: SourceFileType, External: true}, + "@@repo_a//pkg:gen.go": {Name: "@@repo_a//pkg:gen.go", RuleType: GeneratedFileType, External: true}, + "//src:main": {Name: "//src:main", RuleType: "go_binary"}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, markers)) + + assert.NotNil(t, targets["@@repo_a//pkg:file1.py"].Hash) + assert.Equal(t, targets["@@repo_a//pkg:file1.py"].Hash, targets["@@repo_a//pkg:file2.py"].Hash) + assert.Equal(t, targets["@@repo_a//pkg:file1.py"].Hash, targets["@@repo_a//pkg:gen.go"].Hash) + assert.Nil(t, targets["//src:main"].Hash) + }) + + t.Run("errors on repos without marker", func(t *testing.T) { + targets := map[string]*Target{ + "@@no_marker_repo//pkg:file.py": {Name: "@@no_marker_repo//pkg:file.py", RuleType: SourceFileType, External: true}, + } + + err := HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, markers) + require.Error(t, err) + assert.Contains(t, err.Error(), "no_marker_repo") + }) + + t.Run("no-op when no markers provided at all", func(t *testing.T) { + targets := map[string]*Target{ + "@@repo_a//pkg:file.py": {Name: "@@repo_a//pkg:file.py", RuleType: SourceFileType, External: true}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, nil)) + assert.Nil(t, targets["@@repo_a//pkg:file.py"].Hash) + }) + + t.Run("skips rule targets", func(t *testing.T) { + targets := map[string]*Target{ + "@@repo_a//pkg:lib": {Name: "@@repo_a//pkg:lib", RuleType: "go_library", External: true}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, markers)) + assert.Nil(t, targets["@@repo_a//pkg:lib"].Hash) + }) + + t.Run("skips fullHashRepos", func(t *testing.T) { + targets := map[string]*Target{ + "@@repo_a//pkg:file.py": {Name: "@@repo_a//pkg:file.py", RuleType: SourceFileType, External: true}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet("", "repo_a"), nil, markers)) + assert.Nil(t, targets["@@repo_a//pkg:file.py"].Hash) + }) + + t.Run("skips excluded targets", func(t *testing.T) { + targets := map[string]*Target{ + "@@rules_python++pip+foo//pkg:file.whl": {Name: "@@rules_python++pip+foo//pkg:file.whl", RuleType: SourceFileType, External: true}, + "@@rules_python++pip+foo//pkg:module.py": {Name: "@@rules_python++pip+foo//pkg:module.py", RuleType: SourceFileType, External: true}, + } + excluded := []*regexp.Regexp{regexp.MustCompile(`\.whl$`)} + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), excluded, markers)) + + assert.Nil(t, targets["@@rules_python++pip+foo//pkg:file.whl"].Hash) + assert.NotNil(t, targets["@@rules_python++pip+foo//pkg:module.py"].Hash) + }) + + t.Run("different repos get different hashes", func(t *testing.T) { + targets := map[string]*Target{ + "@@repo_a//pkg:file.py": {Name: "@@repo_a//pkg:file.py", RuleType: SourceFileType, External: true}, + "@@repo_b//pkg:file.py": {Name: "@@repo_b//pkg:file.py", RuleType: SourceFileType, External: true}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, markers)) + assert.NotEqual(t, targets["@@repo_a//pkg:file.py"].Hash, targets["@@repo_b//pkg:file.py"].Hash) + }) + + t.Run("already hashed targets are skipped", func(t *testing.T) { + existing := []byte{1, 2, 3} + targets := map[string]*Target{ + "@@repo_a//pkg:file.py": {Name: "@@repo_a//pkg:file.py", RuleType: SourceFileType, External: true, Hash: existing}, + } + + require.NoError(t, HashExternalTargetsBzlmod(targets, set.NewSet(""), nil, markers)) + assert.Equal(t, existing, targets["@@repo_a//pkg:file.py"].Hash) + }) +} + +func TestFromProtoExcludedBzlmodTargetGetsEmptyHash(t *testing.T) { + // End-to-end: an excluded bzlmod source file should get empty hash, + // not a repo-name hash from the collapse. + qr := &buildpb.QueryResult{ + Target: []*buildpb.Target{ + { + Type: buildpb.Target_RULE.Enum(), + Rule: &buildpb.Rule{ + Name: StringPtr("//pkg:app"), + RuleClass: StringPtr("go_binary"), + RuleInput: []string{"@@dep++v1//lib:file.whl"}, + }, + }, + { + Type: buildpb.Target_SOURCE_FILE.Enum(), + SourceFile: &buildpb.SourceFile{Name: StringPtr("@@dep++v1//lib:file.whl")}, + }, + { + Type: buildpb.Target_SOURCE_FILE.Enum(), + SourceFile: &buildpb.SourceFile{Name: StringPtr("@@dep++v1//lib:module.py")}, + }, + }, + } + + excluded := []*regexp.Regexp{regexp.MustCompile(`\.whl$`)} + result, err := fromProto(context.Background(), qr, &noOpHasher{}, "", set.NewSet[string](), set.NewSet[string](), excluded, true, nil) + require.NoError(t, err) + + // Excluded .whl target should have empty hash (not a repo-name hash). + assert.Empty(t, result.Targets["@@dep++v1//lib:file.whl"].Hash) + // Non-excluded .py in same repo should still have a hash from collapse. + // (noOpHasher zeros all hashes, but the target should still exist) + assert.Contains(t, result.Targets, "@@dep++v1//lib:module.py") +} + func Test_fromProto(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.WithValue(context.Background(), struct{}{}, "source-hash") @@ -399,7 +550,7 @@ func Test_fromProto(t *testing.T) { q, err := bazel.FromFile("testdata/test.proto.bin") require.NoError(t, err) - a, err := fromProto(ctx, q, mockHasher, "", set.NewSet[string](), set.NewSet[string](), nil, true) + a, err := fromProto(ctx, q, mockHasher, "", set.NewSet[string](), set.NewSet[string](), nil, true, nil) require.NoError(t, err) assert.Empty(t, a.Warnings) diff --git a/core/targethasher/markers.go b/core/targethasher/markers.go new file mode 100644 index 00000000..3805fb70 --- /dev/null +++ b/core/targethasher/markers.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package targethasher + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/uber/tango/core/bazel" +) + +// ReadRepoMarkerHashes resolves the Bazel output base, then reads the +// external repo marker files and returns a map from canonical repo name +// to a hash derived from the stable lines of each marker file. +// This hash changes whenever the repo's version, URL, or patch file +// contents change. ENV lines are excluded for cross-environment stability. +func ReadRepoMarkerHashes(ctx context.Context, workspacePath, bazelCommand string) (map[string][]byte, error) { + outputBase, err := bazel.OutputBase(ctx, workspacePath, bazelCommand) + if err != nil { + return nil, fmt.Errorf("bazel output base: %w", err) + } + + markerDir := filepath.Join(outputBase, "external") + entries, err := os.ReadDir(markerDir) + if err != nil { + return nil, fmt.Errorf("read marker dir %s: %w", markerDir, err) + } + + hashes := make(map[string][]byte) + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".marker") { + continue + } + repo := strings.TrimPrefix(strings.TrimSuffix(name, ".marker"), "@") + if repo == "" { + continue + } + + h, err := readMarkerHash(filepath.Join(markerDir, name)) + if err != nil { + return nil, fmt.Errorf("read marker for repo %s: %w", repo, err) + } + if len(h) == 0 { + continue + } + hashes[repo] = h + } + + return hashes, nil +} + +// readMarkerHash computes a hash from the stable lines of a marker file. +// The first line is a hash of the repo rule's declarative inputs (URL, +// version, patch paths) but does NOT include the content hashes of patch +// files. Those appear on FILE: lines alongside their SHA-256 content +// hashes. ENV: lines are skipped because environment variables can differ +// between CI environments and would cause unnecessary hash instability. +func readMarkerHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + + h := newHash() + hasContent := false + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if strings.HasPrefix(line, "ENV:") { + continue + } + h.Write([]byte(line)) + hasContent = true + } + if err := scanner.Err(); err != nil { + return nil, err + } + if !hasContent { + return nil, nil + } + return h.Sum(nil), nil +} diff --git a/graphrunner/BUILD.bazel b/graphrunner/BUILD.bazel index 99f8b0a7..3adf7fd0 100644 --- a/graphrunner/BUILD.bazel +++ b/graphrunner/BUILD.bazel @@ -26,6 +26,7 @@ go_test( srcs = ["native_test.go"], embed = [":graphrunner"], deps = [ + "//config", "//core/bazel", "//core/bazel/bazelmock", "//core/git/gitmock", diff --git a/graphrunner/native.go b/graphrunner/native.go index bb2cde9b..0b8fcf2c 100644 --- a/graphrunner/native.go +++ b/graphrunner/native.go @@ -16,6 +16,7 @@ package graphrunner import ( "context" + "fmt" "time" "github.com/uber-go/tally" @@ -96,12 +97,29 @@ func (g *nativeGraphRunner) Compute(ctx context.Context, ws workspace.Workspace) // mutate the shared configuration's backing array. excludedRegex := append([]string(nil), g.config.ExcludedFiles...) excludedRegex = append(excludedRegex, g.extraExcludedFiles...) + + // Read marker files for bzlmod repos so the collapse uses content-aware + // hashes instead of just the repo name string. + var repoMarkerHashes map[string][]byte + if bzlmodEnabled { + markerStart := time.Now() + repoMarkerHashes, err = targethasher.ReadRepoMarkerHashes(ctx, ws.Path(), g.config.BazelCommandPath) + g.emitter.DurationHistogram(_opCompute, "marker_read_duration", metrics.FastDurationBuckets).RecordDuration(time.Since(markerStart)) + if err != nil { + return targethasher.EmptyResult(), fmt.Errorf("read repo marker hashes: %w", err) + } + if len(repoMarkerHashes) == 0 { + return targethasher.EmptyResult(), fmt.Errorf("bzlmod enabled but no repo marker hashes found") + } + } + hashConfig := targethasher.HashConfig{ KnownSourceHashes: knownSourceHashes, FullHashRepos: g.config.FullHashRepos, ExcludedRegex: excludedRegex, UseBzlmod: bzlmodEnabled, AllTargetsFiles: g.config.AllTargetsFiles, + RepoMarkerHashes: repoMarkerHashes, } hashStart := time.Now() diff --git a/graphrunner/native_test.go b/graphrunner/native_test.go index 363c19c1..fe1584b4 100644 --- a/graphrunner/native_test.go +++ b/graphrunner/native_test.go @@ -16,11 +16,15 @@ package graphrunner import ( "context" + "encoding/hex" + "os" + "path/filepath" "testing" buildpb "github.com/bazelbuild/buildtools/build_proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/uber/tango/config" "github.com/uber/tango/core/bazel" "github.com/uber/tango/core/bazel/bazelmock" gitmock "github.com/uber/tango/core/git/gitmock" @@ -28,6 +32,8 @@ import ( "go.uber.org/mock/gomock" ) +func boolPtr(b bool) *bool { return &b } + func TestCompute_CallsBazelAndReturnsResult(t *testing.T) { ctrl := gomock.NewController(t) bazelMock := bazelmock.NewMockBazel(ctrl) @@ -47,7 +53,7 @@ func TestCompute_CallsBazelAndReturnsResult(t *testing.T) { gr := NewNativeGraphRunner(NativeGraphRunnerParams{ BazelClient: bazelMock, GitClient: gitMock, - // leave HashConfig zero; not asserted here + Config: config.RepositoryConfig{BzlmodEnabled: boolPtr(false)}, }) ws := workspace.NewWorkspace(workspace.WorkspaceParams{ Path: "/tmp/ws", @@ -62,6 +68,77 @@ func TestCompute_CallsBazelAndReturnsResult(t *testing.T) { assert.Equal(t, ruleClass, res.Targets[ruleName].Rule.GetRuleClass()) } +func TestCompute_BzlmodCollapsesExternalTargets(t *testing.T) { + // Set up a fake output base with marker files so the bzlmod + // collapse path can read them without a real bazel installation. + outputBase := t.TempDir() + markerDir := filepath.Join(outputBase, "external") + require.NoError(t, os.MkdirAll(markerDir, 0o755)) + + markerHash := hex.EncodeToString([]byte("fake-hash-for-test")) + require.NoError(t, os.WriteFile( + filepath.Join(markerDir, "@myrepo.marker"), + []byte(markerHash+"\n"), + 0o644, + )) + + // Create a fake bazel script that prints the output base. + fakeBazel := filepath.Join(t.TempDir(), "fake-bazel") + require.NoError(t, os.WriteFile(fakeBazel, []byte("#!/bin/sh\necho "+outputBase+"\n"), 0o755)) + + ctrl := gomock.NewController(t) + bazelMock := bazelmock.NewMockBazel(ctrl) + gitMock := gitmock.NewMockInterface(ctrl) + gitMock.EXPECT().FileHashes(gomock.Any(), gomock.Any()).Return(map[string][]byte{}, nil) + + srcName := "@@myrepo//pkg:file.go" + srcType := "source file" + ruleName := "//:a" + ruleClass := "go_library" + bazelMock.EXPECT().ExecuteQuery(gomock.Any(), gomock.Any()).Return(&bazel.QueryResponse{Result: &buildpb.QueryResult{Target: []*buildpb.Target{ + { + Type: buildpb.Target_SOURCE_FILE.Enum(), + SourceFile: &buildpb.SourceFile{ + Name: &srcName, + }, + }, + { + Type: buildpb.Target_RULE.Enum(), + Rule: &buildpb.Rule{ + Name: &ruleName, + RuleClass: &ruleClass, + }, + }, + }}}, nil) + + gr := NewNativeGraphRunner(NativeGraphRunnerParams{ + BazelClient: bazelMock, + GitClient: gitMock, + Config: config.RepositoryConfig{ + BzlmodEnabled: boolPtr(true), + BazelCommandPath: fakeBazel, + }, + }) + ws := workspace.NewWorkspace(workspace.WorkspaceParams{ + Path: t.TempDir(), + }) + + res, err := gr.Compute(context.Background(), ws) + require.NoError(t, err) + require.NotNil(t, res) + + // The external source file should have been collapsed with a hash + // derived from the marker file, not left nil. + extTarget, ok := res.Targets[srcName] + require.True(t, ok, "external target %q should be in results", srcName) + assert.NotNil(t, extTarget.Hash, "collapsed external target should have a hash") + assert.Equal(t, srcType, extTarget.RuleType) + + // The internal rule target should also be present. + _, ok = res.Targets[ruleName] + assert.True(t, ok, "internal target %q should be in results", ruleName) +} + func TestCompute_PropagatesError(t *testing.T) { ctrl := gomock.NewController(t) bazelMock := bazelmock.NewMockBazel(ctrl)