From a0c14c6aab469684a58921b1b8b1abcfa8f02ba5 Mon Sep 17 00:00:00 2001 From: David Marby Date: Wed, 18 Mar 2026 21:14:54 +0100 Subject: [PATCH] Make the fetcher update transitive maven deps in buf.plugin.yml So that we don't error out on duplicate maven deps. When a dependency plugin's Maven version is updated (e.g. protocolbuffers/java moves from v33.5 to v34.0), consumers that have stale pins (e.g. grpc/java still pinning protobuf-java:4.33.5 from the previous copy) would hit version conflicts during deduplication. Now RegenerateMavenDeps walks the transitive dep tree, collects Maven artifact versions, and updates the plugin's buf.plugin.yaml before deduplication. This ensures all downstream consumers have consistent versions and dedup succeeds. --- internal/maven/regenerate.go | 141 +++++++++++++++++++++++++++++- internal/maven/regenerate_test.go | 96 ++++++++++++++++++++ 2 files changed, 235 insertions(+), 2 deletions(-) diff --git a/internal/maven/regenerate.go b/internal/maven/regenerate.go index 960cb59ca..410956f9d 100644 --- a/internal/maven/regenerate.go +++ b/internal/maven/regenerate.go @@ -4,14 +4,19 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin/bufremotepluginconfig" + "golang.org/x/mod/semver" ) // RegenerateMavenDeps processes a Maven plugin version directory by // merging transitive deps, deduplicating, and rendering POM to a -// pom.xml file. Returns nil without changes if the plugin has no -// Maven registry config. +// pom.xml file. When transitive deps bring in newer versions of +// artifacts already pinned in the plugin's buf.plugin.yaml, the YAML +// file is updated first so that deduplication and POM generation see +// consistent versions. Returns nil without changes if the plugin has +// no Maven registry config. func RegenerateMavenDeps(pluginVersionDir, pluginsDir string) error { yamlPath := filepath.Join(pluginVersionDir, "buf.plugin.yaml") pluginConfig, err := bufremotepluginconfig.ParseConfig(yamlPath) @@ -21,6 +26,23 @@ func RegenerateMavenDeps(pluginVersionDir, pluginsDir string) error { if pluginConfig.Registry == nil || pluginConfig.Registry.Maven == nil { return nil } + // Collect the versions declared by transitive deps so we can detect + // stale pins in the plugin's own buf.plugin.yaml. + transitiveDeps, err := collectTransitiveMavenDeps(pluginConfig, pluginsDir) + if err != nil { + return fmt.Errorf("collecting transitive deps: %w", err) + } + // Update buf.plugin.yaml if any direct dep versions are older than + // what transitive deps declare. + if err := updateBufPluginYAML(yamlPath, pluginConfig.Registry.Maven, transitiveDeps); err != nil { + return fmt.Errorf("updating buf.plugin.yaml: %w", err) + } + // Re-parse the (potentially updated) YAML so the in-memory config + // reflects the updated versions. + pluginConfig, err = bufremotepluginconfig.ParseConfig(yamlPath) + if err != nil { + return err + } if err := MergeTransitiveDeps(pluginConfig, pluginsDir); err != nil { return fmt.Errorf("merging transitive deps: %w", err) } @@ -37,3 +59,118 @@ func RegenerateMavenDeps(pluginVersionDir, pluginsDir string) error { } return nil } + +// mavenDepKey returns the deduplication key for a Maven dependency +// (groupId:artifactId, optionally with classifier). +func mavenDepKey(dep bufremotepluginconfig.MavenDependencyConfig) string { + key := dep.GroupID + ":" + dep.ArtifactID + if dep.Classifier != "" { + key += ":" + dep.Classifier + } + return key +} + +// collectTransitiveMavenDeps walks the plugin's dependency tree and +// returns a map of artifact key -> version for every Maven dep found +// in transitive dependencies. This does not mutate pluginConfig. +func collectTransitiveMavenDeps( + pluginConfig *bufremotepluginconfig.Config, + pluginsDir string, +) (map[string]string, error) { + versions := make(map[string]string) + visited := make(map[string]bool) + if err := collectTransitiveMavenDepsRecursive(pluginConfig, pluginsDir, visited, versions); err != nil { + return nil, err + } + return versions, nil +} + +func collectTransitiveMavenDepsRecursive( + pluginConfig *bufremotepluginconfig.Config, + pluginsDir string, + visited map[string]bool, + versions map[string]string, +) error { + for _, dep := range pluginConfig.Dependencies { + depKey := dep.IdentityString() + ":" + dep.Version() + if visited[depKey] { + continue + } + visited[depKey] = true + depPath := filepath.Join( + pluginsDir, dep.Owner(), dep.Plugin(), + dep.Version(), "buf.plugin.yaml", + ) + depConfig, err := bufremotepluginconfig.ParseConfig(depPath) + if err != nil { + return fmt.Errorf("loading dep config %s from %s: %w", depKey, depPath, err) + } + if err := collectTransitiveMavenDepsRecursive(depConfig, pluginsDir, visited, versions); err != nil { + return err + } + if depConfig.Registry == nil || depConfig.Registry.Maven == nil { + continue + } + for _, d := range depConfig.Registry.Maven.Deps { + versions[mavenDepKey(d)] = d.Version + } + for _, rt := range depConfig.Registry.Maven.AdditionalRuntimes { + for _, d := range rt.Deps { + versions[mavenDepKey(d)] = d.Version + } + } + } + return nil +} + +// updateBufPluginYAML rewrites buf.plugin.yaml when transitive deps +// declare newer versions of artifacts already pinned in the plugin's +// Maven config. It performs targeted text replacements of the Maven +// dep strings (group:artifact:oldVer -> group:artifact:newVer) so +// that comments and formatting are preserved. +func updateBufPluginYAML( + yamlPath string, + maven *bufremotepluginconfig.MavenRegistryConfig, + transitiveDeps map[string]string, +) error { + replacements := make(map[string]string) // "group:artifact:old" -> "group:artifact:new" + collectReplacements := func(deps []bufremotepluginconfig.MavenDependencyConfig) { + for _, dep := range deps { + key := mavenDepKey(dep) + transitiveVersion, ok := transitiveDeps[key] + if !ok || transitiveVersion == dep.Version { + continue + } + // Only upgrade if the transitive version is higher. + oldSemver := "v" + dep.Version + newSemver := "v" + transitiveVersion + if !semver.IsValid(oldSemver) || !semver.IsValid(newSemver) { + continue + } + if semver.Compare(newSemver, oldSemver) <= 0 { + continue + } + ref := dep.GroupID + ":" + dep.ArtifactID + if dep.Classifier != "" { + ref += ":" + dep.Classifier + } + replacements[ref+":"+dep.Version] = ref + ":" + transitiveVersion + } + } + collectReplacements(maven.Deps) + for _, rt := range maven.AdditionalRuntimes { + collectReplacements(rt.Deps) + } + if len(replacements) == 0 { + return nil + } + content, err := os.ReadFile(yamlPath) + if err != nil { + return err + } + result := string(content) + for oldRef, newRef := range replacements { + result = strings.ReplaceAll(result, oldRef, newRef) + } + return os.WriteFile(yamlPath, []byte(result), 0644) //nolint:gosec +} diff --git a/internal/maven/regenerate_test.go b/internal/maven/regenerate_test.go index 08ed9aae1..bfac3e512 100644 --- a/internal/maven/regenerate_test.go +++ b/internal/maven/regenerate_test.go @@ -10,6 +10,102 @@ import ( "github.com/stretchr/testify/require" ) +func TestRegenerateMavenDepsUpdatesStaleVersions(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + + // Simulate the grpc/java scenario: the dep plugin (protocolbuffers/java) + // was updated to v34.0 which uses protobuf-java:4.34.0, but the + // grpc/java plugin still has stale pins at 4.33.5 from the previous + // version's copy. + baseDir := filepath.Join(tmpDir, "plugins", "protocolbuffers", "java", "v34.0") + require.NoError(t, os.MkdirAll(baseDir, 0755)) + baseYAML := `version: v1 +name: buf.build/protocolbuffers/java +plugin_version: v34.0 +output_languages: + - java +registry: + maven: + deps: + - com.google.protobuf:protobuf-java:4.34.0 + additional_runtimes: + - name: lite + deps: + - com.google.protobuf:protobuf-javalite:4.34.0 + - build.buf:protobuf-javalite:4.34.0 + opts: [lite] +` + require.NoError(t, os.WriteFile(filepath.Join(baseDir, "buf.plugin.yaml"), []byte(baseYAML), 0644)) + + consumerDir := filepath.Join(tmpDir, "plugins", "grpc", "java", "v1.80.0") + require.NoError(t, os.MkdirAll(consumerDir, 0755)) + // This has the dep updated to v34.0 but Maven pins still at 4.33.5 + // (the old version from the copy step). + consumerYAML := `version: v1 +name: buf.build/grpc/java +plugin_version: v1.80.0 +source_url: https://github.com/grpc/grpc-java +description: Generates Java client and server stubs for the gRPC framework. +deps: + - plugin: buf.build/protocolbuffers/java:v34.0 +output_languages: + - java +spdx_license_id: Apache-2.0 +license_url: https://github.com/grpc/grpc-java/blob/v1.80.0/LICENSE +registry: + maven: + deps: + - io.grpc:grpc-core:1.80.0 + - io.grpc:grpc-protobuf:1.80.0 + - io.grpc:grpc-stub:1.80.0 + # Add direct dependency on newer protobuf as gRPC is still on 3.25.8 + - com.google.protobuf:protobuf-java:4.33.5 + additional_runtimes: + - name: lite + deps: + - io.grpc:grpc-core:1.80.0 + - io.grpc:grpc-protobuf-lite:1.80.0 + - io.grpc:grpc-stub:1.80.0 + # Add direct dependency on newer protobuf as gRPC is still on 3.25.8 + - com.google.protobuf:protobuf-javalite:4.33.5 + - build.buf:protobuf-javalite:4.33.5 + opts: [lite] +` + require.NoError(t, os.WriteFile(filepath.Join(consumerDir, "buf.plugin.yaml"), []byte(consumerYAML), 0644)) + + pluginsDir := filepath.Join(tmpDir, "plugins") + err := RegenerateMavenDeps(consumerDir, pluginsDir) + require.NoError(t, err) + + // Verify buf.plugin.yaml was updated with the correct versions. + updatedYAML, err := os.ReadFile(filepath.Join(consumerDir, "buf.plugin.yaml")) + require.NoError(t, err) + yamlStr := string(updatedYAML) + assert.Contains(t, yamlStr, "com.google.protobuf:protobuf-java:4.34.0") + assert.NotContains(t, yamlStr, "com.google.protobuf:protobuf-java:4.33.5") + assert.Contains(t, yamlStr, "com.google.protobuf:protobuf-javalite:4.34.0") + assert.NotContains(t, yamlStr, "com.google.protobuf:protobuf-javalite:4.33.5") + assert.Contains(t, yamlStr, "build.buf:protobuf-javalite:4.34.0") + assert.NotContains(t, yamlStr, "build.buf:protobuf-javalite:4.33.5") + // grpc deps should be unchanged + assert.Contains(t, yamlStr, "io.grpc:grpc-core:1.80.0") + // Comments should be preserved + assert.Contains(t, yamlStr, "# Add direct dependency on newer protobuf") + + // Verify pom.xml has the correct versions. + pomBytes, err := os.ReadFile(filepath.Join(consumerDir, "pom.xml")) + require.NoError(t, err) + var pom pomProject + require.NoError(t, xml.Unmarshal(pomBytes, &pom)) + var depVersions []string + for _, dep := range pom.Dependencies { + depVersions = append(depVersions, dep.GroupID+":"+dep.ArtifactID+":"+dep.Version) + } + assert.Contains(t, depVersions, "com.google.protobuf:protobuf-java:4.34.0") + assert.Contains(t, depVersions, "io.grpc:grpc-core:1.80.0") +} + func TestRegenerateMavenDeps(t *testing.T) { t.Parallel() tmpDir := t.TempDir()