From d581811be3f2e67120daab8c885ec7ace894539b Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Wed, 22 Jul 2026 16:05:03 +0200 Subject: [PATCH 1/3] fix(autobuilder): drop self-shadowing dependencies, resolve version conflicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways the project model handed the analyzer several copies of one class and left the choice to classpath lookup order. Self-shadowing dependencies. A project routinely depends on an artifact that repackages its own modules — a client bundle, a shaded jar, an earlier release of the module. Conductor declares both `project(':conductor-common')` and `org.conductoross:conductor-client`, and the latter ships 67 of common's classes. When lookup lands on the dependency copy, project code becomes un-analyzable library code and taint dies at every call into it. Drop any dependency that ships a class the project itself compiles: the project's own output already provides it. On conductor this removes both conductor-client jars, and no project class references any of the 468 classes that live only in them. Version conflicts. Dependencies are collected from the dependency graph across every module and configuration, so an artifact survives at each version any module resolved — 225 artifacts at 2..9 versions on conductor, 428 redundant jars holding 184918 classes. Keep one version per group:artifact, highest wins, the way the build tool resolves it. Conductor model: 1178 -> 756 dependency jars, 1.8G -> 1.5G, with an identical finding set (14 taint + 2 syntactic). Note this is a correctness and model-size change, not a performance one: a single-rule probe moved 1020s/18.0GB -> 965s/17.9GB, i.e. noise. Analysis cost on that project is dominated by container access-path growth in the dataflow engine, not by model size. Verified: autobuilder unit tests; rule-tests green on a model rebuilt with this autobuilder (falsePositive=falseNegative=skipped=0, success=338). --- .../org/opentaint/project/ArtifactVersions.kt | 69 ++++++++++++ .../project/GradleProjectResolver.kt | 29 +++-- .../opentaint/project/ProjectAutoBuilder.kt | 1 + .../project/ShadowingDependencies.kt | 76 +++++++++++++ .../opentaint/project/ArtifactVersionsTest.kt | 74 +++++++++++++ .../project/ShadowingDependenciesTest.kt | 102 ++++++++++++++++++ 6 files changed, 344 insertions(+), 7 deletions(-) create mode 100644 core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt create mode 100644 core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt create mode 100644 core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt create mode 100644 core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt new file mode 100644 index 000000000..c3a8af068 --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt @@ -0,0 +1,69 @@ +package org.opentaint.project + +/** + * Version comparison in the spirit of Maven/Gradle conflict resolution: segments are compared one by + * one, numerically where both sides are numeric, and a trailing qualifier (`-rc3`, `-SNAPSHOT`) makes + * a version *lower* than the same version without it. + */ +internal fun compareArtifactVersions(left: String, right: String): Int { + val leftParts = left.split(*VERSION_SEPARATORS) + val rightParts = right.split(*VERSION_SEPARATORS) + + for (i in 0 until maxOf(leftParts.size, rightParts.size)) { + val l = leftParts.getOrNull(i) + val r = rightParts.getOrNull(i) + + // one version ran out of segments: a numeric tail means a later release (1.2.1 > 1.2), + // a qualifier tail means a pre-release of the shorter one (1.2 > 1.2-rc1) + if (l == null) return if (r!!.isNumeric()) -1 else 1 + if (r == null) return if (l.isNumeric()) 1 else -1 + + val cmp = when { + l.isNumeric() && r.isNumeric() -> l.toBigInteger().compareTo(r.toBigInteger()) + l.isNumeric() -> 1 + r.isNumeric() -> -1 + else -> l.compareTo(r) + } + if (cmp != 0) return cmp + } + + return 0 +} + +/** + * Keeps a single version of every artifact — the highest — the way a build tool resolves a conflict. + * Without this the model carries every version any module resolved, which both inflates it and leaves + * the choice between same-named classes to classpath lookup order. + * + * Order is preserved: an artifact keeps the position of its first occurrence. + */ +internal fun List.singleVersionPerArtifact( + artifact: (T) -> Pair, + version: (T) -> String, + onDropped: (kept: T, dropped: T) -> Unit = { _, _ -> }, +): List { + val best = LinkedHashMap, T>() + + for (dependency in this) { + val key = artifact(dependency) + val current = best[key] + + if (current == null) { + best[key] = dependency + continue + } + + if (compareArtifactVersions(version(dependency), version(current)) > 0) { + best[key] = dependency + onDropped(dependency, current) + } else { + onDropped(current, dependency) + } + } + + return best.values.toList() +} + +private val VERSION_SEPARATORS = charArrayOf('.', '-', '_', '+') + +private fun String.isNumeric(): Boolean = isNotEmpty() && all { it.isDigit() } diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt index 7d4d0a051..58afe5ee0 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt @@ -157,15 +157,30 @@ class GradleProjectResolver( fun resolveDependenciesJars(): List { val allDependenciesInfo = dependenciesInfo.entries.sortedBy { it.key } - val resolvedDirectDependencies = allDependenciesInfo - .filter { it.key in directDependencies } - .mapNotNull { resolveJarPath(it.value) } + val directFirst = allDependenciesInfo.filter { it.key in directDependencies }.map { it.value } + + allDependenciesInfo.filterNot { it.key in directDependencies }.map { it.value } + + // Dependencies are collected across every module of the build, so the same artifact shows + // up at each version any module resolved. Resolve that conflict the way the build tool + // does — one version per artifact — instead of handing the analyzer several copies of the + // same classes and leaving the choice to classpath lookup order. + val conflictFree = directFirst.singleVersionPerArtifact( + artifact = { it.groupId to it.artifactId }, + version = { it.version }, + onDropped = { kept, dropped -> + logger.debug { + "Dependency conflict on ${dropped.groupId}:${dropped.artifactId}: " + + "keeping ${kept.version}, dropping ${dropped.version}" + } + } + ) - val resolvedIndirectDependencies = allDependenciesInfo - .filter { it.key !in directDependencies } - .mapNotNull { resolveJarPath(it.value) } + val droppedCount = directFirst.size - conflictFree.size + if (droppedCount > 0) { + logger.info { "Resolved dependency version conflicts: dropped $droppedCount duplicate artifact versions" } + } - return resolvedDirectDependencies + resolvedIndirectDependencies + return conflictFree.mapNotNull { resolveJarPath(it) } } private fun resolveJarPath(dependency: GradleDependencyInfo): Path? { diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt index b06786da7..73eb80362 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt @@ -47,6 +47,7 @@ class ProjectAutoBuilder : CliWithLogger() { } val javaProjects = resolvedProject?.let { Project.flattenJavaProject(it) }.orEmpty() + .dropDependenciesShadowingProjectClasses() val goProjects = GoProjectResolver.resolveProject(projectRoot, resolverWorkDir) if (javaProjects.isEmpty() && goProjects.isEmpty()) { diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt new file mode 100644 index 000000000..978cb1443 --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt @@ -0,0 +1,76 @@ +package org.opentaint.project + +import mu.KLogging +import java.nio.file.Path +import java.util.zip.ZipFile +import kotlin.io.path.extension +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.name +import kotlin.io.path.relativeTo +import kotlin.io.path.walk + +private val logger = object : KLogging() {}.logger + +/** + * Drops dependencies that ship the project's own classes. + * + * A project routinely depends on a published artifact that repackages its own modules (a client + * bundle, a shaded jar, the previous release of the module itself). The project's compiled output + * already carries those classes, so the dependency copy adds nothing — but it does make the class + * name ambiguous on the classpath, and a lookup that lands on the dependency copy turns project code + * into un-analyzable library code, silently killing every taint flow through it. + */ +fun List.dropDependenciesShadowingProjectClasses(): List { + val projectClasses = flatMap { it.modules } + .flatMap { it.moduleClasses } + .flatMapTo(mutableSetOf()) { classNamesOf(it) } + + if (projectClasses.isEmpty()) return this + + val shadowingDependencies = flatMap { it.dependencies } + .distinct() + .mapNotNull { dependency -> + val shadowed = classNamesOf(dependency).count { it in projectClasses } + if (shadowed == 0) null else dependency to shadowed + } + .toMap() + + if (shadowingDependencies.isEmpty()) return this + + for ((dependency, shadowed) in shadowingDependencies) { + logger.warn { + "Dependency ${dependency.name} ships $shadowed classes the project itself compiles; " + + "dropping it from the project model in favour of the project's own output" + } + } + + return map { project -> + project.copy(dependencies = project.dependencies.filter { it !in shadowingDependencies }) + } +} + +@OptIn(kotlin.io.path.ExperimentalPathApi::class) +private fun classNamesOf(path: Path): Set = when { + path.isDirectory() -> path.walk() + .filter { it.extension == CLASS_EXTENSION } + .mapTo(mutableSetOf()) { it.relativeTo(path).toString().toClassName() } + + path.isRegularFile() -> runCatching { + ZipFile(path.toFile()).use { zip -> + zip.entries().asSequence() + .filter { !it.isDirectory && it.name.endsWith(".$CLASS_EXTENSION") } + .mapTo(mutableSetOf()) { it.name.toClassName() } + } + }.getOrElse { + logger.warn { "Cannot read classes of $path: ${it.message}" } + emptySet() + } + + else -> emptySet() +} + +private fun String.toClassName(): String = + removeSuffix(".$CLASS_EXTENSION").replace('\\', '.').replace('/', '.') + +private const val CLASS_EXTENSION = "class" diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt new file mode 100644 index 000000000..80fdb862c --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt @@ -0,0 +1,74 @@ +package org.opentaint.project + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ArtifactVersionsTest { + + private fun assertGreater(left: String, right: String) { + assertTrue(compareArtifactVersions(left, right) > 0, "expected $left > $right") + assertTrue(compareArtifactVersions(right, left) < 0, "expected $right < $left") + } + + @Test + fun `numeric segments compare numerically, not lexicographically`() { + assertGreater("1.10.0", "1.9.0") + assertGreater("4.2.0", "4.0.10") + assertEquals(0, compareArtifactVersions("1.2.3", "1.2.3")) + } + + @Test + fun `a longer numeric version is greater`() { + assertGreater("1.2.1", "1.2") + } + + @Test + fun `a qualifier makes a version pre-release`() { + assertGreater("4.2.0", "4.2.0-rc3") + assertGreater("1.5.21", "1.5.21-SNAPSHOT") + assertGreater("4.2.0-rc3", "4.0.10") + } + + @Test + fun `qualifiers compare to each other lexicographically`() { + assertGreater("1.0-rc2", "1.0-beta1") + } + + @Test + fun `one version per artifact is kept, the highest`() { + val deps = listOf( + Triple("com.google.guava", "guava", "31.0"), + Triple("ch.qos.logback", "logback-classic", "1.5.18"), + Triple("com.google.guava", "guava", "33.2"), + Triple("ch.qos.logback", "logback-classic", "1.5.21"), + Triple("com.google.guava", "guava", "30.1"), + ) + val dropped = mutableListOf>() + + val kept = deps.singleVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + onDropped = { _, d -> dropped += d } + ) + + assertEquals( + listOf( + Triple("com.google.guava", "guava", "33.2"), + Triple("ch.qos.logback", "logback-classic", "1.5.21"), + ), + kept, + "must keep the highest version of each artifact, in first-seen order" + ) + assertEquals(3, dropped.size) + } + + @Test + fun `distinct artifacts of the same group are kept apart`() { + val deps = listOf( + Triple("org.springframework", "spring-core", "6.1.0"), + Triple("org.springframework", "spring-web", "6.1.0"), + ) + assertEquals(deps, deps.singleVersionPerArtifact({ it.first to it.second }, { it.third })) + } +} diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt new file mode 100644 index 000000000..79137685f --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt @@ -0,0 +1,102 @@ +package org.opentaint.project + +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.createDirectories +import kotlin.io.path.createTempDirectory +import kotlin.io.path.outputStream +import kotlin.io.path.writeBytes +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +class ShadowingDependenciesTest { + + private val tempDir: Path = createTempDirectory("shadowing-dependencies") + + @AfterTest + fun cleanup() { + tempDir.toFile().deleteRecursively() + } + + private fun classesDir(name: String, vararg classes: String): Path { + val dir = tempDir.resolve(name) + for (cls in classes) { + val file = dir.resolve(cls.replace('.', '/') + ".class") + file.parent.createDirectories() + file.writeBytes(byteArrayOf(0xCA.toByte(), 0xFE.toByte(), 0xBA.toByte(), 0xBE.toByte())) + } + dir.createDirectories() + return dir + } + + private fun jar(name: String, vararg classes: String): Path { + val path = tempDir.resolve(name) + path.parent.createDirectories() + ZipOutputStream(path.outputStream()).use { zip -> + for (cls in classes) { + zip.putNextEntry(ZipEntry(cls.replace('.', '/') + ".class")) + zip.write(byteArrayOf(0xCA.toByte(), 0xFE.toByte(), 0xBA.toByte(), 0xBE.toByte())) + zip.closeEntry() + } + } + return path + } + + private fun project(modules: List, dependencies: List) = JavaProject( + sourceRoot = tempDir, + modules = listOf(ProjectModuleClasses(moduleSourceRoot = tempDir, moduleClasses = modules)), + dependencies = dependencies, + ) + + @Test + fun `a dependency republishing project classes is dropped`() { + val moduleClasses = classesDir("module", "com.example.Dto", "com.example.Service") + val ownArtifact = jar("example-client-1.0.jar", "com.example.Dto", "com.example.client.Api") + val thirdParty = jar("guava-33.2.jar", "com.google.common.collect.Lists") + + val filtered = listOf(project(listOf(moduleClasses), listOf(ownArtifact, thirdParty))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(listOf(thirdParty), filtered.single().dependencies) + } + + @Test + fun `a dependency sharing no class is kept`() { + val moduleClasses = classesDir("module", "com.example.Dto") + val thirdParty = jar("guava-33.2.jar", "com.google.common.collect.Lists") + + val filtered = listOf(project(listOf(moduleClasses), listOf(thirdParty))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(listOf(thirdParty), filtered.single().dependencies) + } + + @Test + fun `shadowing is judged against every module of every project`() { + // the jar shadows a class of the *other* project's module, and must be dropped from both + val moduleA = classesDir("moduleA", "com.example.a.Dto") + val moduleB = classesDir("moduleB", "com.example.b.Dto") + val shadowing = jar("bundle-1.0.jar", "com.example.b.Dto", "com.example.bundled.Helper") + + val filtered = listOf( + project(listOf(moduleA), listOf(shadowing)), + project(listOf(moduleB), listOf(shadowing)), + ).dropDependenciesShadowingProjectClasses() + + assertEquals(emptyList(), filtered[0].dependencies) + assertEquals(emptyList(), filtered[1].dependencies) + } + + @Test + fun `module classes packaged as a jar also shadow`() { + val moduleJar = jar("module-classes.jar", "com.example.Dto") + val ownArtifact = jar("example-client-1.0.jar", "com.example.Dto") + + val filtered = listOf(project(listOf(moduleJar), listOf(ownArtifact))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(emptyList(), filtered.single().dependencies) + } +} From 1c8a2790659173dfc40d3345945249fc6f1610ab Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Wed, 22 Jul 2026 18:02:20 +0200 Subject: [PATCH 2/3] fix(autobuilder): pick the highest version that actually resolves to a jar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-conflict resolution ran before jar resolution, so an artifact whose highest version has no artifact in the local caches lost its lower, resolvable versions and disappeared from the model entirely. The dependency graph is resolved from metadata alone: a version no configuration ever compiled against leaves only a POM behind. Conductor's graph carries jackson-core at 2.14.2, 2.15.3, 2.17.0, 2.17.3 and 2.18.0, and only 2.18.0 has no jar — the model lost jackson-core outright, which surfaced on CI as "package com.fasterxml.jackson.core does not exist" while compiling the project's dataflow approximations against the project dependencies. Let only versions that resolve take part in the conflict resolution: the highest resolvable version wins, and an artifact drops out only when no version resolves. Replayed over conductor's dependency graph, this recovers 10 artifacts (jackson-core, five other jackson modules, jersey-common, lz4-java, reactor-test, woodstox-core) and loses none: 758 -> 768 jars. Verified: autobuilder unit tests, including the new fallback cases. --- .../org/opentaint/project/ArtifactVersions.kt | 21 +++++++++++ .../project/GradleProjectResolver.kt | 14 +++++--- .../opentaint/project/ArtifactVersionsTest.kt | 36 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt index c3a8af068..a34da0a52 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt @@ -64,6 +64,27 @@ internal fun List.singleVersionPerArtifact( return best.values.toList() } +/** + * Keeps a single *usable* version of every artifact: the highest one that actually resolves to a file. + * + * A version present in the dependency graph is no guarantee of a downloaded artifact — the graph is + * resolved from metadata alone, so a version no configuration ever compiled against leaves only a POM + * in the local cache. Picking the highest version before checking that it resolves therefore drops the + * artifact from the model entirely, taking its classes with it. + */ +internal fun List.singleResolvedVersionPerArtifact( + artifact: (T) -> Pair, + version: (T) -> String, + resolve: (T) -> R?, + onDropped: (kept: T, dropped: T) -> Unit = { _, _ -> }, +): List = mapNotNull { dependency -> resolve(dependency)?.let { dependency to it } } + .singleVersionPerArtifact( + artifact = { artifact(it.first) }, + version = { version(it.first) }, + onDropped = { kept, dropped -> onDropped(kept.first, dropped.first) }, + ) + .map { it.second } + private val VERSION_SEPARATORS = charArrayOf('.', '-', '_', '+') private fun String.isNumeric(): Boolean = isNotEmpty() && all { it.isDigit() } diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt index 58afe5ee0..a52c923dc 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt @@ -163,10 +163,13 @@ class GradleProjectResolver( // Dependencies are collected across every module of the build, so the same artifact shows // up at each version any module resolved. Resolve that conflict the way the build tool // does — one version per artifact — instead of handing the analyzer several copies of the - // same classes and leaving the choice to classpath lookup order. - val conflictFree = directFirst.singleVersionPerArtifact( + // same classes and leaving the choice to classpath lookup order. Only versions that + // resolve to a jar take part: the graph also carries versions nothing ever compiled + // against, which have no artifact in the local caches. + val conflictFree = directFirst.singleResolvedVersionPerArtifact( artifact = { it.groupId to it.artifactId }, version = { it.version }, + resolve = { resolveJarPath(it) }, onDropped = { kept, dropped -> logger.debug { "Dependency conflict on ${dropped.groupId}:${dropped.artifactId}: " + @@ -177,10 +180,13 @@ class GradleProjectResolver( val droppedCount = directFirst.size - conflictFree.size if (droppedCount > 0) { - logger.info { "Resolved dependency version conflicts: dropped $droppedCount duplicate artifact versions" } + logger.info { + "Resolved dependencies: dropped $droppedCount entries " + + "(duplicate artifact versions and versions with no artifact in the local caches)" + } } - return conflictFree.mapNotNull { resolveJarPath(it) } + return conflictFree } private fun resolveJarPath(dependency: GradleDependencyInfo): Path? { diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt index 80fdb862c..fa138eae6 100644 --- a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt @@ -63,6 +63,42 @@ class ArtifactVersionsTest { assertEquals(3, dropped.size) } + @Test + fun `an unresolvable highest version falls back to the highest version that resolves`() { + val deps = listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.14.2"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.17.3"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.15.3"), + ) + // 2.17.3 is in the dependency graph but was never downloaded: metadata only, no jar + val jars = mapOf("2.14.2" to "jackson-core-2.14.2.jar", "2.15.3" to "jackson-core-2.15.3.jar") + + val kept = deps.singleResolvedVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + resolve = { jars[it.third] }, + ) + + assertEquals( + listOf("jackson-core-2.15.3.jar"), + kept, + "an artifact whose highest version has no jar must still reach the model" + ) + } + + @Test + fun `an artifact that resolves at no version is dropped`() { + val deps = listOf(Triple("com.example", "ghost", "1.0"), Triple("com.example", "ghost", "2.0")) + + val kept = deps.singleResolvedVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + resolve = { null }, + ) + + assertEquals(emptyList(), kept) + } + @Test fun `distinct artifacts of the same group are kept apart`() { val deps = listOf( From 014404aadcf0f1c56d9076c436bc3206471317bb Mon Sep 17 00:00:00 2001 From: Aleksandr Misonizhnik Date: Wed, 22 Jul 2026 22:46:17 +0200 Subject: [PATCH 3/3] fix(autobuilder): collapse dependency versions only within a major line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-conflict resolution kept the single highest version of every group:artifact across the whole build. A multi-module build routinely resolves several majors of one artifact, and those are not versions of one thing — they are incompatible APIs the build deliberately keeps apart. Conductor compiles os-persistence-v2 against opensearch-rest-client 2.18.0 and os-persistence-v3 against 3.5.0, shading them so both can coexist. RestClientBuilder's callbacks take Apache HttpClient 4 types (org.apache.http) in 2.x and HttpClient 5 types (org.apache.hc.client5) in 3.x. Collapsing to 3.5.0 left every module on the 2.x line compiled against classes the model no longer had, which surfaced on CI as "incompatible types: org.apache.http.impl.nio.client.HttpAsyncClientBuilder cannot be converted to org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder" while compiling the project's dataflow approximations. Collapse per major-version line instead: drift inside a line is the same API at different patch levels, so the highest still stands in for all of them, while distinct lines both reach the model. A version with no numeric major makes no compatibility claim and is never collapsed into another. Conductor model: 660 -> 736 dependency jars, both opensearch-rest-client jars retained, still 235 fewer than the 971 an unresolved model carries. Findings on that project are a superset of the pre-dedup model's: 7 vs 6, gaining an SSRF in HttpTask.java:173 and losing none. Verified: autobuilder unit tests, including the new major-line cases; conductor scanned end to end under the regression bench's CI settings (JDK 21, 8G, 1200s) to completion in 232s. Rule-tests were not re-run: this change only keeps more jars than the model they were last verified against. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../org/opentaint/project/ArtifactVersions.kt | 31 ++++++++-- .../opentaint/project/ArtifactVersionsTest.kt | 60 +++++++++++++++++-- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt index a34da0a52..2c4be08e5 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt @@ -31,9 +31,26 @@ internal fun compareArtifactVersions(left: String, right: String): Int { } /** - * Keeps a single version of every artifact — the highest — the way a build tool resolves a conflict. - * Without this the model carries every version any module resolved, which both inflates it and leaves - * the choice between same-named classes to classpath lookup order. + * The major-version line a version belongs to, or null when it does not start with a number. + * + * Versions of one line are the same artifact evolving; versions of different lines are different + * artifacts that happen to share a name, and a multi-module build routinely uses several at once. + */ +private fun String.majorVersionLine(): String? = + takeWhile { it !in VERSION_SEPARATORS }.takeIf { it.isNumeric() } + +/** + * Keeps a single version of every artifact per major-version line — the highest — the way a build + * tool resolves a conflict. Without this the model carries every version any module resolved, which + * both inflates it and leaves the choice between same-named classes to classpath lookup order. + * + * Collapsing stops at the major-version boundary. Drift inside a line is the same API at different + * patch levels, so the highest stands in for all of them; across lines the APIs are incompatible by + * construction, and a build that resolves two majors of one artifact genuinely needs both (conductor + * compiles one module against `opensearch-rest-client` 2.x and another against 3.x, whose callbacks + * take Apache HttpClient 4 and 5 types respectively). Keeping only the highest would leave every + * module on the older line compiled against classes the model no longer has. A version with no + * numeric major makes no compatibility claim at all and is never collapsed into another. * * Order is preserved: an artifact keeps the position of its first occurrence. */ @@ -42,10 +59,14 @@ internal fun List.singleVersionPerArtifact( version: (T) -> String, onDropped: (kept: T, dropped: T) -> Unit = { _, _ -> }, ): List { - val best = LinkedHashMap, T>() + val best = LinkedHashMap, T>() for (dependency in this) { - val key = artifact(dependency) + val (groupId, artifactId) = artifact(dependency) + val dependencyVersion = version(dependency) + // no numeric major: key on the whole version so the entry stands on its own + val line = dependencyVersion.majorVersionLine() ?: dependencyVersion + val key = Triple(groupId, artifactId, line) val current = best[key] if (current == null) { diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt index fa138eae6..7376f0501 100644 --- a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt @@ -38,11 +38,11 @@ class ArtifactVersionsTest { @Test fun `one version per artifact is kept, the highest`() { val deps = listOf( - Triple("com.google.guava", "guava", "31.0"), + Triple("com.google.guava", "guava", "33.0-jre"), Triple("ch.qos.logback", "logback-classic", "1.5.18"), - Triple("com.google.guava", "guava", "33.2"), + Triple("com.google.guava", "guava", "33.2-jre"), Triple("ch.qos.logback", "logback-classic", "1.5.21"), - Triple("com.google.guava", "guava", "30.1"), + Triple("com.google.guava", "guava", "33.1-jre"), ) val dropped = mutableListOf>() @@ -54,7 +54,7 @@ class ArtifactVersionsTest { assertEquals( listOf( - Triple("com.google.guava", "guava", "33.2"), + Triple("com.google.guava", "guava", "33.2-jre"), Triple("ch.qos.logback", "logback-classic", "1.5.21"), ), kept, @@ -63,6 +63,58 @@ class ArtifactVersionsTest { assertEquals(3, dropped.size) } + @Test + fun `versions of different major lines coexist`() { + // conductor builds os-persistence-v2 against opensearch-rest-client 2.x and os-persistence-v3 + // against 3.x — different majors of one artifact, kept apart by the build itself (it shades + // them). Their APIs are incompatible: RestClientBuilder's callbacks take Apache HttpClient 4 + // types in 2.x and HttpClient 5 types in 3.x. Collapsing the two loses the API a module + // actually compiles against. + val deps = listOf( + Triple("org.opensearch.client", "opensearch-rest-client", "2.18.0"), + Triple("org.opensearch.client", "opensearch-rest-client", "3.5.0"), + ) + + assertEquals( + deps, + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "different majors are incompatible artifacts, not versions of one" + ) + } + + @Test + fun `the highest version of each major line is kept`() { + val deps = listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.14.2"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.18.0"), + Triple("com.fasterxml.jackson.core", "jackson-core", "1.9.13"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.15.3"), + ) + + assertEquals( + listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.18.0"), + Triple("com.fasterxml.jackson.core", "jackson-core", "1.9.13"), + ), + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "drift inside a major line collapses; the 1.x line survives on its own" + ) + } + + @Test + fun `a non-numeric version is never collapsed into another`() { + val deps = listOf( + Triple("com.example", "lib", "RELEASE"), + Triple("com.example", "lib", "MILESTONE"), + ) + + assertEquals( + deps, + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "without a numeric major there is no compatibility claim to make" + ) + } + @Test fun `an unresolvable highest version falls back to the highest version that resolves`() { val deps = listOf(