diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt new file mode 100644 index 000000000..2c4be08e5 --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ArtifactVersions.kt @@ -0,0 +1,111 @@ +package org.opentaint.project + +/** + * Version comparison in the spirit of Maven/Gradle conflict resolution: segments are compared one by + * one, numerically where both sides are numeric, and a trailing qualifier (`-rc3`, `-SNAPSHOT`) makes + * a version *lower* than the same version without it. + */ +internal fun compareArtifactVersions(left: String, right: String): Int { + val leftParts = left.split(*VERSION_SEPARATORS) + val rightParts = right.split(*VERSION_SEPARATORS) + + for (i in 0 until maxOf(leftParts.size, rightParts.size)) { + val l = leftParts.getOrNull(i) + val r = rightParts.getOrNull(i) + + // one version ran out of segments: a numeric tail means a later release (1.2.1 > 1.2), + // a qualifier tail means a pre-release of the shorter one (1.2 > 1.2-rc1) + if (l == null) return if (r!!.isNumeric()) -1 else 1 + if (r == null) return if (l.isNumeric()) 1 else -1 + + val cmp = when { + l.isNumeric() && r.isNumeric() -> l.toBigInteger().compareTo(r.toBigInteger()) + l.isNumeric() -> 1 + r.isNumeric() -> -1 + else -> l.compareTo(r) + } + if (cmp != 0) return cmp + } + + return 0 +} + +/** + * The major-version line a version belongs to, or null when it does not start with a number. + * + * Versions of one line are the same artifact evolving; versions of different lines are different + * artifacts that happen to share a name, and a multi-module build routinely uses several at once. + */ +private fun String.majorVersionLine(): String? = + takeWhile { it !in VERSION_SEPARATORS }.takeIf { it.isNumeric() } + +/** + * Keeps a single version of every artifact per major-version line — the highest — the way a build + * tool resolves a conflict. Without this the model carries every version any module resolved, which + * both inflates it and leaves the choice between same-named classes to classpath lookup order. + * + * Collapsing stops at the major-version boundary. Drift inside a line is the same API at different + * patch levels, so the highest stands in for all of them; across lines the APIs are incompatible by + * construction, and a build that resolves two majors of one artifact genuinely needs both (conductor + * compiles one module against `opensearch-rest-client` 2.x and another against 3.x, whose callbacks + * take Apache HttpClient 4 and 5 types respectively). Keeping only the highest would leave every + * module on the older line compiled against classes the model no longer has. A version with no + * numeric major makes no compatibility claim at all and is never collapsed into another. + * + * Order is preserved: an artifact keeps the position of its first occurrence. + */ +internal fun List.singleVersionPerArtifact( + artifact: (T) -> Pair, + version: (T) -> String, + onDropped: (kept: T, dropped: T) -> Unit = { _, _ -> }, +): List { + val best = LinkedHashMap, T>() + + for (dependency in this) { + val (groupId, artifactId) = artifact(dependency) + val dependencyVersion = version(dependency) + // no numeric major: key on the whole version so the entry stands on its own + val line = dependencyVersion.majorVersionLine() ?: dependencyVersion + val key = Triple(groupId, artifactId, line) + val current = best[key] + + if (current == null) { + best[key] = dependency + continue + } + + if (compareArtifactVersions(version(dependency), version(current)) > 0) { + best[key] = dependency + onDropped(dependency, current) + } else { + onDropped(current, dependency) + } + } + + return best.values.toList() +} + +/** + * Keeps a single *usable* version of every artifact: the highest one that actually resolves to a file. + * + * A version present in the dependency graph is no guarantee of a downloaded artifact — the graph is + * resolved from metadata alone, so a version no configuration ever compiled against leaves only a POM + * in the local cache. Picking the highest version before checking that it resolves therefore drops the + * artifact from the model entirely, taking its classes with it. + */ +internal fun List.singleResolvedVersionPerArtifact( + artifact: (T) -> Pair, + version: (T) -> String, + resolve: (T) -> R?, + onDropped: (kept: T, dropped: T) -> Unit = { _, _ -> }, +): List = mapNotNull { dependency -> resolve(dependency)?.let { dependency to it } } + .singleVersionPerArtifact( + artifact = { artifact(it.first) }, + version = { version(it.first) }, + onDropped = { kept, dropped -> onDropped(kept.first, dropped.first) }, + ) + .map { it.second } + +private val VERSION_SEPARATORS = charArrayOf('.', '-', '_', '+') + +private fun String.isNumeric(): Boolean = isNotEmpty() && all { it.isDigit() } diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt index 7d4d0a051..a52c923dc 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/GradleProjectResolver.kt @@ -157,15 +157,36 @@ class GradleProjectResolver( fun resolveDependenciesJars(): List { val allDependenciesInfo = dependenciesInfo.entries.sortedBy { it.key } - val resolvedDirectDependencies = allDependenciesInfo - .filter { it.key in directDependencies } - .mapNotNull { resolveJarPath(it.value) } + val directFirst = allDependenciesInfo.filter { it.key in directDependencies }.map { it.value } + + allDependenciesInfo.filterNot { it.key in directDependencies }.map { it.value } + + // Dependencies are collected across every module of the build, so the same artifact shows + // up at each version any module resolved. Resolve that conflict the way the build tool + // does — one version per artifact — instead of handing the analyzer several copies of the + // same classes and leaving the choice to classpath lookup order. Only versions that + // resolve to a jar take part: the graph also carries versions nothing ever compiled + // against, which have no artifact in the local caches. + val conflictFree = directFirst.singleResolvedVersionPerArtifact( + artifact = { it.groupId to it.artifactId }, + version = { it.version }, + resolve = { resolveJarPath(it) }, + onDropped = { kept, dropped -> + logger.debug { + "Dependency conflict on ${dropped.groupId}:${dropped.artifactId}: " + + "keeping ${kept.version}, dropping ${dropped.version}" + } + } + ) - val resolvedIndirectDependencies = allDependenciesInfo - .filter { it.key !in directDependencies } - .mapNotNull { resolveJarPath(it.value) } + val droppedCount = directFirst.size - conflictFree.size + if (droppedCount > 0) { + logger.info { + "Resolved dependencies: dropped $droppedCount entries " + + "(duplicate artifact versions and versions with no artifact in the local caches)" + } + } - return resolvedDirectDependencies + resolvedIndirectDependencies + return conflictFree } private fun resolveJarPath(dependency: GradleDependencyInfo): Path? { diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt index b06786da7..73eb80362 100644 --- a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ProjectAutoBuilder.kt @@ -47,6 +47,7 @@ class ProjectAutoBuilder : CliWithLogger() { } val javaProjects = resolvedProject?.let { Project.flattenJavaProject(it) }.orEmpty() + .dropDependenciesShadowingProjectClasses() val goProjects = GoProjectResolver.resolveProject(projectRoot, resolverWorkDir) if (javaProjects.isEmpty() && goProjects.isEmpty()) { diff --git a/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt new file mode 100644 index 000000000..978cb1443 --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/main/kotlin/org/opentaint/project/ShadowingDependencies.kt @@ -0,0 +1,76 @@ +package org.opentaint.project + +import mu.KLogging +import java.nio.file.Path +import java.util.zip.ZipFile +import kotlin.io.path.extension +import kotlin.io.path.isDirectory +import kotlin.io.path.isRegularFile +import kotlin.io.path.name +import kotlin.io.path.relativeTo +import kotlin.io.path.walk + +private val logger = object : KLogging() {}.logger + +/** + * Drops dependencies that ship the project's own classes. + * + * A project routinely depends on a published artifact that repackages its own modules (a client + * bundle, a shaded jar, the previous release of the module itself). The project's compiled output + * already carries those classes, so the dependency copy adds nothing — but it does make the class + * name ambiguous on the classpath, and a lookup that lands on the dependency copy turns project code + * into un-analyzable library code, silently killing every taint flow through it. + */ +fun List.dropDependenciesShadowingProjectClasses(): List { + val projectClasses = flatMap { it.modules } + .flatMap { it.moduleClasses } + .flatMapTo(mutableSetOf()) { classNamesOf(it) } + + if (projectClasses.isEmpty()) return this + + val shadowingDependencies = flatMap { it.dependencies } + .distinct() + .mapNotNull { dependency -> + val shadowed = classNamesOf(dependency).count { it in projectClasses } + if (shadowed == 0) null else dependency to shadowed + } + .toMap() + + if (shadowingDependencies.isEmpty()) return this + + for ((dependency, shadowed) in shadowingDependencies) { + logger.warn { + "Dependency ${dependency.name} ships $shadowed classes the project itself compiles; " + + "dropping it from the project model in favour of the project's own output" + } + } + + return map { project -> + project.copy(dependencies = project.dependencies.filter { it !in shadowingDependencies }) + } +} + +@OptIn(kotlin.io.path.ExperimentalPathApi::class) +private fun classNamesOf(path: Path): Set = when { + path.isDirectory() -> path.walk() + .filter { it.extension == CLASS_EXTENSION } + .mapTo(mutableSetOf()) { it.relativeTo(path).toString().toClassName() } + + path.isRegularFile() -> runCatching { + ZipFile(path.toFile()).use { zip -> + zip.entries().asSequence() + .filter { !it.isDirectory && it.name.endsWith(".$CLASS_EXTENSION") } + .mapTo(mutableSetOf()) { it.name.toClassName() } + } + }.getOrElse { + logger.warn { "Cannot read classes of $path: ${it.message}" } + emptySet() + } + + else -> emptySet() +} + +private fun String.toClassName(): String = + removeSuffix(".$CLASS_EXTENSION").replace('\\', '.').replace('/', '.') + +private const val CLASS_EXTENSION = "class" diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt new file mode 100644 index 000000000..7376f0501 --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ArtifactVersionsTest.kt @@ -0,0 +1,162 @@ +package org.opentaint.project + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class ArtifactVersionsTest { + + private fun assertGreater(left: String, right: String) { + assertTrue(compareArtifactVersions(left, right) > 0, "expected $left > $right") + assertTrue(compareArtifactVersions(right, left) < 0, "expected $right < $left") + } + + @Test + fun `numeric segments compare numerically, not lexicographically`() { + assertGreater("1.10.0", "1.9.0") + assertGreater("4.2.0", "4.0.10") + assertEquals(0, compareArtifactVersions("1.2.3", "1.2.3")) + } + + @Test + fun `a longer numeric version is greater`() { + assertGreater("1.2.1", "1.2") + } + + @Test + fun `a qualifier makes a version pre-release`() { + assertGreater("4.2.0", "4.2.0-rc3") + assertGreater("1.5.21", "1.5.21-SNAPSHOT") + assertGreater("4.2.0-rc3", "4.0.10") + } + + @Test + fun `qualifiers compare to each other lexicographically`() { + assertGreater("1.0-rc2", "1.0-beta1") + } + + @Test + fun `one version per artifact is kept, the highest`() { + val deps = listOf( + Triple("com.google.guava", "guava", "33.0-jre"), + Triple("ch.qos.logback", "logback-classic", "1.5.18"), + Triple("com.google.guava", "guava", "33.2-jre"), + Triple("ch.qos.logback", "logback-classic", "1.5.21"), + Triple("com.google.guava", "guava", "33.1-jre"), + ) + val dropped = mutableListOf>() + + val kept = deps.singleVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + onDropped = { _, d -> dropped += d } + ) + + assertEquals( + listOf( + Triple("com.google.guava", "guava", "33.2-jre"), + Triple("ch.qos.logback", "logback-classic", "1.5.21"), + ), + kept, + "must keep the highest version of each artifact, in first-seen order" + ) + assertEquals(3, dropped.size) + } + + @Test + fun `versions of different major lines coexist`() { + // conductor builds os-persistence-v2 against opensearch-rest-client 2.x and os-persistence-v3 + // against 3.x — different majors of one artifact, kept apart by the build itself (it shades + // them). Their APIs are incompatible: RestClientBuilder's callbacks take Apache HttpClient 4 + // types in 2.x and HttpClient 5 types in 3.x. Collapsing the two loses the API a module + // actually compiles against. + val deps = listOf( + Triple("org.opensearch.client", "opensearch-rest-client", "2.18.0"), + Triple("org.opensearch.client", "opensearch-rest-client", "3.5.0"), + ) + + assertEquals( + deps, + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "different majors are incompatible artifacts, not versions of one" + ) + } + + @Test + fun `the highest version of each major line is kept`() { + val deps = listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.14.2"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.18.0"), + Triple("com.fasterxml.jackson.core", "jackson-core", "1.9.13"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.15.3"), + ) + + assertEquals( + listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.18.0"), + Triple("com.fasterxml.jackson.core", "jackson-core", "1.9.13"), + ), + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "drift inside a major line collapses; the 1.x line survives on its own" + ) + } + + @Test + fun `a non-numeric version is never collapsed into another`() { + val deps = listOf( + Triple("com.example", "lib", "RELEASE"), + Triple("com.example", "lib", "MILESTONE"), + ) + + assertEquals( + deps, + deps.singleVersionPerArtifact({ it.first to it.second }, { it.third }), + "without a numeric major there is no compatibility claim to make" + ) + } + + @Test + fun `an unresolvable highest version falls back to the highest version that resolves`() { + val deps = listOf( + Triple("com.fasterxml.jackson.core", "jackson-core", "2.14.2"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.17.3"), + Triple("com.fasterxml.jackson.core", "jackson-core", "2.15.3"), + ) + // 2.17.3 is in the dependency graph but was never downloaded: metadata only, no jar + val jars = mapOf("2.14.2" to "jackson-core-2.14.2.jar", "2.15.3" to "jackson-core-2.15.3.jar") + + val kept = deps.singleResolvedVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + resolve = { jars[it.third] }, + ) + + assertEquals( + listOf("jackson-core-2.15.3.jar"), + kept, + "an artifact whose highest version has no jar must still reach the model" + ) + } + + @Test + fun `an artifact that resolves at no version is dropped`() { + val deps = listOf(Triple("com.example", "ghost", "1.0"), Triple("com.example", "ghost", "2.0")) + + val kept = deps.singleResolvedVersionPerArtifact( + artifact = { it.first to it.second }, + version = { it.third }, + resolve = { null }, + ) + + assertEquals(emptyList(), kept) + } + + @Test + fun `distinct artifacts of the same group are kept apart`() { + val deps = listOf( + Triple("org.springframework", "spring-core", "6.1.0"), + Triple("org.springframework", "spring-web", "6.1.0"), + ) + assertEquals(deps, deps.singleVersionPerArtifact({ it.first to it.second }, { it.third })) + } +} diff --git a/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt new file mode 100644 index 000000000..79137685f --- /dev/null +++ b/core/opentaint-jvm-autobuilder/src/test/kotlin/org/opentaint/project/ShadowingDependenciesTest.kt @@ -0,0 +1,102 @@ +package org.opentaint.project + +import java.nio.file.Path +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.createDirectories +import kotlin.io.path.createTempDirectory +import kotlin.io.path.outputStream +import kotlin.io.path.writeBytes +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +class ShadowingDependenciesTest { + + private val tempDir: Path = createTempDirectory("shadowing-dependencies") + + @AfterTest + fun cleanup() { + tempDir.toFile().deleteRecursively() + } + + private fun classesDir(name: String, vararg classes: String): Path { + val dir = tempDir.resolve(name) + for (cls in classes) { + val file = dir.resolve(cls.replace('.', '/') + ".class") + file.parent.createDirectories() + file.writeBytes(byteArrayOf(0xCA.toByte(), 0xFE.toByte(), 0xBA.toByte(), 0xBE.toByte())) + } + dir.createDirectories() + return dir + } + + private fun jar(name: String, vararg classes: String): Path { + val path = tempDir.resolve(name) + path.parent.createDirectories() + ZipOutputStream(path.outputStream()).use { zip -> + for (cls in classes) { + zip.putNextEntry(ZipEntry(cls.replace('.', '/') + ".class")) + zip.write(byteArrayOf(0xCA.toByte(), 0xFE.toByte(), 0xBA.toByte(), 0xBE.toByte())) + zip.closeEntry() + } + } + return path + } + + private fun project(modules: List, dependencies: List) = JavaProject( + sourceRoot = tempDir, + modules = listOf(ProjectModuleClasses(moduleSourceRoot = tempDir, moduleClasses = modules)), + dependencies = dependencies, + ) + + @Test + fun `a dependency republishing project classes is dropped`() { + val moduleClasses = classesDir("module", "com.example.Dto", "com.example.Service") + val ownArtifact = jar("example-client-1.0.jar", "com.example.Dto", "com.example.client.Api") + val thirdParty = jar("guava-33.2.jar", "com.google.common.collect.Lists") + + val filtered = listOf(project(listOf(moduleClasses), listOf(ownArtifact, thirdParty))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(listOf(thirdParty), filtered.single().dependencies) + } + + @Test + fun `a dependency sharing no class is kept`() { + val moduleClasses = classesDir("module", "com.example.Dto") + val thirdParty = jar("guava-33.2.jar", "com.google.common.collect.Lists") + + val filtered = listOf(project(listOf(moduleClasses), listOf(thirdParty))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(listOf(thirdParty), filtered.single().dependencies) + } + + @Test + fun `shadowing is judged against every module of every project`() { + // the jar shadows a class of the *other* project's module, and must be dropped from both + val moduleA = classesDir("moduleA", "com.example.a.Dto") + val moduleB = classesDir("moduleB", "com.example.b.Dto") + val shadowing = jar("bundle-1.0.jar", "com.example.b.Dto", "com.example.bundled.Helper") + + val filtered = listOf( + project(listOf(moduleA), listOf(shadowing)), + project(listOf(moduleB), listOf(shadowing)), + ).dropDependenciesShadowingProjectClasses() + + assertEquals(emptyList(), filtered[0].dependencies) + assertEquals(emptyList(), filtered[1].dependencies) + } + + @Test + fun `module classes packaged as a jar also shadow`() { + val moduleJar = jar("module-classes.jar", "com.example.Dto") + val ownArtifact = jar("example-client-1.0.jar", "com.example.Dto") + + val filtered = listOf(project(listOf(moduleJar), listOf(ownArtifact))) + .dropDependenciesShadowingProjectClasses() + + assertEquals(emptyList(), filtered.single().dependencies) + } +}