diff --git a/.github/workflows/analytics-engine-compat.yml b/.github/workflows/analytics-engine-compat.yml new file mode 100644 index 00000000000..9c3bd9c9f99 --- /dev/null +++ b/.github/workflows/analytics-engine-compat.yml @@ -0,0 +1,44 @@ +name: Analytics Engine Compatibility + +on: + pull_request: + push: + branches-ignore: + - 'backport/**' + - 'dependabot/**' + paths: + - '**/*.java' + - '**gradle*' + - 'integ-test/**' + - '.github/workflows/analytics-engine-compat.yml' + merge_group: + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + analytics-engine-compat: + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - uses: actions/checkout@v4 + + - name: Set up JDK 25 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 25 + + - name: Run analytics-engine compatibility smoke test + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew :integ-test:analyticsEngineCompatIT" diff --git a/build.gradle b/build.gradle index 5916afc915d..9a52b144c10 100644 --- a/build.gradle +++ b/build.gradle @@ -127,11 +127,6 @@ allprojects { version += "-SNAPSHOT" } - // Path to the analytics-engine plugin ZIP. Override with - // `-PanalyticsEngineZip=/path/to/zip` if needed. - ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?: - "${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip" - plugins.withId('java') { java { sourceCompatibility = JavaVersion.VERSION_21 diff --git a/core/build.gradle b/core/build.gradle index ea0c419b106..f567fb85653 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -64,11 +64,11 @@ dependencies { } api 'org.apache.calcite:calcite-linq4j:1.41.0' api project(':common') - compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") - // Needed because the analytics-framework's QueryPlanExecutor signature uses + compileOnly 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' + // Needed because analytics-api's QueryPlanExecutor signature uses // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type. compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" - testImplementation files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") + testImplementation 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" diff --git a/doctest/build.gradle b/doctest/build.gradle index 934ad263a1d..d9dca3fd5c2 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -15,6 +15,7 @@ plugins { apply plugin: 'opensearch.testclusters' + def path = project(':').projectDir // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from // plugin module, and will only build ppl in it. @@ -195,7 +196,6 @@ testClusters { })) */ plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload)) - plugin provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) } plugin ':opensearch-sql-plugin' testDistribution = 'archive' } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 62e1b9861af..e079db222ae 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -270,8 +270,34 @@ def getGeoSpatialPlugin() { } } +// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead). +ext.pluginVersion = opensearch_version.tokenize('-')[0] +ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" +ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" + +task downloadAnalyticsEngineZip(type: Download) { + src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip" + dest analyticsEngineZipDest + overwrite false + onlyIfModified true + onlyIf { !project.findProperty('analyticsEngineZip') } +} + +task downloadArrowFlightRpcZip(type: Download) { + src "${featureBuildBase}/0-arrow-flight-rpc-${pluginVersion}.zip" + dest arrowFlightRpcZipDest + overwrite false + onlyIfModified true + onlyIf { !project.findProperty('arrowFlightRpcZip') } +} + def getAnalyticsEnginePlugin() { - provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) } + provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) } +} + +def getArrowFlightRpcPlugin() { + provider { (RegularFile) (() -> file(project.findProperty('arrowFlightRpcZip') ?: arrowFlightRpcZipDest)) } } testClusters { @@ -279,7 +305,6 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) - plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } @@ -287,7 +312,6 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) - plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } @@ -295,18 +319,23 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin(getGeoSpatialPlugin()) - plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } integTestWithSecurity { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) - plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } remoteIntegTestWithSecurity { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) + plugin ":opensearch-sql-plugin" + } + // Smoke test: verify sql loads cleanly alongside analytics-engine. + analyticsEngineCompat { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } @@ -360,9 +389,17 @@ task stopPrometheus(type: KillProcessTask) { stopPrometheus.mustRunAfter startPrometheus +task analyticsEngineCompatIT(type: RestIntegTestTask) { + useCluster testClusters.analyticsEngineCompat + dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip + systemProperty 'tests.security.manager', 'false' + filter { + includeTestsMatching 'org.opensearch.sql.plugin.AnalyticsEngineCompatIT' + } +} + task integJdbcTest(type: RestIntegTestTask) { testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with { - plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" } diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java new file mode 100644 index 00000000000..5cd89fa7cd9 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin; + +import org.junit.Test; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +/** + * Smoke test: verifies that opensearch-sql loads cleanly alongside arrow-flight-rpc and + * analytics-engine. A successful cluster start is the only assertion — no sql-specific logic runs. + */ +public class AnalyticsEngineCompatIT extends OpenSearchRestTestCase { + + @Test + public void testClusterStarted() { + // If the cluster booted, all three plugins loaded without classloader errors. + } +} diff --git a/libs/analytics-engine-3.7.0-SNAPSHOT.jar b/libs/analytics-engine-3.7.0-SNAPSHOT.jar deleted file mode 100644 index 448f6a421bf..00000000000 Binary files a/libs/analytics-engine-3.7.0-SNAPSHOT.jar and /dev/null differ diff --git a/libs/analytics-engine-3.7.0-SNAPSHOT.zip b/libs/analytics-engine-3.7.0-SNAPSHOT.zip deleted file mode 100644 index 601225ea102..00000000000 Binary files a/libs/analytics-engine-3.7.0-SNAPSHOT.zip and /dev/null differ diff --git a/libs/analytics-framework-3.7.0-SNAPSHOT.jar b/libs/analytics-framework-3.7.0-SNAPSHOT.jar deleted file mode 100644 index a5cfcc32946..00000000000 Binary files a/libs/analytics-framework-3.7.0-SNAPSHOT.jar and /dev/null differ diff --git a/plugin/build.gradle b/plugin/build.gradle index 14b9fadae18..708c4b18b35 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -1,5 +1,4 @@ import java.util.concurrent.Callable -import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin /* * Copyright OpenSearch Contributors @@ -161,10 +160,7 @@ dependencies { api project(":ppl") api project(':api') - // Bundled: analytics-framework interfaces must resolve even when the plugin is absent. - api files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar") - // Not bundled: classes here (e.g. OpenSearchSchemaBuilder) only load when the plugin is installed. - compileOnly files("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.jar") + implementation("org.opensearch.sandbox:analytics-api:${opensearch_version}") api project(':legacy') api project(':opensearch') api project(':prometheus') @@ -310,7 +306,6 @@ def getJobSchedulerPlugin() { testClusters.integTest { plugin(getJobSchedulerPlugin()) - plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) } plugin(project.tasks.bundlePlugin.archiveFile) testDistribution = "ARCHIVE" @@ -326,4 +321,3 @@ testClusters.integTest { run { useCluster testClusters.integTest } -