Analytics engine explain api

.github/workflows/calcite-snapshots.yml

@@ -0,0 +1,86 @@
+# This workflow will check out, build, and publish snapshots of calcite.
+
+name: OpenSearch Calcite Revision
+
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    inputs:
+      ref:
+        description: 'Calcite ref in github.com/apache/calcite, default to calcite-1.41.0 tag (c838dd471ca36f5648ef13e5c3c34c6ca0815322)'
+        type: string
+        required: false
+        default: 'c838dd471ca36f5648ef13e5c3c34c6ca0815322'
+      java_version:
+        description: 'Java version to use'
+        type: string
+        required: false
+        default: '21'
+      patch_file_path:
+        description: 'The patch file, default to sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch'
+        type: string
+        required: false
+        default: 'sandbox/patches/calcite/0001-CALCITE-3745-prefer-TCCL-for-Janino-parent-classloader.patch'
+
+jobs:
+  publish-snapshots:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout Calcite ref:${{ github.event.inputs.ref }}
+        uses: actions/checkout@v6
+        with:
+          repository: 'apache/calcite'
+          ref: ${{ github.event.inputs.ref }}
+          persist-credentials: false
+
+      - name: Checkout OpenSearch main
+        uses: actions/checkout@v6
+        with:
+          repository: 'opensearch-project/OpenSearch'
+          ref: 'main'
+          persist-credentials: false
+          path: 'os_main'
+
+      - name: Setup JDK ${{ github.event.inputs.java_version }}
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ github.event.inputs.java_version }}
+          distribution: 'temurin'
+
+      - name: Apply Patches and build calcite jars
+        run: |
+            git apply os_main/${{ github.event.inputs.patch_file_path }}
+            BASE_VER=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"`
+            REV=`cat os_main/gradle/libs.versions.toml | grep -E "^calcite_os_rev" | grep -Eo "[0-9]+"`
+            CALCITE_VER=$BASE_VER-opensearch-$REV
+            sed -i "s/calcite\.version.*/calcite.version=$CALCITE_VER/" gradle.properties
+            ./gradlew :core:publishToMavenLocal :linq4j:publishToMavenLocal -Prelease -PskipSign -PskipJavadoc -x test --no-daemon
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }}
+          aws-region: us-east-1
+
+      - name: Get S3 Bucket
+        id: get_s3_bucket
+        run: |
+          lucene_snapshots_bucket=`aws secretsmanager get-secret-value --secret-id jenkins-artifact-bucket-name --query SecretString --output text`
+          echo "::add-mask::$lucene_snapshots_bucket"
+          echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+
+      - name: Copy files to S3 with the aws CLI
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/calcite/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/maven/org/apache/calcite/  --recursive --no-progress

.github/workflows/issue-dedupe.yml

@@ -0,0 +1,40 @@
+---
+name: Issue Dedupe Main
+on:
+  issues:
+    types: [opened]
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+    inputs:
+      issue_number:
+        description: 'Issue number to check for duplicates'
+        required: true
+        type: string
+
+jobs:
+  detect-issue:
+    if: >-
+      (github.event_name == 'workflow_dispatch' &&
+       github.repository == 'opensearch-project/OpenSearch') ||
+      (github.event_name == 'issues' &&
+       github.event.issue.user.type != 'Bot' &&
+       github.repository == 'opensearch-project/OpenSearch')
+    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@main
+    permissions:
+      contents: read
+      issues: write
+      id-token: write
+    secrets:
+      BEDROCK_ACCESS_ROLE_ISSUE_DEDUPE: ${{ secrets.BEDROCK_ACCESS_ROLE_ISSUE_DEDUPE }}
+    with:
+      issue_number: ${{ inputs.issue_number || '' }}
+      grace_days: ${{ vars.DUPLICATE_GRACE_DAYS || '7' }}
+
+  auto-close-issue:
+    if: github.event_name == 'schedule' && github.repository == 'opensearch-project/OpenSearch'
+    uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@main
+    permissions:
+      issues: write
+    with:
+      grace_days: ${{ vars.DUPLICATE_GRACE_DAYS || '7' }}

.github/workflows/publish-maven-snapshots.yml

@@ -19,11 +19,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v6
-      - name: Set up JDK 21
+      - name: Set up JDK 25 (required for sandbox publishing, default min support is still 21)
         uses: actions/setup-java@v5
         with:
           distribution: temurin
-          java-version: 21
+          java-version: 25  # TODO: switch back to jdk21 once sandbox plugins set min compat to 21
 
       - name: Install protoc (Linux)
         run: |
@@ -50,6 +50,6 @@ jobs:
           role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
           aws-region: us-east-1
 
-      - name: Publish snapshots to maven
+      - name: Publish snapshots to maven (with sandbox)
         run: |
-          ./gradlew publishNebulaPublicationToSnapshotsRepository -Pcrypto.standard=FIPS-140-3
+          ./gradlew publishNebulaPublicationToSnapshotsRepository -Dsandbox.enabled=true -Pcrypto.standard=FIPS-140-3

.github/workflows/sandbox-check.yml

@@ -32,8 +32,18 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
       - name: Install protobuf compiler
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+      - name: Check out SQL repo (mustang-ppl-integration)
+        uses: actions/checkout@v6
+        with:
+          repository: opensearch-project/sql
+          ref: feature/mustang-ppl-integration
+          path: sql
+      - name: Publish unified-query artifacts to maven local
+        working-directory: sql
+        continue-on-error: true
+        run: ./gradlew publishUnifiedQueryPublicationToMavenLocal
       - name: Run sandbox check
-        run: ./gradlew check -p sandbox -Dsandbox.enabled=true
+        run: ./gradlew check -p sandbox -Dsandbox.enabled=true -Drepos.mavenLocal=true -PrustDebug
       - name: Upload test results
         if: always()
         uses: actions/upload-artifact@v4

build.gradle

@@ -421,7 +421,11 @@ gradle.projectsEvaluated {
         task.jvmArgs += [
             "--add-modules=jdk.incubator.vector",
             "--add-exports=java.base/com.sun.crypto.provider=ALL-UNNAMED",
-            "--enable-native-access=ALL-UNNAMED"
+            "--enable-native-access=ALL-UNNAMED",
+            // Disable ByteBuddy's Unsafe-based class injection path to avoid
+            // "sun.misc.Unsafe::objectFieldOffset has been called by ByteBuddy" JVM warnings on JDK 21+.
+            // ByteBuddy still falls back to Lookup/Reflection injection strategies.
+            "-Dnet.bytebuddy.safe=true"
         ]
 
         // Add Java Agent for security sandboxing

buildSrc/src/main/java/org/opensearch/gradle/info/FipsBuildParams.java

@@ -9,6 +9,7 @@
 package org.opensearch.gradle.info;
 
 import java.util.function.Function;
+import java.util.function.Supplier;
 
 public class FipsBuildParams {
 
@@ -18,6 +19,7 @@ public class FipsBuildParams {
     public static final String DEFAULT_FIPS_MODE = "FIPS-140-3";
 
     private static String fipsMode;
+    static Supplier<String> fipsModeEnvSupplier = () -> System.getenv("OPENSEARCH_FIPS_MODE");
 
     public static void init(Function<String, Object> fipsValue) {
         var fipsBuildParamForTests = Boolean.parseBoolean((String) fipsValue.apply(FIPS_BUILD_PARAM_FOR_TESTS));
@@ -37,7 +39,7 @@ public static boolean isInFipsMode() {
     }
 
     public static boolean isInFipsApprovedOnlyMode() {
-        return isInFipsMode() && "true".equals(System.getProperty("org.bouncycastle.fips.approved_only"));
+        return isInFipsMode() && "true".equalsIgnoreCase(fipsModeEnvSupplier.get());
     }
 
     public static String getFipsMode() {

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "25.0.2+10";
+    private static final String SYSTEM_JDK_VERSION = "25.0.3+9";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "25.0.2+10";
+    private static final String GRADLE_JDK_VERSION = "25.0.3+9";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-7.4.1
+9.4.1

buildSrc/src/test/java/org/opensearch/gradle/info/FipsBuildParamsTests.java

@@ -14,6 +14,30 @@
 
 public class FipsBuildParamsTests extends GradleUnitTestCase {
 
+    public void testIsInFipsApprovedOnlyMode() {
+        FipsBuildParams.init(cryptoEntryFnWithStringParam);
+
+        FipsBuildParams.fipsModeEnvSupplier = () -> "true";
+        assertTrue(FipsBuildParams.isInFipsApprovedOnlyMode());
+
+        FipsBuildParams.fipsModeEnvSupplier = () -> "TRUE";
+        assertTrue(FipsBuildParams.isInFipsApprovedOnlyMode());
+
+        FipsBuildParams.fipsModeEnvSupplier = () -> "false";
+        assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode());
+
+        FipsBuildParams.fipsModeEnvSupplier = () -> null;
+        assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode());
+
+        // Not in FIPS mode — should always be false regardless of env var
+        FipsBuildParams.init(param -> null);
+        FipsBuildParams.fipsModeEnvSupplier = () -> "true";
+        assertFalse(FipsBuildParams.isInFipsApprovedOnlyMode());
+
+        // Reset
+        FipsBuildParams.fipsModeEnvSupplier = () -> System.getenv("OPENSEARCH_FIPS_MODE");
+    }
+
     public void testIsInFipsMode() {
         FipsBuildParams.init(cryptoEntryFnWithStringParam);
         assertTrue(FipsBuildParams.isInFipsMode());

client/rest/licenses/httpclient5-5.6.1.jar.sha1

@@ -0,0 +1 @@
+b418ba210ace28adf920f1decf64d673953d07cf

client/sniffer/licenses/httpclient5-5.6.1.jar.sha1

@@ -0,0 +1 @@
+b418ba210ace28adf920f1decf64d673953d07cf

client/sniffer/licenses/jackson-core-3.1.3.jar.sha1

@@ -0,0 +1 @@
+2f1dbeb81fe57c51e660534d3678003e514c1eb7

gradle/libs.versions.toml

@@ -3,21 +3,21 @@ opensearch        = "3.7.0"
 lucene            = "10.4.0"
 
 bundled_jdk_vendor = "adoptium"
-bundled_jdk = "25.0.2+10"
+bundled_jdk = "25.0.3+9"
 
 # optional dependencies
 spatial4j               = "0.7"
 jts                     = "1.15.0"
 jackson_annotations     = "2.21"
-jackson                 = "2.21.2"
-jackson_databind        = "2.21.2"
-jackson3                = "3.1.2"
-jackson3_databind       = "3.1.2"
+jackson                 = "2.21.3"
+jackson_databind        = "2.21.3"
+jackson3                = "3.1.3"
+jackson3_databind       = "3.1.3"
 snakeyaml               = "2.6"
 snakeyaml_engine        = "3.0.1"
 icu4j                   = "77.1"
 supercsv                = "2.4.0"
-log4j                   = "2.25.3"
+log4j                   = "2.25.4"
 error_prone_annotations = "2.45.0"
 slf4j                   = "2.0.17"
 asm                     = "9.9.1"
@@ -40,7 +40,7 @@ json_smart              = "2.5.2"
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = "5.16.0"
 
-netty             = "4.2.12.Final"
+netty             = "4.2.13.Final"
 joda              = "2.12.7"
 roaringbitmap     = "1.3.0"
 
@@ -49,7 +49,7 @@ reactor_netty     = "1.3.5"
 reactor           = "3.8.5"
 
 # client dependencies
-httpclient5       = "5.6"
+httpclient5       = "5.6.1"
 httpcore5         = "5.4"
 httpclient        = "4.5.14"
 httpcore          = "4.4.16"
@@ -97,13 +97,22 @@ jzlib             = "1.1.3"
 resteasy          = "6.2.4.Final"
 
 # opentelemetry dependencies
-opentelemetry         = "1.61.0"
-opentelemetrysemconv  = "1.40.0"
+opentelemetry         = "1.62.0"
+opentelemetrysemconv  = "1.41.0"
 
 # arrow dependencies
 arrow                 = "18.1.0"
 flatbuffers           = "2.0.0"
 
+# calcite is locally patched and published to OpenSearch maven snapshots; see .github/workflows/calcite-snapshots.yml.
+# Published as org.apache.calcite:calcite-core:${calcite}-opensearch-${calcite_os_rev}.
+calcite               = "1.41.0"
+calcite_os_rev        = "1"
+
+# property-based testing
+jqwik                 = "1.9.2"
+junit_jupiter         = "5.11.3"
+junit_platform        = "1.11.3"
 [libraries]
 antlr4-runtime = { group = "org.antlr", name = "antlr4-runtime", version.ref = "antlr4" }
 asm-analysis = { group = "org.ow2.asm", name = "asm-analysis", version.ref = "asm" }

gradle/missing-javadoc.gradle

@@ -308,7 +308,9 @@ class MissingJavadocTask extends DefaultTask {
       opts << [ '--missing-method', String.join(',', javadocMissingMethod) ]
     }
     opts << [ '-quiet' ]
-    opts << [ '--release', 21 ]
+
+    // To support modules with JDK 25 and above as well
+    opts << [ '--release', Math.max(project.java.sourceCompatibility.majorVersion.toInteger(), 21) ]
     opts << '-Xdoclint:all,-missing'
 
     // Temporary file that holds all javadoc options for the current task.

gradle/run.gradle

@@ -90,6 +90,15 @@ testClusters {
           systemProperty 'io.netty.tryUnsafe', 'true'
           systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
         }
+        if (p.equals("parquet-data-format") || p.equals("analytics-backend-datafusion")) {
+          // Composite engine / DataFusion requires pluggable dataformat feature flag
+          systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true'
+          // Native lib path for DataFusion FFM bridge
+          def nativeLibDir = new File(project(':sandbox:libs:dataformat-native').projectDir, 'rust/target/release').absolutePath
+          systemProperty 'java.library.path', nativeLibDir
+          jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED'
+          jvmArgs '--enable-native-access=ALL-UNNAMED'
+        }
       }
     }
   }

libs/agent-sm/agent-policy/src/main/java/org/opensearch/secure_sm/AccessController.java

@@ -17,7 +17,7 @@
  * removal. All new code should use this class instead of the JDK's {@code AccessController}.
  *
  * Running code in a privileged context will ensure that the code has the necessary permissions
- * without traversing through the entire call stack. See {@code org.opensearch.javaagent.StackCallerProtectionDomainChainExtractor}
+ * without traversing through the entire call stack. See {@code org.opensearch.javaagent.bootstrap.internal.StackCallerProtectionDomainChainExtractor}
  *
  * Example usages:
  * <pre>