Adding dsl-query-executor plugin to sandbox

sandbox/libs/analytics-framework/build.gradle

@@ -30,6 +30,7 @@ dependencies {
   runtimeOnly 'com.google.guava:failureaccess:1.0.2'
   // SLF4J — Calcite's logging facade
   runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "commons-io:commons-io:${versions.commonsio}"
 
   // Calcite optional deps required at runtime — BuiltInMethod.<clinit> reflectively loads ALL
   // methods which triggers class loading for every type referenced in Calcite's SqlFunctions.

sandbox/plugins/dsl-query-executor/README.md

@@ -0,0 +1,35 @@
+# dsl-query-executor
+
+A front-end sandbox plugin to the analytics engine that intercepts `_search` requests, converts DSL queries into Calcite RelNode logical plans, and executes them through the analytics engine's query pipeline.
+
+## Architecture
+
+```
+_search request
+  → SearchActionFilter (intercepts SearchAction)
+    → TransportDslExecuteAction (resolves index, orchestrates pipeline)
+      → SearchSourceConverter (DSL → Calcite RelNode)
+      → DslQueryPlanExecutor (delegates to analytics engine)
+      → SearchResponseBuilder (builds SearchResponse)
+```
+
+## Dependencies
+
+- `analytics-engine` — provides `QueryPlanExecutor` and `EngineContext` via Guice (declared as `extendedPlugins`)
+- `analytics-framework` — provides Calcite and shared SPI interfaces
+
+## Running locally
+
+```bash
+./gradlew run -PinstalledPlugins="['analytics-engine','dsl-query-executor']"
+```
+
+## Testing
+
+```bash
+# Unit tests
+./gradlew :sandbox:plugins:dsl-query-executor:test
+
+# Integration tests
+./gradlew :sandbox:plugins:dsl-query-executor:internalClusterTest
+```

sandbox/plugins/dsl-query-executor/build.gradle

@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+apply plugin: 'opensearch.internal-cluster-test'
+
+opensearchplugin {
+  description = 'Sandbox plugin for DSL query execution via Calcite logical plans.'
+  classname = 'org.opensearch.dsl.DslQueryExecutorPlugin'
+  extendedPlugins = ['analytics-engine']
+}
+
+dependencies {
+  compileOnly project(':server')
+  compileOnly project(':sandbox:libs:analytics-framework')
+  compileOnly project(':sandbox:plugins:analytics-engine')
+  compileOnly 'org.apache.calcite.avatica:avatica-core:1.27.0'
+
+  testImplementation project(':test:framework')
+  testImplementation "org.mockito:mockito-core:${versions.mockito}"
+
+  internalClusterTestImplementation project(':server')
+  internalClusterTestImplementation project(':test:framework')
+}

sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslAggregationIT.java

@@ -0,0 +1,76 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl;
+
+import org.opensearch.search.aggregations.AggregationBuilders;
+import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.opensearch.search.builder.SearchSourceBuilder;
+
+/**
+ * Integration tests for DSL aggregation conversion.
+ * Uses matchAllQuery; focus is on aggregation plan building.
+ */
+public class DslAggregationIT extends DslIntegTestBase {
+
+    public void testMetricOnly() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(AggregationBuilders.avg("avg_price").field("price"))
+        ));
+    }
+
+    public void testMultipleMetrics() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(AggregationBuilders.avg("avg_price").field("price"))
+            .aggregation(AggregationBuilders.sum("total_price").field("price"))
+            .aggregation(AggregationBuilders.min("min_price").field("price"))
+            .aggregation(AggregationBuilders.max("max_price").field("price"))
+        ));
+    }
+
+    public void testTermsBucket() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(new TermsAggregationBuilder("by_brand").field("brand"))
+        ));
+    }
+
+    public void testTermsBucketWithMetric() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(new TermsAggregationBuilder("by_brand").field("brand")
+                .subAggregation(AggregationBuilders.avg("avg_price").field("price")))
+        ));
+    }
+
+    public void testNestedBuckets() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(new TermsAggregationBuilder("by_brand").field("brand")
+                .subAggregation(AggregationBuilders.sum("total").field("price"))
+                .subAggregation(new TermsAggregationBuilder("by_name").field("name")
+                    .subAggregation(AggregationBuilders.avg("avg_price").field("price"))))
+        ));
+    }
+
+    public void testAggsWithHits() {
+        createTestIndex();
+        // size > 0 with aggs produces both HITS + AGGREGATION plans
+        assertOk(search(new SearchSourceBuilder()
+            .size(10)
+            .aggregation(AggregationBuilders.avg("avg_price").field("price"))
+        ));
+    }
+}

sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslIntegTestBase.java

@@ -0,0 +1,56 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl;
+
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.analytics.AnalyticsPlugin;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.Collection;
+import java.util.List;
+
+// TODO: once end-to-end execution returns real results, update ITs to verify
+//  actual hit count, field values, sort order, and aggregation buckets.
+/**
+ * Base class for DSL query executor integration tests.
+ * Provides shared index setup and search helper.
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1)
+public abstract class DslIntegTestBase extends OpenSearchIntegTestCase {
+
+    protected static final String INDEX = "test-index";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return List.of(AnalyticsPlugin.class, DslQueryExecutorPlugin.class);
+    }
+
+    protected void createTestIndex() {
+        createIndex(INDEX);
+        ensureGreen();
+        client().prepareIndex(INDEX)
+            .setId("1")
+            .setSource("{\"name\":\"laptop\",\"price\":1200,\"brand\":\"brandX\",\"rating\":4.5}", XContentType.JSON)
+            .get();
+        refresh(INDEX);
+    }
+
+    protected SearchResponse search(SearchSourceBuilder source) {
+        return client().search(new SearchRequest(INDEX).source(source)).actionGet();
+    }
+
+    protected void assertOk(SearchResponse response) {
+        assertNotNull(response);
+        assertEquals(200, response.status().getStatus());
+    }
+}

sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslProjectIT.java

@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl;
+
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.fetch.subphase.FetchSourceContext;
+
+/**
+ * Integration tests for DSL _source filtering (projection) conversion.
+ * Uses matchAllQuery; focus is on _source includes/excludes behavior.
+ */
+public class DslProjectIT extends DslIntegTestBase {
+
+    public void testNoSourceFiltering() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()));
+    }
+
+    public void testIncludeSpecificFields() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().fetchSource(new String[]{"name", "price"}, null)));
+    }
+
+    public void testExcludeFields() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().fetchSource(new String[]{}, new String[]{"rating"})));
+    }
+
+    public void testSourceDisabled() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().fetchSource(false)));
+    }
+
+    public void testWildcardIncludes() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().fetchSource(new String[]{"na*"}, null)));
+    }
+
+    public void testWildcardExcludes() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().fetchSource(new String[]{}, new String[]{"ra*"})));
+    }
+}

sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslQueryIT.java

@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl;
+
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.builder.SearchSourceBuilder;
+
+/**
+ * Integration tests for DSL query conversion (filter path).
+ * Uses various query types; sort and projection use defaults.
+ */
+public class DslQueryIT extends DslIntegTestBase {
+
+    public void testNoQuery() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()));
+    }
+
+    public void testMatchAll() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())));
+    }
+
+    public void testTermQuery() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().query(QueryBuilders.termQuery("name", "laptop"))));
+    }
+
+    public void testWildcardQueryWithUnresolvedNode() {
+        createTestIndex();
+        // Wildcard query is not converted to standard Rex — wraps in UnresolvedQueryCall.
+        assertOk(search(new SearchSourceBuilder().query(QueryBuilders.wildcardQuery("name", "lap*"))));
+    }
+
+    public void testFailsForNonexistentIndex() {
+        expectThrows(Exception.class, () ->
+            client().search(new SearchRequest("nonexistent-index").source(new SearchSourceBuilder())).actionGet()
+        );
+    }
+
+    public void testFailsForMultipleIndices() {
+        createTestIndex();
+        createIndex("test-index-2");
+        ensureGreen();
+
+        expectThrows(Exception.class, () ->
+            client().search(new SearchRequest(INDEX, "test-index-2").source(new SearchSourceBuilder())).actionGet()
+        );
+    }
+}

sandbox/plugins/dsl-query-executor/src/internalClusterTest/java/org/opensearch/dsl/DslSortIT.java

@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dsl;
+
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.SortOrder;
+
+/**
+ * Integration tests for DSL sort and pagination conversion.
+ * Uses matchAllQuery; focus is on sort/from/size behavior.
+ */
+public class DslSortIT extends DslIntegTestBase {
+
+    public void testDefaultPagination() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder()));
+    }
+
+    public void testSortAscending() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().sort("name", SortOrder.ASC)));
+    }
+
+    public void testSortDescending() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().sort("price", SortOrder.DESC)));
+    }
+
+    public void testMultipleSortFields() {
+        createTestIndex();
+        assertOk(search(
+            new SearchSourceBuilder()
+                .sort("brand", SortOrder.ASC)
+                .sort("price", SortOrder.DESC)
+        ));
+    }
+
+    public void testCustomSize() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().size(5)));
+    }
+
+    public void testFromAndSize() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().from(0).size(5)));
+    }
+
+    public void testFromOffset() {
+        createTestIndex();
+        assertOk(search(new SearchSourceBuilder().from(10).size(5)));
+    }
+}