From 8d18065d0c4c2791336089cea75684e1048a546a Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 17 Jun 2026 17:24:06 -0700 Subject: [PATCH 01/41] Propagate request-task cancellation into the analytics PPL route (#5563) Signed-off-by: Marc Handalian --- .../sql/plugin/rest/RestPPLQueryAction.java | 46 ++++++------ .../plugin/rest/RestUnifiedQueryAction.java | 70 +++++++++++++++---- .../transport/TransportPPLQueryAction.java | 3 + 3 files changed, 87 insertions(+), 32 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 5c6266beee1..b6347bdf8e1 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -22,6 +22,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestCancellableNodeClient; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.datasources.exceptions.DataSourceClientException; @@ -113,27 +114,32 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod TransportPPLQueryRequest transportPPLQueryRequest = new TransportPPLQueryRequest(PPLQueryRequestFactory.getPPLRequest(request)); - return channel -> - nodeClient.execute( - PPLQueryAction.INSTANCE, - transportPPLQueryRequest, - new ActionListener<>() { - @Override - public void onResponse(TransportPPLQueryResponse response) { - sendResponse(channel, OK, response.getContentType(), response.getResult()); + // RestCancellableNodeClient cancels the PPLQueryTask on client disconnect, which cascades to + // the analytics query + fragments. + return channel -> { + RestCancellableNodeClient cancellableClient = + new RestCancellableNodeClient(nodeClient, request.getHttpChannel()); + cancellableClient.execute( + PPLQueryAction.INSTANCE, + transportPPLQueryRequest, + new ActionListener<>() { + @Override + public void onResponse(TransportPPLQueryResponse response) { + sendResponse(channel, OK, response.getContentType(), response.getResult()); + } + + @Override + public void onFailure(Exception e) { + RestStatus status = loggedErrorCode(e); + if (transportPPLQueryRequest.isExplainRequest()) { + LOG.error("Error happened during explain (status {})", status, e); + } else { + LOG.error("Error happened during query handling (status {})", status, e); } - - @Override - public void onFailure(Exception e) { - RestStatus status = loggedErrorCode(e); - if (transportPPLQueryRequest.isExplainRequest()) { - LOG.error("Error happened during explain (status {})", status, e); - } else { - LOG.error("Error happened during query handling (status {})", status, e); - } - reportError(channel, e, status); - } - }); + reportError(channel, e, status); + } + }); + }; } private void sendResponse( diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index d81ce7b5137..62f3dd0c346 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -17,6 +17,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.ThreadContext; +import org.opensearch.analytics.QueryRequestContext; import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.cluster.service.ClusterService; @@ -42,6 +43,7 @@ import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import org.opensearch.sql.utils.SystemIndexUtils; +import org.opensearch.tasks.Task; import org.opensearch.transport.client.node.NodeClient; /** @@ -144,12 +146,32 @@ private boolean isPluggableDataformatIndex(String indexName) { && "composite".equals(IndexSettings.PLUGGABLE_DATAFORMAT_VALUE_SETTING.get(settings)); } - /** Execute a query through the unified query pipeline on the sql-worker thread pool. */ + /** Execute with no parent task (SQL path): the analytics query runs detached, not cancellable. */ public void execute( String query, QueryType queryType, boolean profiling, ActionListener listener) { + doExecute(query, queryType, profiling, null, listener); + } + + /** Execute linked to {@code parentTask} so a front-end cancel propagates into the engine. */ + public void execute( + String query, + QueryType queryType, + boolean profiling, + Task parentTask, + ActionListener listener) { + assert parentTask != null : "parentTask required for cancellation propagation"; + doExecute(query, queryType, profiling, parentTask, listener); + } + + private void doExecute( + String query, + QueryType queryType, + boolean profiling, + Task parentTask, + ActionListener listener) { client .threadPool() .schedule( @@ -158,8 +180,9 @@ public void execute( // Ask the engine for a per-query context — it binds the snapshot // (cluster state + schema built from it) and returns the pair, so the // schema we plan against and the state the executor uses are the same view. - org.opensearch.analytics.QueryRequestContext queryCtx = - contextProvider.getContext(); + // Carry the front-end task so cancellation propagates into the engine. + QueryRequestContext queryCtx = + withParentTask(contextProvider.getContext(), parentTask); // Disable SQL-layer phase profiling when analytics engine profiling is active. // Our QueryProfile (stages, tasks, timing) is strictly more detailed and replaces // it. @@ -192,22 +215,39 @@ public void execute( SQL_WORKER_THREAD_POOL_NAME); } - /** - * Explain a query through the unified query pipeline on the sql-worker thread pool. Returns - * ExplainResponse via ResponseListener so the caller can format it. - */ + /** Explain with no parent task (SQL path). */ public void explain( String query, QueryType queryType, ExplainMode mode, ResponseListener listener) { + doExplain(query, queryType, mode, null, listener); + } + + /** Explain linked to {@code parentTask} so a front-end cancel propagates into the engine. */ + public void explain( + String query, + QueryType queryType, + ExplainMode mode, + Task parentTask, + ResponseListener listener) { + assert parentTask != null : "parentTask required for cancellation propagation"; + doExplain(query, queryType, mode, parentTask, listener); + } + + private void doExplain( + String query, + QueryType queryType, + ExplainMode mode, + Task parentTask, + ResponseListener listener) { client .threadPool() .schedule( withCurrentContext( () -> { - org.opensearch.analytics.QueryRequestContext queryCtx = - contextProvider.getContext(); + QueryRequestContext queryCtx = + withParentTask(contextProvider.getContext(), parentTask); try (UnifiedQueryContext context = buildContext(queryType, false, queryCtx)) { UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); RelNode plan = planner.plan(query); @@ -231,9 +271,7 @@ private UnifiedQueryContext buildParsingContext(QueryType queryType) { } private UnifiedQueryContext buildContext( - QueryType queryType, - boolean profiling, - org.opensearch.analytics.QueryRequestContext queryCtx) { + QueryType queryType, boolean profiling, QueryRequestContext queryCtx) { return applyClusterOverrides( UnifiedQueryContext.builder() .language(queryType) @@ -244,6 +282,14 @@ private UnifiedQueryContext buildContext( .build(); } + /** Returns {@code ctx} carrying {@code parentTask}, or unchanged when there is none. */ + private static QueryRequestContext withParentTask(QueryRequestContext ctx, Task parentTask) { + if (parentTask == null) { + return ctx; + } + return new QueryRequestContext(ctx.clusterState(), ctx.schema(), ctx.querySource(), parentTask); + } + /** * Routes operator-configured cluster overrides into the builder via the existing {@code * setting(String, Object)} API, keeping {@link UnifiedQueryContext} decoupled from any specific diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index fd231c29076..33dd8235443 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -176,17 +176,20 @@ protected void doExecute( if (unifiedQueryHandler != null && unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) { LOG.info("[{}] Routing PPL query to analytics engine", QueryContext.getRequestId()); + // Pass this PPL task so the analytics engine links its query task to it for cancellation. if (transformedRequest.isExplainRequest()) { unifiedQueryHandler.explain( transformedRequest.getRequest(), QueryType.PPL, transformedRequest.mode(), + task, createExplainResponseListener(transformedRequest, clearingListener)); } else { unifiedQueryHandler.execute( transformedRequest.getRequest(), QueryType.PPL, transformedRequest.profile(), + task, clearingListener); } return; From 9367c2ad4db23385a2539c37218454e9b8afe5fc Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Thu, 18 Jun 2026 10:44:56 -0700 Subject: [PATCH 02/41] Stabilize PPL ITs on the analytics-engine route (case/string/full-text/like/appendpipe) (#5561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Analytics-engine route parity for several PPL IT classes; test-only. Uses the @RequiresCapability annotation + Capability registry (#5560) plus matching excludeTestsMatching entries. CalcitePPLCaseFunctionIT: - Guard the weblogs raw-PUT seeding (appendDataForBadResponse) on a pre-load isIndexExist check — the append-only AE store inflated counts per method. - Skip the otel_logs load on the AE route (multi-value keyword the parquet store rejects); only testNestedCaseAggWithAutoDateHistogram uses it, and that test requires BIN_TIME_FIELD_BUCKETING (bucket column typed string). CalcitePPLStringBuiltinFunctionIT: 7 tests re-PUT a shared _id with different data; the append-only AE store can't replace docs (DELETE unsupported) -> DOC_MUTATION. MultiMatchIT / QueryStringIT / SimpleQueryStringIT wildcard tests: full-text relevance functions with no DataFusion equivalent -> new FULLTEXT_RELEVANCE_FUNC. CalciteLikeQueryIT.test_the_default_3rd_option: AE LIKE is case-insensitive but v2/Calcite is case-sensitive -> new LIKE_CASE_SENSITIVITY. CalcitePPLAppendPipeCommandIT.testDoubleAppendPipeWithFilter: appendpipe drops the main pipeline's rows on the AE route -> new APPENDPIPE_MAIN_RESULT_DROPPED. v2/Calcite route unchanged (all run, 0 skips). Signed-off-by: Kai Huang --- integ-test/build.gradle | 34 +++++++++++++++++-- .../calcite/remote/CalciteLikeQueryIT.java | 5 +++ .../remote/CalcitePPLAppendPipeCommandIT.java | 5 +++ .../remote/CalcitePPLCaseFunctionIT.java | 18 ++++++++-- .../CalcitePPLStringBuiltinFunctionIT.java | 23 +++++++++++++ .../org/opensearch/sql/ppl/MultiMatchIT.java | 5 +++ .../org/opensearch/sql/ppl/QueryStringIT.java | 5 +++ .../sql/ppl/SimpleQueryStringIT.java | 5 +++ .../org/opensearch/sql/util/Capability.java | 30 +++++++++++++++- 9 files changed, 124 insertions(+), 6 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d7fbf19b348..2e565aab8f2 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1075,10 +1075,11 @@ task integTestRemote(type: RestIntegTestTask) { // GeoPointFormatsIT: EVERY test reads a geo_point field — whole class doomed. excludeTestsMatching 'org.opensearch.sql.ppl.GeoPointFormatsIT' // datatypes_index_mapping: nested_value=nested, geo_point_value=geo_point. - excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_nonnumeric_data_types' - excludeTestsMatching 'org.opensearch.sql.ppl.SystemFunctionIT.typeof_opensearch_types' + // Glob matches the Calcite subclasses too (CalciteDataTypeIT, CalciteSystemFunctionIT). + excludeTestsMatching '*DataTypeIT.test_nonnumeric_data_types' + excludeTestsMatching '*SystemFunctionIT.typeof_opensearch_types' // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`. - excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_alias_data_type' + excludeTestsMatching '*DataTypeIT.test_alias_data_type' // CalciteAliasFieldAggregationIT: raw-PUT alias index can't be created on the AE route // and every test queries alias fields directly — whole class doomed. excludeTestsMatching 'org.opensearch.sql.calcite.remote.CalciteAliasFieldAggregationIT' @@ -1176,6 +1177,33 @@ task integTestRemote(type: RestIntegTestTask) { // - max() over int operands reports bigint on the AE route (DataFusion widens integers // to Int64) where the v2/Calcite path reports int. excludeTestsMatching '*CalcitePPLEvalMaxMinFunctionIT.testEvalMaxNumeric' + + // === Excludes: CalcitePPLCaseFunctionIT route divergences === + // bin @timestamp then group by it: bucket column typed string (not timestamp) on AE. + excludeTestsMatching '*CalcitePPLCaseFunctionIT.testNestedCaseAggWithAutoDateHistogram' + + // === Excludes: CalcitePPLStringBuiltinFunctionIT route divergences === + // Re-PUT a shared _id with different data; the append-only AE store can't replace docs. + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testConcatWithField' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testConcatWs' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testReverse' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testRight' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testTrim' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testRTrim' + excludeTestsMatching '*CalcitePPLStringBuiltinFunctionIT.testLTrim' + + // === Excludes: full-text relevance functions (unsupported on DataFusion) === + excludeTestsMatching '*MultiMatchIT.test_wildcard_multi_match' + excludeTestsMatching '*QueryStringIT.wildcard_test' + excludeTestsMatching '*SimpleQueryStringIT.test_wildcard_simple_query_string' + + // === Excludes: CalciteLikeQueryIT route divergence === + // LIKE is case-insensitive on AE (DataFusion); v2/Calcite treats LIKE as case-sensitive. + excludeTestsMatching '*CalciteLikeQueryIT.test_the_default_3rd_option' + + // === Excludes: CalcitePPLAppendPipeCommandIT route divergence === + // appendpipe drops the main pipeline's rows on AE (subpipe filter applied in place). + excludeTestsMatching '*CalcitePPLAppendPipeCommandIT.testDoubleAppendPipeWithFilter' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java index 4debe504dad..8c17f6f6e1b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.Capability.LIKE_CASE_SENSITIVITY; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; @@ -15,6 +16,7 @@ import org.junit.Test; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.LikeQueryIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteLikeQueryIT extends LikeQueryIT { @Override @@ -49,6 +51,9 @@ public void test_ilike_is_case_insensitive() throws IOException { } @Test + @RequiresCapability( + value = LIKE_CASE_SENSITIVITY, + note = "case-sensitive LIKE (legacy=false) expects 0 rows; AE LIKE is case-insensitive.") public void test_the_default_3rd_option() throws IOException { // only work in v3 String query = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java index 6ae37a027ba..8414571812a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendPipeCommandIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.Capability.APPENDPIPE_MAIN_RESULT_DROPPED; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAppendPipeCommandIT extends PPLIntegTestCase { @Override @@ -145,6 +147,9 @@ public void testTripleAppendPipe() throws IOException { /** Regression test: double appendpipe with non-aggregation (filter) subpipeline. */ @Test + @RequiresCapability( + value = APPENDPIPE_MAIN_RESULT_DROPPED, + note = "appendpipe drops the main pipeline's rows on the AE route (filter applied in place).") public void testDoubleAppendPipeWithFilter() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index 84159c1bb96..bc9d4388d5b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -5,10 +5,12 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -23,6 +25,7 @@ import org.opensearch.client.Request; import org.opensearch.sql.legacy.TestsConstants; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLCaseFunctionIT extends PPLIntegTestCase { @@ -31,12 +34,20 @@ public void init() throws Exception { super.init(); enableCalcite(); + // Seed only on first creation: the AE parquet store is append-only on same-_id PUT. + boolean weblogsExisted = isIndexExist(client(), TEST_INDEX_WEBLOGS); loadIndex(Index.WEBLOG); loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.STATE_COUNTRY_WITH_NULL); loadIndex(Index.BANK); - loadIndex(Index.OTELLOGS); - appendDataForBadResponse(); + // otel_logs has a multi-value keyword the AE store rejects at load; only the (AE-skipped) + // testNestedCaseAggWithAutoDateHistogram needs it. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.OTELLOGS); + } + if (!weblogsExisted) { + appendDataForBadResponse(); + } } private void appendDataForBadResponse() throws IOException { @@ -478,6 +489,9 @@ public void testCaseAggWithNullValues() throws IOException { } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = "bin @timestamp then group by it: bucket column typed string (not timestamp) on AE.") public void testNestedCaseAggWithAutoDateHistogram() throws IOException { // TODO: Remove after resolving: https://github.com/opensearch-project/sql/issues/4578 Assume.assumeFalse( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java index f3d3852f938..e703e0fc366 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLStringBuiltinFunctionIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; +import static org.opensearch.sql.util.Capability.DOC_MUTATION; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLStringBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -60,6 +62,9 @@ public void testConcat() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testConcatWithField() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -78,6 +83,9 @@ public void testConcatWithField() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testConcatWs() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -212,6 +220,9 @@ public void testPosition() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -226,6 +237,9 @@ public void testTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testRTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -240,6 +254,9 @@ public void testRTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testLTrim() throws IOException { prepareTrim(); JSONObject actual = @@ -254,6 +271,9 @@ public void testLTrim() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testReverse() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); @@ -272,6 +292,9 @@ public void testReverse() throws IOException { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Re-PUTs a shared _id with different data; the append-only AE store can't replace it.") public void testRight() throws IOException { Request request1 = new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java index 36e9e2c4c56..45b8a6c0e68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class MultiMatchIT extends PPLIntegTestCase { @@ -45,6 +47,9 @@ public void test_multi_match_all_params() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void test_wildcard_multi_match() throws IOException { String query1 = "SOURCE=" + TEST_INDEX_BEER + " | WHERE multi_match(['Tags'], 'taste') | fields Id"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java index d800de51c80..44778c40839 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class QueryStringIT extends PPLIntegTestCase { @@ -56,6 +58,9 @@ public void all_params_test() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void wildcard_test() throws IOException { String query1 = "source=" + TEST_INDEX_BEER + " | where query_string(['Tags'], 'taste')"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java index d450ba46c51..d6be3a6d765 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java @@ -6,10 +6,12 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SimpleQueryStringIT extends PPLIntegTestCase { @Override @@ -45,6 +47,9 @@ public void test_simple_query_string_all_params() throws IOException { } @Test + @RequiresCapability( + value = FULLTEXT_RELEVANCE_FUNC, + note = "multi_match/query_string/simple_query_string are full-text relevance funcs.") public void test_wildcard_simple_query_string() throws IOException { String query1 = "SOURCE=" + TEST_INDEX_BEER + " | WHERE simple_query_string(['Tags'], 'taste') | fields Id"; diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index e0041e57c8c..f78e479bbd7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -192,7 +192,35 @@ public enum Capability { EVAL_MAX_MIN_INT_WIDENING( "eval max()/min() over integer operands reports the result column as bigint on the" + " analytics-engine route (DataFusion widens integers to Int64), whereas the v2/Calcite" - + " path reports int."); + + " path reports int."), + + /** + * Lucene full-text relevance functions ({@code match}, {@code multi_match}, {@code query_string}, + * {@code simple_query_string}, …) are unsupported on the analytics-engine route — DataFusion has + * no relevance scorer, so a query that filters on one returns no rows. + */ + FULLTEXT_RELEVANCE_FUNC( + "Full-text relevance functions (match/multi_match/query_string/simple_query_string) are" + + " unsupported on the analytics-engine route: DataFusion has no relevance scorer, so the" + + " filter returns no rows."), + + /** + * LIKE is case-insensitive on the analytics-engine route (DataFusion), whereas the v2/Calcite + * path treats {@code LIKE} as case-sensitive (only {@code ILIKE} is case-insensitive). + */ + LIKE_CASE_SENSITIVITY( + "LIKE is case-insensitive on the analytics-engine route (DataFusion), whereas the v2/Calcite" + + " path treats LIKE as case-sensitive."), + + /** + * {@code appendpipe [subpipe]} drops the main pipeline's rows on the analytics-engine route: the + * subpipe's filter is applied to the main result instead of its output being appended to it, so + * the original rows are lost (e.g. {@code stats ... | appendpipe [where gender='F']} returns only + * the filtered F rows, not the originals plus the filtered copy). + */ + APPENDPIPE_MAIN_RESULT_DROPPED( + "appendpipe drops the main pipeline's rows on the analytics-engine route: the subpipe filter" + + " is applied to the main result instead of appended, so the originals are lost."); private final String reason; From 7f2b60fa88c8b6fb19ae52e2a21200997d005683 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:28:13 -0700 Subject: [PATCH 03/41] Stabilize PPL ITs on the analytics-engine route (array/map-path/datatype/basic) (#5562) Analytics-engine route parity for four PPL IT classes; test-only. Uses the @RequiresCapability annotation + Capability registry (#5560) plus matching excludeTestsMatching entries. CalciteArrayFunctionIT: - Skip the array-index load on the AE route (multi-value 'numbers' field the parquet store rejects); no test queries it (all build arrays inline). - 16 higher-order lambda functions (transform/mvmap, reduce, filter, exists, forall) -> new ARRAY_HIGHER_ORDER_FUNC (no DataFusion lambda execution). CalcitePPLMapPathIT: - mvcombine lowers to ARRAY_AGG, unregistered on the analytics backend -> new MVCOMBINE_ARRAY_AGG. - addtotals crashes the DataFusion backend with a join panic -> new ADDTOTALS_JOIN_PANIC. CalciteDataTypeIT (guards on base DataTypeIT; build.gradle globs broadened to '*' so they cover the Calcite subclass): - test_nonnumeric_data_types / test_alias_data_type: nested/object/geo/alias types stripped (NESTED_FIELDS). - test_numeric_data_types: scaled_float reported as bigint not double -> new SCALED_FLOAT_TYPE. - testNumericFieldFromString: empty-string -> numeric coerces to null not 0 -> new STRING_TO_NUMERIC_COERCION. - testBooleanFieldFromNumberAcrossWildcardIndices: cross-index incompatible field types rejected -> new CROSS_INDEX_INCOMPATIBLE_TYPES. - testBooleanFieldFromString: seeds+deletes a doc; DELETE unsupported (DOC_MUTATION). CalcitePPLBasicIT.testRegexpFilter: REGEXP filter throws a backend NullPointerException on the AE route -> new REGEXP_FILTER. v2/Calcite route unchanged (all run, 0 skips). Signed-off-by: Kai Huang --- integ-test/build.gradle | 35 ++++++++ .../remote/CalciteArrayFunctionIT.java | 88 ++++++++++++++++++- .../sql/calcite/remote/CalcitePPLBasicIT.java | 5 ++ .../calcite/remote/CalcitePPLMapPathIT.java | 9 ++ .../org/opensearch/sql/ppl/DataTypeIT.java | 24 +++++ .../org/opensearch/sql/util/Capability.java | 61 ++++++++++++- 6 files changed, 220 insertions(+), 2 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2e565aab8f2..dd3053473b6 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1080,6 +1080,12 @@ task integTestRemote(type: RestIntegTestTask) { excludeTestsMatching '*SystemFunctionIT.typeof_opensearch_types' // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`. excludeTestsMatching '*DataTypeIT.test_alias_data_type' + // scaled_float reports bigint (not double); string->numeric coerces to null (not 0); + // cross-index incompatible field types are rejected; boolean-from-string test DELETEs. + excludeTestsMatching '*DataTypeIT.test_numeric_data_types' + excludeTestsMatching '*DataTypeIT.testNumericFieldFromString' + excludeTestsMatching '*DataTypeIT.testBooleanFieldFromNumberAcrossWildcardIndices' + excludeTestsMatching '*DataTypeIT.testBooleanFieldFromString' // CalciteAliasFieldAggregationIT: raw-PUT alias index can't be created on the AE route // and every test queries alias fields directly — whole class doomed. excludeTestsMatching 'org.opensearch.sql.calcite.remote.CalciteAliasFieldAggregationIT' @@ -1204,6 +1210,35 @@ task integTestRemote(type: RestIntegTestTask) { // === Excludes: CalcitePPLAppendPipeCommandIT route divergence === // appendpipe drops the main pipeline's rows on AE (subpipe filter applied in place). excludeTestsMatching '*CalcitePPLAppendPipeCommandIT.testDoubleAppendPipeWithFilter' + + // === Excludes: CalciteArrayFunctionIT route divergences === + // Higher-order array functions (transform/mvmap, reduce, filter, exists, forall) take a + // PPL lambda the AE backends can't execute ('No backend supports scalar function [...]'). + excludeTestsMatching '*CalciteArrayFunctionIT.testForAll' + excludeTestsMatching '*CalciteArrayFunctionIT.testExists' + excludeTestsMatching '*CalciteArrayFunctionIT.testFilter' + excludeTestsMatching '*CalciteArrayFunctionIT.testTransform' + excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForTwoInput' + excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForWithDouble' + excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForWithUDF' + excludeTestsMatching '*CalciteArrayFunctionIT.testReduce' + excludeTestsMatching '*CalciteArrayFunctionIT.testReduce2' + excludeTestsMatching '*CalciteArrayFunctionIT.testReduce3' + excludeTestsMatching '*CalciteArrayFunctionIT.testReduceWithUDF' + excludeTestsMatching '*CalciteArrayFunctionIT.testMvmap' + excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithAddition' + excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithEvalFieldReference' + excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithNestedFunction' + excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithOtherFieldReference' + + // === Excludes: CalcitePPLMapPathIT route divergences === + // mvcombine -> ARRAY_AGG (no such AggregateFunction enum); addtotals -> DataFusion join panic. + excludeTestsMatching '*CalcitePPLMapPathIT.testMvcombineOnMapPath' + excludeTestsMatching '*CalcitePPLMapPathIT.testAddtotalsOnMapPath' + + // === Excludes: CalcitePPLBasicIT route divergence === + // REGEXP filter throws a backend NullPointerException on the AE route. + excludeTestsMatching '*CalcitePPLBasicIT.testRegexpFilter' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java index 26e8ecf73f6..12b572f44a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.Capability.ARRAY_HIGHER_ORDER_FUNC; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; @@ -15,6 +16,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteArrayFunctionIT extends PPLIntegTestCase { @Override @@ -22,7 +24,11 @@ public void init() throws Exception { super.init(); enableCalcite(); loadIndex(Index.BANK); - loadIndex(Index.ARRAY); + // No test queries the array index (all build arrays inline via array()); its multi-value + // numbers field can't be bulk-loaded into the parquet store, so skip it on the AE route. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.ARRAY); + } } @Test @@ -85,6 +91,11 @@ public void testArrayLength() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testForAll() throws IOException { JSONObject actual = executeQuery( @@ -99,6 +110,11 @@ public void testForAll() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testExists() throws IOException { JSONObject actual = executeQuery( @@ -113,6 +129,11 @@ public void testExists() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testFilter() throws IOException { JSONObject actual = executeQuery( @@ -127,6 +148,11 @@ public void testFilter() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testTransform() throws IOException { JSONObject actual = executeQuery( @@ -141,6 +167,11 @@ public void testTransform() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testTransformForTwoInput() throws IOException { JSONObject actual = executeQuery( @@ -155,6 +186,11 @@ public void testTransformForTwoInput() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testTransformForWithDouble() throws IOException { JSONObject actual = executeQuery( @@ -169,6 +205,11 @@ public void testTransformForWithDouble() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testTransformForWithUDF() throws IOException { JSONObject actual = executeQuery( @@ -185,6 +226,11 @@ public void testTransformForWithUDF() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testReduce() throws IOException { JSONObject actual = executeQuery( @@ -202,6 +248,11 @@ public void testReduce() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testReduce2() throws IOException { JSONObject actual = executeQuery( @@ -216,6 +267,11 @@ public void testReduce2() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testReduce3() throws IOException { JSONObject actual = executeQuery( @@ -231,6 +287,11 @@ public void testReduce3() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testReduceWithUDF() throws IOException { JSONObject actual = executeQuery( @@ -796,6 +857,11 @@ public void testSplitWithEmptyDelimiter() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testMvmap() throws IOException { JSONObject actual = executeQuery( @@ -809,6 +875,11 @@ public void testMvmap() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testMvmapWithAddition() throws IOException { JSONObject actual = executeQuery( @@ -822,6 +893,11 @@ public void testMvmapWithAddition() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testMvmapWithNestedFunction() throws IOException { // Test mvmap with mvindex as first argument - extracts field name from nested function // Equivalent to Splunk: mvmap(mvindex(arr, 1, 3), arr * 10) @@ -839,6 +915,11 @@ public void testMvmapWithNestedFunction() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testMvmapWithOtherFieldReference() throws IOException { // Test mvmap with reference to another field in the expression // The first record in bank has age=32, so array(1,2,3) * 32 = [32, 64, 96] @@ -854,6 +935,11 @@ public void testMvmapWithOtherFieldReference() throws IOException { } @Test + @RequiresCapability( + value = ARRAY_HIGHER_ORDER_FUNC, + note = + "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a" + + " lambda.") public void testMvmapWithEvalFieldReference() throws IOException { // Test mvmap with reference to another field created by eval // array(1,2,3) * 10 = [10, 20, 30] diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index dbf7f32ca96..1a826266f19 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -8,6 +8,7 @@ import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.REGEXP_FILTER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +22,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBasicIT extends PPLIntegTestCase { @@ -154,6 +156,9 @@ public void testFilterQuery4() throws IOException { } @Test + @RequiresCapability( + value = REGEXP_FILTER, + note = "REGEXP filter throws a backend NullPointerException on the AE route.") public void testRegexpFilter() throws IOException { JSONObject actual = executeQuery("source=test | where name REGEXP 'he.*' | fields name, age"); verifySchema(actual, schema("name", "string"), schema("age", "bigint")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java index 0e24bc33d38..04ccba25b6c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.util.Capability.ADDTOTALS_JOIN_PANIC; +import static org.opensearch.sql.util.Capability.MVCOMBINE_ARRAY_AGG; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +24,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for PPL queries that reference MAP dotted paths (e.g. {@code doc.user.name}). @@ -141,6 +144,9 @@ public void testFieldsExclusionOnMapPath() throws IOException { } @Test + @RequiresCapability( + value = ADDTOTALS_JOIN_PANIC, + note = "addtotals crashes the DataFusion backend with a join panic on the AE route.") public void testAddtotalsOnMapPath() throws IOException { JSONObject result = ppl( @@ -163,6 +169,9 @@ public void testAddtotalsOnMapPath() throws IOException { } @Test + @RequiresCapability( + value = MVCOMBINE_ARRAY_AGG, + note = "mvcombine lowers to ARRAY_AGG, unregistered on the analytics backend.") public void testMvcombineOnMapPath() throws IOException { JSONObject result = ppl( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 25e7c12ffff..1af872a8ab6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -11,6 +11,11 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.CROSS_INDEX_INCOMPATIBLE_TYPES; +import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; +import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; +import static org.opensearch.sql.util.Capability.STRING_TO_NUMERIC_COERCION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -23,6 +28,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.sql.util.RequiresCapability; public class DataTypeIT extends PPLIntegTestCase { @@ -35,6 +41,9 @@ public void init() throws Exception { } @Test + @RequiresCapability( + value = SCALED_FLOAT_TYPE, + note = "scaled_float reports bigint (not double) on the AE route.") public void test_numeric_data_types() throws IOException { JSONObject result = executeQuery(String.format("source=%s", TEST_INDEX_DATATYPE_NUMERIC)); verifySchema( @@ -50,6 +59,9 @@ public void test_numeric_data_types() throws IOException { } @Test + @RequiresCapability( + value = NESTED_FIELDS, + note = "nested_value/object_value/geo_point_value: stripped/unsupported on the AE route.") public void test_nonnumeric_data_types() throws IOException { JSONObject result = executeQuery(String.format("source=%s", TEST_INDEX_DATATYPE_NONNUMERIC)); verifySchemaInOrder( @@ -102,6 +114,9 @@ public void test_long_integer_data_type() throws IOException { } @Test + @RequiresCapability( + value = NESTED_FIELDS, + note = "alias_col is type=alias, stripped from the mapping on the AE route.") public void test_alias_data_type() throws IOException { JSONObject result = executeQuery( @@ -113,6 +128,9 @@ public void test_alias_data_type() throws IOException { } @Test + @RequiresCapability( + value = STRING_TO_NUMERIC_COERCION, + note = "empty-string -> numeric coerces to 0 on v2/Calcite but null on the AE route.") public void testNumericFieldFromString() throws Exception { final int docId = 2; Request insertRequest = @@ -146,6 +164,9 @@ public void testNumericFieldFromString() throws Exception { } @Test + @RequiresCapability( + value = CROSS_INDEX_INCOMPATIBLE_TYPES, + note = "AE route rejects incompatible cross-index field types instead of coercing.") public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { // Reproduce issue #5269: querying across indices where same field has conflicting types // (boolean vs text) and the text-typed index stores a numeric value like 0. @@ -187,6 +208,9 @@ public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { } @Test + @RequiresCapability( + value = DOC_MUTATION, + note = "Seeds + deletes a doc; the AE store doesn't support DELETE.") public void testBooleanFieldFromString() throws Exception { final int docId = 2; Request insertRequest = diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index f78e479bbd7..d54b3ac74a9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -220,7 +220,66 @@ public enum Capability { */ APPENDPIPE_MAIN_RESULT_DROPPED( "appendpipe drops the main pipeline's rows on the analytics-engine route: the subpipe filter" - + " is applied to the main result instead of appended, so the originals are lost."); + + " is applied to the main result instead of appended, so the originals are lost."), + + /** + * Higher-order array functions that take a lambda ({@code transform}/{@code mvmap}, {@code + * reduce}, {@code filter}, {@code exists}, {@code forall}) are unsupported on the + * analytics-engine route: the capability registry rejects them ({@code No backend supports scalar + * function [...] among [lucene, datafusion]}) since the backends can't execute a PPL lambda. + */ + ARRAY_HIGHER_ORDER_FUNC( + "Higher-order array functions (transform/mvmap, reduce, filter, exists, forall) are" + + " unsupported on the analytics-engine route: the backends can't execute a PPL lambda."), + + /** + * {@code scaled_float} fields are reported as {@code bigint} on the analytics-engine route + * (DataFusion stores the underlying scaled long) rather than {@code double} as on v2/Calcite. + */ + SCALED_FLOAT_TYPE( + "scaled_float is reported as bigint on the analytics-engine route (DataFusion stores the" + + " scaled long), whereas the v2/Calcite path reports double."), + + /** + * Coercing an empty string to a numeric field yields {@code null} on the analytics-engine route, + * whereas the v2/Calcite path coerces it to {@code 0}. + */ + STRING_TO_NUMERIC_COERCION( + "Coercing an empty/invalid string to a numeric field yields null on the analytics-engine" + + " route, whereas the v2/Calcite path coerces it to 0."), + + /** + * A wildcard/alias source whose member indices map the same field to incompatible types (e.g. + * {@code text} in one, {@code boolean} in another) is rejected on the analytics-engine route + * ({@code resolves to indices with incompatible field types}); the v2/Calcite path coerces. + */ + CROSS_INDEX_INCOMPATIBLE_TYPES( + "A wildcard/alias source with incompatible field types across member indices is rejected on" + + " the analytics-engine route, whereas the v2/Calcite path coerces."), + + /** + * The {@code REGEXP} filter operator throws a backend NullPointerException on the + * analytics-engine route. + */ + REGEXP_FILTER( + "The REGEXP filter operator throws a backend NullPointerException on the analytics-engine" + + " route."), + + /** + * {@code mvcombine} lowers to an {@code ARRAY_AGG} aggregate the analytics-engine backend doesn't + * register ({@code No enum constant ... AggregateFunction.ARRAY_AGG}). + */ + MVCOMBINE_ARRAY_AGG( + "mvcombine lowers to ARRAY_AGG, which the analytics-engine backend does not support (no" + + " AggregateFunction.ARRAY_AGG enum constant)."), + + /** + * {@code addtotals} crashes the DataFusion backend with a join panic (out-of-range slice index) + * on the analytics-engine route. + */ + ADDTOTALS_JOIN_PANIC( + "addtotals crashes the DataFusion backend with a join panic (out-of-range slice index) on the" + + " analytics-engine route."); private final String reason; From 08d8ba10a76b89d900c2a77dc2cbb9216e47855c Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:44:13 -0700 Subject: [PATCH 04/41] Stabilize PPL ITs on the analytics-engine route (percentile/float/datetime/json/dedup/union/rename/chart) (#5564) Brings 14 PPL IT classes to parity on the analytics-engine route (-Dtests.analytics.parquet_indices=true). Each gated test is skipped ONLY on the AE route via @RequiresCapability + a matching build.gradle excludeTestsMatching; the v2/Calcite path runs all of them unchanged. Engine divergences gated (route-only, behavior is correct elsewhere): - PERCENTILE_APPROXIMATE: percentile/median is approximate on DataFusion - FLOAT_ARITHMETIC_PRECISION: float/half_float arithmetic keeps 32-bit - DATETIME_FORMAT_RENDERING: date_format/strftime token rendering differs - UNIX_TIMESTAMP_SUBSECOND: unix_timestamp drops sub-second precision - JSON_DOLLAR_PATH: json_set/json_delete with $-path is a no-op - DEDUP_NONDETERMINISTIC: dedup surviving-row selection is unstable - SAME_INDEX_UNION_CONFLATION: same-index union conflates (delegate leak) - WILDCARD_COLUMN_ORDER: rename * column order differs - BIN_TIME_FIELD_BUCKETING: span() time bucketing differs - MULTI_VALUE_FIELD_LOAD: otel_logs/game_of_thrones multi-value field can't bulk-load into the parquet store Two cases were harness-predicate gaps, not real divergences, so they now PASS on AE instead of being gated: testStatsPercentileWithMin and testTimestampDiff branch on isCalciteEnabled() for the result type, but the AE route runs the Calcite path while the cluster setting reads false; extended the predicate with isAnalyticsParquetIndicesEnabled(). Guarded the otel_logs / game_of_thrones loads in CalciteChartCommandIT and CalcitePPLJsonBuiltinFunctionIT init() so a multi-value bulk-load failure no longer aborts init() and mislabels unrelated tests. CalciteAnalyticsDatetimeWireFormatIT (AE-only via assumeTrue) updated to assert the date/time UDT types AE now preserves. Results (this batch, on the AE route): - CalcitePPLAggregationIT: 100 run, 2 skip - CalcitePPLBuiltinFunctionIT: 26 run, 3 skip - CalciteDateTimeFunctionIT: 65 run, 5 skip - CalcitePPLDedupIT: 15 run, 3 skip - CalcitePPLJsonBuiltinFunctionIT: 22 run, 2 skip - CalcitePPLRenameIT: 24 run, 1 skip - CalciteUnionCommandIT: 15 run, 2 skip - CalciteChartCommandIT: 15 run, 5 skip - DateTimeFunctionIT: 59 run, 4 skip - StatsCommandIT: 59 run, 6 skip - SystemFunctionIT: 1 run - CalciteAnalyticsDatetimeWireFormatIT: 11 run AE route: 0 failures (was 30+ fail across the batch). V2 baseline: 402 run, 0 fail, 3 pre-existing skips, 0 from these gates. Signed-off-by: Kai Huang --- integ-test/build.gradle | 46 ++++++++++ .../CalciteAnalyticsDatetimeWireFormatIT.java | 17 ++-- .../calcite/remote/CalciteChartCommandIT.java | 36 +++++++- .../remote/CalciteDateTimeFunctionIT.java | 5 ++ .../remote/CalcitePPLAggregationIT.java | 8 ++ .../remote/CalcitePPLBuiltinFunctionIT.java | 11 +++ .../sql/calcite/remote/CalcitePPLDedupIT.java | 13 +++ .../CalcitePPLJsonBuiltinFunctionIT.java | 15 +++- .../calcite/remote/CalcitePPLRenameIT.java | 5 ++ .../calcite/remote/CalciteUnionCommandIT.java | 8 ++ .../sql/ppl/DateTimeFunctionIT.java | 19 +++- .../opensearch/sql/ppl/StatsCommandIT.java | 22 ++++- .../opensearch/sql/ppl/SystemFunctionIT.java | 5 ++ .../org/opensearch/sql/util/Capability.java | 87 ++++++++++++++++++- 14 files changed, 281 insertions(+), 16 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index dd3053473b6..9f51ba2c222 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1239,6 +1239,52 @@ task integTestRemote(type: RestIntegTestTask) { // === Excludes: CalcitePPLBasicIT route divergence === // REGEXP filter throws a backend NullPointerException on the AE route. excludeTestsMatching '*CalcitePPLBasicIT.testRegexpFilter' + + // === Excludes: percentile is approximate on AE (DataFusion) but exact on v2/Calcite === + excludeTestsMatching '*StatsCommandIT.testStatsPercentileWithNull' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileByNullValue' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileByNullValueNonNullBucket' + excludeTestsMatching '*StatsCommandIT.testStatsPercentileBySpan' + excludeTestsMatching '*CalcitePPLAggregationIT.testPercentile' + excludeTestsMatching '*CalcitePPLAggregationIT.testPercentileShortcutsFloatingPoint' + + // === Excludes: span() time-field bucketing differs on the AE route === + excludeTestsMatching '*StatsCommandIT.testStatsBySpanTimeWithNullBucket' + excludeTestsMatching '*CalciteChartCommandIT.testChartMaxValueByTimestampSpanDayAndWeek' + + // === Excludes: float/half_float arithmetic keeps 32-bit precision on AE === + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testDivide' + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testModFloatAndNegative' + excludeTestsMatching '*CalcitePPLBuiltinFunctionIT.testModShouldReturnWiderTypes' + + // === Excludes: date_format/strftime render some tokens differently on AE === + excludeTestsMatching '*DateTimeFunctionIT.testDateFormat' + excludeTestsMatching '*CalciteDateTimeFunctionIT.testStrftimeWithDateFields' + + // === Excludes: unix_timestamp drops sub-second precision on AE === + excludeTestsMatching '*DateTimeFunctionIT.testUnixTimestampWithTimestampString' + + // === Excludes: json_set/json_delete with a $-prefixed path is a no-op on AE === + excludeTestsMatching '*CalcitePPLJsonBuiltinFunctionIT.testJsonSetWithDollarPrefixedPath' + excludeTestsMatching '*CalcitePPLJsonBuiltinFunctionIT.testJsonDeleteWithDollarPrefixedPath' + + // === Excludes: dedup surviving-row selection is non-deterministic on AE === + excludeTestsMatching '*CalcitePPLDedupIT.testDedupComplex' + excludeTestsMatching '*CalcitePPLDedupIT.testDedupExpr' + excludeTestsMatching '*CalcitePPLDedupIT.testConsecutiveImplicitFallbackV2' + + // === Excludes: same-index union conflates on AE (delegated predicate leak) === + excludeTestsMatching '*CalciteUnionCommandIT.testUnionThreeSubsearches' + excludeTestsMatching '*CalciteUnionCommandIT.testUnionMidPipeline_SingleExplicitDataset' + + // === Excludes: rename * returns columns in a different order on AE === + excludeTestsMatching '*CalcitePPLRenameIT.testRenameFullWildcardExcludesMetadataFields' + + // === Excludes: otel_logs multi-value field can't load into the parquet store === + excludeTestsMatching '*CalciteChartCommandIT.testChartLimit0WithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitBottomWithUseOther' + excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithMinAgg' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java index 36dcf5697c8..3805759c65e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java @@ -92,29 +92,24 @@ public void testTimestampRootColumnSpaceFormat() throws IOException { verifyDataRows(result, rows("2024-03-15 10:30:00")); } - /** - * DATE-mapped col: AE widens to TIMESTAMP at scan time; value must use space separator, not ISO - * {@code T}. - */ + /** DATE-mapped col: AE preserves the DATE type (date UDT) and renders {@code yyyy-MM-dd}. */ @Test public void testDateRootColumnYmdFormat() throws IOException { String query = "source=" + INDEX + " | where d = '2024-03-15' | fields d"; assertRoutedToAnalyticsEngine(query); JSONObject result = executeQuery(query); - verifySchema(result, schema("d", "timestamp")); - verifyDataRows(result, rows("2024-03-15 00:00:00")); + verifySchema(result, schema("d", "date")); + verifyDataRows(result, rows("2024-03-15")); } - /** TIME-mapped col: AE widens to TIMESTAMP; value must use space separator, not ISO {@code T}. */ + /** TIME-mapped col: AE preserves the TIME type (time UDT) and renders {@code HH:mm:ss}. */ @Test public void testTimeRootColumnHmsFormat() throws IOException { String query = "source=" + INDEX + " | sort t | head 1 | fields t"; assertRoutedToAnalyticsEngine(query); JSONObject result = executeQuery(query); - verifySchema(result, schema("t", "timestamp")); - Assert.assertFalse( - "Time-mapped column must not surface as ISO T-separator literal", - result.getJSONArray("datarows").getJSONArray(0).getString(0).contains("T")); + verifySchema(result, schema("t", "time")); + verifyDataRows(result, rows("10:30:00")); } /** Eval-derived TIMESTAMP follows the same wire-format contract as a root column. */ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java index e687751ef0c..ed360bc79d4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteChartCommandIT.java @@ -9,6 +9,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; +import static org.opensearch.sql.util.Capability.MULTI_VALUE_FIELD_LOAD; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -20,6 +22,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteChartCommandIT extends PPLIntegTestCase { @Override @@ -28,7 +31,13 @@ public void init() throws Exception { enableCalcite(); loadIndex(Index.BANK); loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.OTELLOGS); + // otel_logs has a multi-value array for a scalar-mapped field, which the parquet store rejects + // at bulk load (MULTI_VALUE_FIELD_LOAD); skip the load on the AE route so it doesn't abort + // init() for the otel-independent tests. The otel tests themselves are + // @RequiresCapability-gated. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.OTELLOGS); + } loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.EVENTS_NULL); } @@ -142,6 +151,11 @@ public void testChartMaxValueOverCategoryByTimestampSpanWeek() throws IOExceptio } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = + "span=2weeks anchors the bucket to a different week origin on the AE route" + + " (BIN_TIME_FIELD_BUCKETING).") public void testChartMaxValueByTimestampSpanDayAndWeek() throws IOException { JSONObject result = executeQuery( @@ -165,6 +179,11 @@ public void testChartMaxValueByTimestampSpanDayAndWeek() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimit0WithUseOther() throws IOException { JSONObject result = executeQuery( @@ -208,6 +227,11 @@ public void testChartLimit0WithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitTopWithUseOther() throws IOException { JSONObject result = executeQuery( @@ -230,6 +254,11 @@ public void testChartLimitTopWithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitBottomWithUseOther() throws IOException { JSONObject result = executeQuery( @@ -246,6 +275,11 @@ public void testChartLimitBottomWithUseOther() throws IOException { } @Test + @RequiresCapability( + value = MULTI_VALUE_FIELD_LOAD, + note = + "reads otel_logs whose multi-value field can't load on the AE store" + + " (MULTI_VALUE_FIELD_LOAD).") public void testChartLimitTopWithMinAgg() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java index ef0c0599b57..cb74251f5d8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDateTimeFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ppl.DateTimeFunctionIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteDateTimeFunctionIT extends DateTimeFunctionIT { @Override @@ -73,6 +75,9 @@ public void testStrftimeWithVariousInputTypes() throws IOException { } @Test + @RequiresCapability( + value = DATETIME_FORMAT_RENDERING, + note = "strftime renders sub-second precision differently on the AE route.") public void testStrftimeWithDateFields() throws IOException { // Test strftime with different date field types from indices loadIndex(Index.DATE_FORMATS); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a5937d06f31..a2ab93b6599 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TELEMETRY; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -28,6 +29,7 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAggregationIT extends PPLIntegTestCase { @@ -967,6 +969,9 @@ public void testTake() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentile() throws IOException { JSONObject actual = executeQuery( @@ -1184,6 +1189,9 @@ public void testPercentileShortcutsWithDecimals() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testPercentileShortcutsFloatingPoint() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java index 58f4cb849b3..cbd94683fd1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.FLOAT_ARITHMETIC_PRECISION; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -255,6 +257,9 @@ public void testModWithSortAndFields() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModFloatAndNegative() throws IOException { JSONObject actual = executeQuery( @@ -267,6 +272,9 @@ public void testModFloatAndNegative() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float modulo keeps 32-bit precision on AE; v2 widens to double.") public void testModShouldReturnWiderTypes() throws IOException { JSONObject actual = executeQuery( @@ -347,6 +355,9 @@ public void testSignAndRound() throws IOException { } @Test + @RequiresCapability( + value = FLOAT_ARITHMETIC_PRECISION, + note = "float/half_float division keeps 32-bit precision on AE; v2 widens to double.") public void testDivide() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java index 71e9e69e3ae..9c93b12e6ac 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java @@ -7,12 +7,14 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE; +import static org.opensearch.sql.util.Capability.DEDUP_NONDETERMINISTIC; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLDedupIT extends PPLIntegTestCase { @@ -95,6 +97,9 @@ public void testDedupMultipleFieldsKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = "dedup CONSECUTIVE behavior diverges on the AE route.") public void testConsecutiveImplicitFallbackV2() throws IOException { JSONObject actual = executeQuery( @@ -252,6 +257,10 @@ public void testReorderDedupFieldsShouldNotAffectResult() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupComplex() throws IOException { JSONObject actual = executeQuery(String.format("source=%s | dedup 1 name", TEST_INDEX_DUPLICATION_NULLABLE)); @@ -364,6 +373,10 @@ public void testSortThenDedupKeepEmpty() throws IOException { } @Test + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "dedup surviving-duplicate selection diverges on the AE route (no stable merge order).") public void testDedupExpr() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java index 99af10302ae..02ebae95e8e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJsonBuiltinFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.JSON_DOLLAR_PATH; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -15,6 +16,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLJsonBuiltinFunctionIT extends PPLIntegTestCase { @Override @@ -30,7 +32,12 @@ public void init() throws Exception { loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); loadIndex(Index.JSON_TEST); - loadIndex(Index.GAME_OF_THRONES); + // game_of_thrones has a multi-value array for the scalar-mapped `titles` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init() for + // the rest of the suite. No test in this class queries game_of_thrones. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.GAME_OF_THRONES); + } } @Test @@ -297,6 +304,9 @@ public void testJsonSetPartialSet() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_set with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonSetWithDollarPrefixedPath() throws IOException { // Issue #5167: json_set with $.key path should not double-prefix JSONObject actual = @@ -313,6 +323,9 @@ public void testJsonSetWithDollarPrefixedPath() throws IOException { } @Test + @RequiresCapability( + value = JSON_DOLLAR_PATH, + note = "json_delete with a $-prefixed path is a no-op on the AE route (JSON_DOLLAR_PATH).") public void testJsonDeleteWithDollarPrefixedPath() throws IOException { // Issue #5167: json_delete with $.key path should remove the key JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 3503d7c533c..6bac3032bd2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; +import static org.opensearch.sql.util.Capability.WILDCARD_COLUMN_ORDER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +22,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLRenameIT extends PPLIntegTestCase { @@ -204,6 +206,9 @@ public void testRenameFullWildcard() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = "rename * returns columns in a different order on the AE route.") public void testRenameFullWildcardExcludesMetadataFields() throws IOException { JSONObject result = executeQuery(String.format("source = %s | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java index 1dbd34357ab..c0151a3b232 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteUnionCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOCATIONS_TYPE_CONFLICT; +import static org.opensearch.sql.util.Capability.SAME_INDEX_UNION_CONFLATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteUnionCommandIT extends PPLIntegTestCase { @@ -48,6 +50,9 @@ public void testBasicUnionTwoSubsearches() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionThreeSubsearches() throws IOException { JSONObject result = executeQuery( @@ -155,6 +160,9 @@ public void testUnionAllDatasetsDifferentSchemas() throws IOException { } @Test + @RequiresCapability( + value = SAME_INDEX_UNION_CONFLATION, + note = "same-index union conflates on the AE route (delegated predicate leak).") public void testUnionMidPipeline_SingleExplicitDataset() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 1a2911794f9..97b1d9601a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -7,7 +7,9 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.UNIX_TIMESTAMP_SUBSECOND; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -1219,6 +1221,9 @@ void verifyDateFormat(String date, String type, String format, String formatted) } @Test + @RequiresCapability( + value = DATETIME_FORMAT_RENDERING, + note = "date_format renders some tokens differently on the AE route.") public void testDateFormat() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; String timestampFormat = @@ -1369,6 +1374,11 @@ public void testUnixTimeStamp() throws IOException { } @Test + @RequiresCapability( + value = UNIX_TIMESTAMP_SUBSECOND, + note = + "unix_timestamp drops the sub-second fraction on the AE route" + + " (UNIX_TIMESTAMP_SUBSECOND).") public void testUnixTimestampWithTimestampString() throws IOException { var result = executeQuery( @@ -1557,7 +1567,14 @@ public void testTimestampDiff() throws IOException { "source=%s | eval f = timestampdiff(YEAR, '1997-01-01 00:00:00', '2001-03-06" + " 00:00:00') | fields f", TEST_INDEX_DATE)); - verifySchema(result, schema("f", null, isCalciteEnabled() ? "bigint" : "timestamp")); + // The AE route runs the Calcite path, returning bigint even though the cluster's calcite + // setting reads false. + verifySchema( + result, + schema( + "f", + null, + isCalciteEnabled() || isAnalyticsParquetIndicesEnabled() ? "bigint" : "timestamp")); verifySome(result.getJSONArray("datarows"), rows(4)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index adbe04bfd8a..7417fd112ec 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -9,6 +9,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATE_NULL; +import static org.opensearch.sql.util.Capability.BIN_TIME_FIELD_BUCKETING; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -19,6 +21,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.util.RequiresCapability; public class StatsCommandIT extends PPLIntegTestCase { @@ -622,8 +625,10 @@ public void testStatsPercentileWithMin() throws IOException { "source=%s | eval decimal=ceil(balance/100000.0) | stats percentile(decimal, 50)," + " min(decimal)", TEST_INDEX_BANK)); + // The AE route runs the Calcite path, so it returns the Calcite (double) type even though the + // cluster's calcite setting reads false; treat it like the Calcite branch. String returnType = "bigint"; - if (isCalciteEnabled()) { + if (isCalciteEnabled() || isAnalyticsParquetIndicesEnabled()) { returnType = "double"; } @@ -635,6 +640,9 @@ public void testStatsPercentileWithMin() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileWithNull() throws IOException { JSONObject response = executeQuery( @@ -673,6 +681,9 @@ public void testStatsPercentileWhere() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValue() throws IOException { JSONObject response = executeQuery( @@ -691,6 +702,9 @@ public void testStatsPercentileByNullValue() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileByNullValueNonNullBucket() throws IOException { JSONObject response = executeQuery( @@ -708,6 +722,9 @@ public void testStatsPercentileByNullValueNonNullBucket() throws IOException { } @Test + @RequiresCapability( + value = PERCENTILE_APPROXIMATE, + note = "percentile is approximate on the AE route but exact on v2/Calcite.") public void testStatsPercentileBySpan() throws IOException { JSONObject response = executeQuery( @@ -755,6 +772,9 @@ public void testDisableLegacyPreferred() throws IOException { } @Test + @RequiresCapability( + value = BIN_TIME_FIELD_BUCKETING, + note = "span() time bucketing differs on the AE route (bucket set/null bucket).") public void testStatsBySpanTimeWithNullBucket() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index b0e119bffb1..276ab01da4c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -9,12 +9,14 @@ import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SystemFunctionIT extends PPLIntegTestCase { @@ -53,6 +55,9 @@ public void typeof_sql_types() throws IOException { } @Test + @RequiresCapability( + value = SCALED_FLOAT_TYPE, + note = "typeof(scaled_float) is bigint on the AE route, not double.") public void typeof_opensearch_types() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index d54b3ac74a9..b7e95bfc4f1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -279,7 +279,92 @@ public enum Capability { */ ADDTOTALS_JOIN_PANIC( "addtotals crashes the DataFusion backend with a join panic (out-of-range slice index) on the" - + " analytics-engine route."); + + " analytics-engine route."), + + /** + * {@code percentile}/{@code median} is approximate on the analytics-engine route (DataFusion's + * approx percentile) but exact on the v2/Calcite path, so percentile values, null-bucket rows, + * and by-span groupings diverge. + */ + PERCENTILE_APPROXIMATE( + "percentile/median is approximate on the analytics-engine route (DataFusion) but exact on the" + + " v2/Calcite path, so the values diverge."), + + /** + * Arithmetic over {@code float}/{@code half_float}-typed fields keeps 32-bit float precision on + * the analytics-engine route (DataFusion), whereas the v2/Calcite path widens to double, so the + * least-significant digits diverge (e.g. 0.2 vs 0.19999981). + */ + FLOAT_ARITHMETIC_PRECISION( + "Arithmetic over float/half_float fields keeps 32-bit precision on the analytics-engine route" + + " (DataFusion) but widens to double on the v2/Calcite path, so the values diverge in" + + " the least-significant digits."), + + /** + * Datetime formatting functions ({@code date_format}, {@code strftime}) render some tokens / + * sub-second precision differently on the analytics-engine route than on the v2/Calcite path. + */ + DATETIME_FORMAT_RENDERING( + "date_format/strftime render some format tokens and sub-second precision differently on the" + + " analytics-engine route than the v2/Calcite path."), + + /** + * {@code json_set}/{@code json_delete} with a {@code $}-prefixed path ({@code $.key}) is a no-op + * on the analytics-engine route (the JSON UDF doesn't strip the {@code $} prefix), whereas the + * v2/Calcite path applies the modification. + */ + JSON_DOLLAR_PATH( + "json_set/json_delete with a $-prefixed path is a no-op on the analytics-engine route (the" + + " JSON UDF doesn't handle the $ prefix), whereas the v2/Calcite path applies it."), + + /** + * A dataset whose document has a multi-value array for a scalar-mapped field can't be bulk-loaded + * into the parquet/composite store ({@code Cannot accept multiple values for field ...}), so + * tests reading that dataset fail at setup on the analytics-engine route. + */ + MULTI_VALUE_FIELD_LOAD( + "A multi-value array for a scalar-mapped field can't be bulk-loaded into the parquet store on" + + " the analytics-engine route, so the dataset fails to load."), + + /** + * {@code dedup} returns a different/non-deterministic row set on the analytics-engine route — the + * engine merges per-fragment batches without a stable tiebreaker, so which duplicate survives + * (and {@code CONSECUTIVE=true} behavior) diverges from the v2/Calcite path. + */ + DEDUP_NONDETERMINISTIC( + "dedup returns a different row set on the analytics-engine route: per-fragment merge order" + + " has no stable tiebreaker, so the surviving duplicate (and CONSECUTIVE behavior)" + + " diverges."), + + /** + * {@code union}/{@code multisearch} over subsearches that read the same index conflates on the + * analytics-engine route: a delegated predicate from one branch leaks onto the co-located shard + * fragment and is applied to all branches, so counts/rows are wrong. Same root cause as {@link + * #MULTISEARCH_SAME_INDEX_CONFLATION} / {@link #APPENDPIPE_MAIN_RESULT_DROPPED}. + */ + SAME_INDEX_UNION_CONFLATION( + "union over same-index subsearches conflates on the analytics-engine route: a delegated" + + " predicate from one branch leaks across the co-located shard fragment, so counts/rows" + + " are wrong."), + + /** + * A wildcard projection/rename ({@code rename * as ...}, {@code fields *}) returns columns in a + * different order on the analytics-engine route (e.g. not mapping order) than the v2/Calcite + * path, so row-position-sensitive assertions diverge even though the values are correct. + */ + WILDCARD_COLUMN_ORDER( + "A wildcard projection/rename returns columns in a different order on the analytics-engine" + + " route than the v2/Calcite path."), + + /** + * {@code unix_timestamp()} over a timestamp string with sub-second precision drops the fractional + * seconds on the analytics-engine route (e.g. {@code unix_timestamp('1984-06-06 + * 12:00:00.123456')} returns {@code 455371200} instead of {@code 455371200.123456}), whereas the + * v2/Calcite path preserves them. + */ + UNIX_TIMESTAMP_SUBSECOND( + "unix_timestamp() drops sub-second precision on the analytics-engine route (returns whole" + + " seconds), whereas the v2/Calcite path preserves the fractional seconds."); private final String reason; From 7dca5cc324e38130afbbf217cddc102fd2a80084 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Fri, 19 Jun 2026 10:50:31 -0700 Subject: [PATCH 05/41] fix: Honor PPL fetch_size on the analytics-engine route (#5567) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Honor PPL fetch_size on the analytics-engine route PPL's fetch_size caps a response to N rows (no cursor) — the V2 path lowers it to a top-level `head N` in AstStatementBuilder.visitPplStatement. The analytics-engine route bypasses that builder: TransportPPLQueryAction forwards only the query string to RestUnifiedQueryAction.execute(), so fetch_size was dropped and the engine returned the full result set. Thread the request's fetchSize through execute() and apply an equivalent top-level limit on the planned RelNode (addFetchSizeLimit), using the same relBuilder.limit primitive that `head` lowers to. fetchSize <= 0 keeps the prior "system default" behavior; the SQL path (separate cursor-based fetch_size) is unchanged. Before/after (CalcitePPLFetchSizeIT-equivalent, analytics-engine route): before: 9/19 pass (10 fail — fetch_size ignored, full set returned) after: 19/19 pass Signed-off-by: Kai Huang * chore: spotlessApply on RestUnifiedQueryAction (javadoc reflow + signature wrap) Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../plugin/rest/RestUnifiedQueryAction.java | 25 +++++++++++++++++-- .../transport/TransportPPLQueryAction.java | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 62f3dd0c346..5debf4702d4 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -152,7 +152,7 @@ public void execute( QueryType queryType, boolean profiling, ActionListener listener) { - doExecute(query, queryType, profiling, null, listener); + doExecute(query, queryType, profiling, 0, null, listener); } /** Execute linked to {@code parentTask} so a front-end cancel propagates into the engine. */ @@ -160,16 +160,18 @@ public void execute( String query, QueryType queryType, boolean profiling, + int fetchSize, Task parentTask, ActionListener listener) { assert parentTask != null : "parentTask required for cancellation propagation"; - doExecute(query, queryType, profiling, parentTask, listener); + doExecute(query, queryType, profiling, fetchSize, parentTask, listener); } private void doExecute( String query, QueryType queryType, boolean profiling, + int fetchSize, Task parentTask, ActionListener listener) { client @@ -193,6 +195,10 @@ private void doExecute( UnifiedQueryPlanner planner = new UnifiedQueryPlanner(context); RelNode plan = planner.plan(query); CalcitePlanContext planContext = context.getPlanContext(); + // PPL fetch_size caps the response to N rows (no cursor) — the V2 path attaches + // a `head N` in AstStatementBuilder; the unified path parses only the query + // string, so apply the equivalent top-level limit here before the system cap. + plan = addFetchSizeLimit(plan, planContext, fetchSize); plan = addQuerySizeLimit(plan, planContext); if (profiling) { analyticsEngine.executeWithProfile( @@ -339,6 +345,21 @@ private static RelNode addQuerySizeLimit(RelNode plan, CalcitePlanContext contex context.relBuilder.literal(context.sysLimit.querySizeLimit())); } + /** + * Cap the result to {@code fetchSize} rows when {@code fetchSize > 0}, mirroring PPL's {@code + * fetch_size} (which the V2 path lowers to a top-level {@code head N} in AstStatementBuilder). + * {@code fetchSize <= 0} means "use system default", so no limit is added. Uses the same {@code + * relBuilder.limit} primitive that {@code head} lowers to, so the analytics backend sees an + * ordinary fetch limit. + */ + private static RelNode addFetchSizeLimit( + RelNode plan, CalcitePlanContext context, int fetchSize) { + if (fetchSize <= 0) { + return plan; + } + return context.relBuilder.push(plan).limit(0, fetchSize).build(); + } + private ResponseListener createQueryListener( QueryType queryType, ActionListener transportListener) { ResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 33dd8235443..171ac0a57e7 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -189,6 +189,7 @@ protected void doExecute( transformedRequest.getRequest(), QueryType.PPL, transformedRequest.profile(), + transformedRequest.getFetchSize(), task, clearingListener); } From d055163dc56b715ebe58dde36b9992c084ec3786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tr=E1=BA=A7n=20Quang=20D=E1=BB=B1?= Date: Fri, 19 Jun 2026 21:31:55 +0300 Subject: [PATCH 06/41] [BugFix] Return all columns (struct and nested fields) listed when using head (#5518) Signed-off-by: Du Tran --- .../sql/calcite/CalciteRelNodeVisitor.java | 5 +++-- docs/user/ppl/cmd/fields.md | 14 +++++++------- .../sql/calcite/remote/CalciteEvalCommandIT.java | 12 +++++++++++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index b07f308f91f..c4bb8bcd910 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -516,8 +516,9 @@ private RelNode handleAllFieldsProject(Project node, CalcitePlanContext context) "Invalid field exclusion: operation would exclude all fields from the result set"); } AllFields allFields = (AllFields) node.getProjectList().getFirst(); - if (!(allFields instanceof AllFieldsExcludeMeta)) { - // Should not remove nested fields for AllFieldsExcludeMeta. + if (!(allFields instanceof AllFieldsExcludeMeta) && !context.isProjectVisited()) { + // Should not remove nested fields for AllFieldsExcludeMeta + // and when no explicit project has already curated the schema tryToRemoveNestedFields(context); } tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta); diff --git a/docs/user/ppl/cmd/fields.md b/docs/user/ppl/cmd/fields.md index b4bf110f7f2..1eb7694a19e 100644 --- a/docs/user/ppl/cmd/fields.md +++ b/docs/user/ppl/cmd/fields.md @@ -198,8 +198,8 @@ fetched rows / total rows = 3/3 ## Example 8: Selecting all fields -The following query selects all fields defined in the index schema using `` `*` ``. Fields with null values are included in the result set: - +The following query selects all fields defined in the index schema using `` `*` ``. Struct fields are returned alongside their flattened sub-fields, and fields with null values are included in the result set: + ```ppl source=otellogs | where severityText = 'WARN' @@ -211,11 +211,11 @@ The query returns the following results: ```text fetched rows / total rows = 1/1 -+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ -| spanId | traceId | @timestamp | instrumentationScope | severityText | resource | flags | attributes | droppedAttributesCount | severityNumber | time | body | -|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------| -| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | WARN | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | 0 | {} | 0 | 13 | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms | -+----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ ++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ +| spanId | traceId | @timestamp | instrumentationScope | instrumentationScope.droppedAttributesCount | instrumentationScope.name | instrumentationScope.version | severityText | resource | resource.attributes | resource.attributes.host | resource.attributes.host.name | resource.attributes.service | resource.attributes.service.name | resource.droppedAttributesCount | flags | attributes | droppedAttributesCount | severityNumber | time | body | +|----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------| +| span0003 | abcd1234efgh5678 | 2024-02-01 09:12:00 | {'name': 'go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc', 'droppedAttributesCount': 0, 'version': '0.49.0'} | 0 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc | 0.49.0 | WARN | {'attributes': {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}}, 'droppedAttributesCount': 0} | {'service': {'name': 'product-catalog'}, 'host': {'name': 'productcatalog-7c9d-zn4p2'}} | {'name': 'productcatalog-7c9d-zn4p2'} | productcatalog-7c9d-zn4p2 | {'name': 'product-catalog'} | product-catalog | 0 | 0 | {} | 0 | 13 | 2024-02-01 09:12:00 | Slow query detected: SELECT * FROM products WHERE category = 'electronics' took 3200ms | ++----------+------------------+---------------------+-------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+-----------------------------------------------------------------------------+------------------------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------+-----------------------------------------------------------------------------------------+---------------------------------------+-------------------------------+-----------------------------+----------------------------------+---------------------------------+-------+------------+------------------------+----------------+---------------------+----------------------------------------------------------------------------------------+ ``` diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java index 87bd412907d..85413cb5d83 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TELEMETRY; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -29,6 +30,7 @@ public void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.TELEMETRY); + loadIndex(Index.DEEP_NESTED); // Pre-create test_eval through the helper so the analytics-engine compatibility run // (tests.analytics.parquet_indices=true) provisions it as a parquet-backed composite @@ -160,7 +162,7 @@ public void testEvalDottedNamePreservesStructParent_ImplicitProject() throws IOE // longer dropped by `shouldOverrideField`'s prefix branch. JSONObject result = executeQuery("source=test_eval_agent | fields agent | eval `agent.name` = 'test'"); - verifySchema(result, schema("agent", "struct")); + verifySchema(result, schema("agent", "struct"), schema("agent.name", "string")); } @Test @@ -233,4 +235,12 @@ public void testEvalStringConcatenationWithExistingData() throws IOException { rows("Hattie", "Bond", "Hattie Bond"), rows("Nanette", "Bates", "Nanette Bates")); } + + @Test + public void testStruckFieldAndSubFieldWithHead() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | fields city.name, city | head", TEST_INDEX_DEEP_NESTED)); + verifySchema(result, schema("city.name", "string"), schema("city", "struct")); + } } From d249a47eacd9ec755f004f25abcc31c41583a55d Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 19 Jun 2026 13:39:35 -0700 Subject: [PATCH 07/41] [Enhancement] Classify unsupported-feature errors as client errors (4xx) on the SQL path (#5569) Signed-off-by: Jialiang Liang --- .../org/opensearch/sql/api/UnifiedQueryPlanner.java | 5 +++++ .../opensearch/sql/api/UnifiedQueryPlannerTest.java | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java index a84300e65f8..9440833503f 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java @@ -27,6 +27,7 @@ import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.QueryEngineException; import org.opensearch.sql.exception.SemanticCheckException; @@ -73,6 +74,10 @@ public RelNode plan(String query) { } return plan; }); + } catch (CalciteUnsupportedException e) { + // Unsupported feature (e.g. table functions) is an invalid query, i.e. a client error. + // Must precede the QueryEngineException branch as it is a subclass. + throw new SemanticCheckException(e.getMessage(), e); } catch (SyntaxCheckException | QueryEngineException | UnsupportedOperationException diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java index 296e9eb2519..bb2d1e4a53f 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -16,6 +16,7 @@ import org.junit.Test; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.executor.QueryType; @@ -147,6 +148,17 @@ public void invalidTableIsRethrownAsSemanticCheckException() { .assertCauseType(CalciteException.class); } + @Test + public void unsupportedFeatureIsRethrownAsSemanticCheckException() { + // A feature unsupported on the analytics engine (here a PPL command that raises + // CalciteUnsupportedException; SQL table functions like vectorSearch() take the same path) is + // an invalid query, normalized to a SemanticCheckException so callers classify it as a 4xx. + givenInvalidQuery("source = catalog.employees | kmeans") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteUnsupportedException.class) + .assertErrorMessageContains("unsupported in Calcite"); + } + @Test public void assertionErrorIsWrappedAsSemanticCheckException() { // Remove when the underlying Calcite assertion is fixed. From 7ed954a17c22774cc55ff88f419e9bba22d25bfc Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 19 Jun 2026 13:49:53 -0700 Subject: [PATCH 08/41] Merge analytics-engine profile into SQL-layer profile (#5571) Signed-off-by: Chen Dai --- .../analytics/AnalyticsExecutionEngine.java | 4 + .../profile/DefaultProfileContext.java | 9 ++- .../sql/monitor/profile/ProfileContext.java | 3 + .../sql/monitor/profile/QueryProfile.java | 5 +- .../analytics/AnalyticsEngineProfileIT.java | 34 +++++--- .../plugin/rest/RestUnifiedQueryAction.java | 80 +++++++++++-------- 6 files changed, 88 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index c4fd2541afb..18d87fc18d0 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -169,6 +169,9 @@ public void executeWithProfile( org.opensearch.analytics.QueryRequestContext queryCtx, ResponseListener listener) { + ProfileContext profileCtx = QueryProfiling.current(); + long execStart = System.nanoTime(); + planExecutor.executeWithProfile( plan, queryCtx, @@ -178,6 +181,7 @@ public void onResponse(ProfiledResult result) { try { // ProfiledResult delivers the profile on BOTH success and failure paths // so users get stage timing visibility even when a query partially fails. + profileCtx.getOrCreateMetric(MetricName.EXECUTE).set(System.nanoTime() - execStart); QueryResponse response = buildProfiledResponse(plan, result); listener.onResponse(response); } catch (Exception e) { diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java index acc5b8521be..63327c2d6dd 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java @@ -17,6 +17,7 @@ public class DefaultProfileContext implements ProfileContext { private boolean finished; private final Map metrics = new ConcurrentHashMap<>(); private ProfilePlanNode planRoot; + private Object enginePlan; private QueryProfile profile; public DefaultProfileContext() {} @@ -40,6 +41,11 @@ public synchronized void setPlanRoot(ProfilePlanNode planRoot) { } } + @Override + public synchronized void setEnginePlan(Object enginePlan) { + this.enginePlan = enginePlan; + } + /** {@inheritDoc} */ @Override public synchronized QueryProfile finish() { @@ -55,7 +61,8 @@ public synchronized QueryProfile finish() { snapshot.put(metricName, millis); } double totalMillis = ProfileUtils.roundToMillis(endNanos - startNanos); - QueryProfile.PlanNode planSnapshot = planRoot == null ? null : planRoot.snapshot(); + Object planSnapshot = + enginePlan != null ? enginePlan : (planRoot == null ? null : planRoot.snapshot()); profile = new QueryProfile(totalMillis, snapshot, planSnapshot); return profile; } diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java index 0c8048cef3f..be17f162de7 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/ProfileContext.java @@ -27,6 +27,9 @@ public interface ProfileContext { */ void setPlanRoot(ProfilePlanNode planRoot); + /** TODO: merge with planRoot into one generic execution-engine-specific plan profile field. */ + default void setEnginePlan(Object plan) {} + /** * Finalize profiling and return a snapshot. * diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java index 15177ead1ca..d9d2a785868 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java @@ -21,7 +21,8 @@ public final class QueryProfile { private final Map phases; - private final PlanNode plan; + /** Execution-engine-specific plan profile: a {@link PlanNode} tree, or a pre-rendered object. */ + private final Object plan; /** * Create a new query profile snapshot. @@ -40,7 +41,7 @@ public QueryProfile(double totalTimeMillis, Map phases) { * @param phases metric values keyed by {@link MetricName} * @param plan plan tree profiling output */ - public QueryProfile(double totalTimeMillis, Map phases, PlanNode plan) { + public QueryProfile(double totalTimeMillis, Map phases, Object plan) { this.summary = new Summary(totalTimeMillis); this.phases = buildPhases(phases); this.plan = plan; diff --git a/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java index 4a28115897a..1899213f960 100644 --- a/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java @@ -103,12 +103,21 @@ public void testPplProfileReturnsStages() throws IOException { assertTrue("has profile", result.has("profile")); JSONObject profile = result.getJSONObject("profile"); - assertTrue("has query_id", profile.has("query_id")); - assertTrue("has planning_time_ms", profile.has("planning_time_ms")); - assertTrue("has execution_time_ms", profile.has("execution_time_ms")); - assertTrue("has full_plan", profile.has("full_plan")); - - JSONArray stages = profile.getJSONArray("stages"); + assertTrue("has summary", profile.has("summary")); + assertTrue("summary has total_time_ms", profile.getJSONObject("summary").has("total_time_ms")); + assertTrue("has phases", profile.has("phases")); + JSONObject phases = profile.getJSONObject("phases"); + assertTrue("phase analyze", phases.getJSONObject("analyze").has("time_ms")); + assertTrue("phase execute", phases.getJSONObject("execute").has("time_ms")); + assertTrue("phase format", phases.getJSONObject("format").has("time_ms")); + + JSONObject plan = profile.getJSONObject("plan"); + assertTrue("has query_id", plan.has("query_id")); + assertTrue("has planning_time_ms", plan.has("planning_time_ms")); + assertTrue("has execution_time_ms", plan.has("execution_time_ms")); + assertTrue("has full_plan", plan.has("full_plan")); + + JSONArray stages = plan.getJSONArray("stages"); assertTrue("at least one stage", stages.length() >= 1); JSONObject stage = stages.getJSONObject(0); @@ -129,9 +138,12 @@ public void testSqlProfileReturnsStages() throws IOException { assertTrue("has profile", result.has("profile")); JSONObject profile = result.getJSONObject("profile"); - assertTrue("has query_id", profile.has("query_id")); - assertTrue("has stages", profile.has("stages")); - JSONArray stages = profile.getJSONArray("stages"); + assertTrue("has summary", profile.has("summary")); + assertTrue("has phases", profile.has("phases")); + JSONObject plan = profile.getJSONObject("plan"); + assertTrue("has query_id", plan.has("query_id")); + assertTrue("has stages", plan.has("stages")); + JSONArray stages = plan.getJSONArray("stages"); assertTrue("at least one stage", stages.length() >= 1); } @@ -142,7 +154,7 @@ public void testPplProfileStagesShowSucceeded() throws IOException { executeWithProfile("source = " + INDEX + " | fields name, score", "/_plugins/_ppl"); JSONObject profile = result.getJSONObject("profile"); - JSONArray stages = profile.getJSONArray("stages"); + JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); for (int i = 0; i < stages.length(); i++) { JSONObject stage = stages.getJSONObject(i); @@ -158,7 +170,7 @@ public void testPplProfileTasksHaveNodeAndTiming() throws IOException { executeWithProfile("source = " + INDEX + " | fields name", "/_plugins/_ppl"); JSONObject profile = result.getJSONObject("profile"); - JSONArray stages = profile.getJSONArray("stages"); + JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); boolean foundTasks = false; for (int i = 0; i < stages.length(); i++) { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 5debf4702d4..84ee147d34a 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -10,6 +10,9 @@ import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.util.Map; import java.util.Optional; import org.apache.calcite.rel.RelNode; @@ -22,7 +25,10 @@ import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.indices.IndicesService; import org.opensearch.sql.api.UnifiedQueryContext; @@ -38,6 +44,8 @@ import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.executor.analytics.AnalyticsExecutionEngine; import org.opensearch.sql.lang.LangSpec; +import org.opensearch.sql.monitor.profile.ProfileContext; +import org.opensearch.sql.monitor.profile.QueryProfiling; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.ResponseFormatter; @@ -185,10 +193,9 @@ private void doExecute( // Carry the front-end task so cancellation propagates into the engine. QueryRequestContext queryCtx = withParentTask(contextProvider.getContext(), parentTask); - // Disable SQL-layer phase profiling when analytics engine profiling is active. - // Our QueryProfile (stages, tasks, timing) is strictly more detailed and replaces - // it. - UnifiedQueryContext context = buildContext(queryType, false, queryCtx); + + UnifiedQueryContext context = buildContext(queryType, profiling, queryCtx); + ProfileContext profileCtx = QueryProfiling.current(); ActionListener closingListener = wrapWithContextClose(context, listener); try { @@ -205,13 +212,13 @@ private void doExecute( plan, planContext, queryCtx, - createQueryListener(queryType, closingListener)); + createQueryListener(queryType, profileCtx, closingListener)); } else { analyticsEngine.execute( plan, planContext, queryCtx, - createQueryListener(queryType, closingListener)); + createQueryListener(queryType, profileCtx, closingListener)); } } catch (Exception e) { closingListener.onFailure(e); @@ -361,19 +368,32 @@ private static RelNode addFetchSizeLimit( } private ResponseListener createQueryListener( - QueryType queryType, ActionListener transportListener) { + QueryType queryType, + ProfileContext profileCtx, + ActionListener transportListener) { ResponseFormatter formatter = new SimpleJsonResponseFormatter(PRETTY); return new ResponseListener() { @Override public void onResponse(QueryResponse response) { LangSpec langSpec = queryType == QueryType.PPL ? PPL_SPEC : LangSpec.SQL_SPEC; - String result = - formatter.format( - new QueryResult( - response.getSchema(), response.getResults(), response.getCursor(), langSpec)); + + // Set the engine profile as the plan so the formatter serializes it in one pass. if (response.getProfile() != null) { - // Append profile and error (if any) to the JSON response - result = appendProfileToJson(result, response.getProfile(), response.getError()); + profileCtx.setEnginePlan(toJsonElement(response.getProfile())); + } + + String result = + QueryProfiling.withCurrentContext( + profileCtx, + () -> + formatter.format( + new QueryResult( + response.getSchema(), + response.getResults(), + response.getCursor(), + langSpec))); + if (response.getError() != null) { + result = appendError(result, response.getError()); } transportListener.onResponse(new TransportPPLQueryResponse(result)); } @@ -385,30 +405,24 @@ public void onFailure(Exception e) { }; } - private static String appendProfileToJson(String json, QueryProfile profile, Throwable error) { + private static JsonElement toJsonElement(QueryProfile profile) { try { - StringBuilder extra = new StringBuilder(); - // Append profile - org.opensearch.core.xcontent.XContentBuilder builder = - org.opensearch.common.xcontent.XContentFactory.jsonBuilder(); - profile.toXContent(builder, org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS); - extra.append(",\"profile\":").append(builder.toString()); - // Append error if query partially failed - if (error != null) { - extra - .append(",\"error\":{\"type\":\"") - .append(error.getClass().getSimpleName()) - .append("\",\"reason\":\"") - .append(error.getMessage() != null ? error.getMessage().replace("\"", "\\\"") : "") - .append("\"}"); - } - if (json.endsWith("}")) { - return json.substring(0, json.length() - 1) + extra + "}"; - } - return json; + XContentBuilder builder = XContentFactory.jsonBuilder(); + profile.toXContent(builder, ToXContent.EMPTY_PARAMS); + return JsonParser.parseString(builder.toString()); } catch (Exception e) { + return null; + } + } + + private static String appendError(String json, Throwable error) { + if (!json.endsWith("}")) { return json; } + JsonObject err = new JsonObject(); + err.addProperty("type", error.getClass().getSimpleName()); + err.addProperty("reason", error.getMessage() != null ? error.getMessage() : ""); + return json.substring(0, json.length() - 1) + ",\"error\":" + err + "}"; } private static Runnable withCurrentContext(final Runnable task) { From 5aa6ea03a9a9a0d059d410c6da8bedf40e1f2c72 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:59:42 -0700 Subject: [PATCH 09/41] Stabilize more PPL ITs on the analytics-engine route (sort/streamstats/IP-UDT/metadata/strip-verifier) (#5566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings 16 more PPL IT classes to parity on the analytics-engine route (-Dtests.analytics.parquet_indices=true). Route-only divergences are gated AE-only via @RequiresCapability + a matching build.gradle excludeTestsMatching; the v2/Calcite path runs every test unchanged. Strip-verifier (the #5541 guardrail): - AnalyticsUnsupportedFieldStripVerifyIT was failing because 8 datasets carry a multi-value JSON array for a scalar-mapped field, which the parquet store rejects at bulk load. That's a cardinality limitation, not an unsupported field *type*, so it's out of scope for the type strip — the same situation as the existing `join` out-of-scope skip. Added a curated MULTI_VALUE_DATASETS allowlist + safeToSkipForMultiValueLoad that skips only the exact multi-value signature on a known dataset; any other failure still surfaces loudly, and Legs 2-3 still type-check every index that loads. Init-load contamination (not divergences — fixed, not gated): - CalciteWhereCommandIT failed on testDoubleEqual* because init() loaded game_of_thrones (base) and deep_nested (subclass), both multi-value datasets whose bulk-load failure aborted init() and mislabeled the first test. Guarded both loads with isAnalyticsParquetIndicesEnabled(); no test in the hierarchy queries them on the AE route. 32/32 now pass. Non-deterministic sort ties stabilized (not gated — full coverage kept): - The three CalcitePPLSortIT tie tests (testSortWithNullValue, testSortAgeAndFieldsNameAge, testSortWithAutoCast) sorted on a non-unique key, so the tied rows had no defined order and the AE route ordered them differently than the captured Lucene doc order. Added a unique secondary sort key (firstname) to each, making the order deterministic and engine-independent. Verified identical on BOTH routes: 18/18 on AE and 18/18 on v2/Calcite. No gate needed. Engine divergences gated (new capabilities): - INVALID_DATETIME_ERROR_SHAPE: dayname/monthname over an invalid literal throw a different message shape (2 tests) - RAND_SEED_UNSUPPORTED: seeded RAND(seed) is rejected on AE - IP_UDT_BINARY_REPRESENTATION: the IP UDT is materialized as BINARY, so cast(... as IP) and cidrmatch over an IP column fail (2 tests) - TIME_TYPE_WIDENED_TO_TIMESTAMP: a TIME field reads back as TIMESTAMP, defeating TIMEDIFF's [TIME,TIME] signature - BINARY_FIELD_STRIPPED: binary fields are stripped at load - VALUES_LIMIT_NOT_HONORED: values()/list() ignore the configured limit - INDEX_METADATA: _index metadata not exposed (sibling of ID_METADATA) - CROSS_INDEX_OBJECT_LEAF_MERGE: an object leaf in only some wildcard member indices resolves to FIELD_NOT_FOUND - TEXT_KEYWORD_PUSHDOWN_REWRITE: like() doesn't rewrite to .keyword in the explain plan (no Lucene term-pushdown) - LUCENE_PUSHDOWN_EXPLAIN: a test asserting a Lucene SORT-> pushdown fragment can't match the DataFusion plan Reused existing capabilities: - WILDCARD_COLUMN_ORDER: streamstats carries all source columns through; AE returns them in a different column order (values and row order are correct, so a sort can't fix it) (4 CalciteReverseCommandIT tests) - HEAD_WITHOUT_STABLE_SORT: the non-determinism is which rows head N keeps, before the trailing sort, so a sort can't recover it (testHeadThenSort, testAppendWithMergedColumn) - DEDUP_NONDETERMINISTIC: consecutive dedup has no working V2 fallback on the AE route Out of scope: - FieldsCommandIT.testEnhancedFieldsWhenCalciteDisabled asserts the Calcite-DISABLED error; the AE route is always Calcite-enabled. build.gradle exclude only. Results (this batch, on the AE route): 16 classes, 0 failures (was 24 failures), with the 3 sort tests now passing rather than skipped. V2 baseline: 0 failures, only pre-existing/by-design skips (none from these gates). Signed-off-by: Kai Huang --- integ-test/build.gradle | 43 +++++++ ...nalyticsUnsupportedFieldStripVerifyIT.java | 49 ++++++++ .../calcite/remote/CalciteDedupCommandIT.java | 8 ++ .../remote/CalciteMultiValueStatsIT.java | 11 ++ .../remote/CalcitePPLAppendCommandIT.java | 13 +++ ...tePPLBuiltinDatetimeFunctionInvalidIT.java | 12 ++ .../CalcitePPLBuiltinFunctionsNullIT.java | 7 ++ .../sql/calcite/remote/CalcitePPLSortIT.java | 20 +++- .../remote/CalciteReverseCommandIT.java | 22 ++++ .../calcite/remote/CalciteSortCommandIT.java | 7 ++ .../calcite/remote/CalciteWhereCommandIT.java | 7 +- .../opensearch/sql/ppl/CastFunctionIT.java | 7 ++ .../opensearch/sql/ppl/FieldsCommandIT.java | 13 +++ .../org/opensearch/sql/ppl/LikeQueryIT.java | 7 ++ .../sql/ppl/MathematicalFunctionIT.java | 5 + .../org/opensearch/sql/ppl/SortCommandIT.java | 7 ++ .../opensearch/sql/ppl/WhereCommandIT.java | 7 +- .../org/opensearch/sql/util/Capability.java | 107 +++++++++++++++++- 18 files changed, 343 insertions(+), 9 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 9f51ba2c222..d4780f90ea9 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1285,6 +1285,49 @@ task integTestRemote(type: RestIntegTestTask) { excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithUseOther' excludeTestsMatching '*CalciteChartCommandIT.testChartLimitBottomWithUseOther' excludeTestsMatching '*CalciteChartCommandIT.testChartLimitTopWithMinAgg' + + // === Excludes: streamstats carries all columns through; AE reorders them === + excludeTestsMatching '*CalciteReverseCommandIT.testStreamstatsWithReverse' + excludeTestsMatching '*CalciteReverseCommandIT.testStreamstatsByWithReverse' + excludeTestsMatching '*CalciteReverseCommandIT.testStreamstatsWindowWithReverse' + excludeTestsMatching '*CalciteReverseCommandIT.testStreamstatsWithSortThenReverse' + + // === Excludes: invalid-datetime error-message shape differs on AE === + excludeTestsMatching '*CalcitePPLBuiltinDatetimeFunctionInvalidIT.testDAYNAMEInvalid' + excludeTestsMatching '*CalcitePPLBuiltinDatetimeFunctionInvalidIT.testMONTHNAMEInvalid' + + // === Excludes: seeded RAND(seed) unsupported on AE === + excludeTestsMatching '*MathematicalFunctionIT.testRand' + + // === Excludes: IP UDT is materialized as BINARY/byte[] on AE === + excludeTestsMatching '*CastFunctionIT.testCastToIP' + excludeTestsMatching '*CalcitePPLAppendCommandIT.testAppendSchemaMergeWithIpUDT' + + // === Excludes: append head-without-stable-sort + TIME-widened-to-TIMESTAMP signature === + excludeTestsMatching '*CalcitePPLAppendCommandIT.testAppendWithMergedColumn' + excludeTestsMatching '*CalcitePPLBuiltinFunctionsNullIT.testTimediffNull' + + // === Excludes: consecutive dedup has no working V2 fallback on AE === + excludeTestsMatching '*CalciteDedupCommandIT.testConsecutiveDedup' + + // === Excludes: binary field stripped at load; values() ignores configured limit === + excludeTestsMatching '*CalciteMultiValueStatsIT.testListFunctionWithBinary' + excludeTestsMatching '*CalciteMultiValueStatsIT.testValuesFunctionRespectsConfiguredLimit' + + // === Excludes: _id/_index metadata not exposed; cross-index object-leaf merge === + excludeTestsMatching '*FieldsCommandIT.testMetadataFields' + excludeTestsMatching '*FieldsCommandIT.testMergedObjectFields' + // OOS: asserts the Calcite-disabled error, but the AE route is always Calcite-enabled. + excludeTestsMatching '*FieldsCommandIT.testEnhancedFieldsWhenCalciteDisabled' + + // === Excludes: head N without a stable sort returns a non-deterministic row set === + excludeTestsMatching '*SortCommandIT.testHeadThenSort' + + // === Excludes: like() doesn't rewrite to .keyword in the explain plan on AE === + excludeTestsMatching '*LikeQueryIT.test_convert_field_text_to_keyword' + + // === Excludes: asserts a Lucene pushdown fragment absent on the AE route === + excludeTestsMatching '*CalciteSortCommandIT.testPushdownSortCastToDoubleExpression' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java index a194b2999e2..ed74a7f932c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java @@ -70,6 +70,32 @@ public class AnalyticsUnsupportedFieldStripVerifyIT extends PPLIntegTestCase { */ private static final Set OUT_OF_SCOPE_TYPES = Set.of("join"); + /** + * Datasets that carry a multi-value JSON array for a scalar-mapped field (e.g. a {@code text} or + * {@code long} field given several values in a doc). The parquet/composite store rejects these at + * bulk load with {@code Cannot accept multiple values for field} — a cardinality limitation, not + * an unsupported field *type*, so it is out of scope for the type strip ({@link + * org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig}) the same way {@link + * #OUT_OF_SCOPE_TYPES} is. Tracked by the {@code MULTI_VALUE_FIELD_LOAD} capability. An index + * here is skipped only when its failure is the exact multi-value signature AND it is on this + * curated list (see {@link #safeToSkipForMultiValueLoad}); any other failure surfaces loudly. + * Legs 2-3 still type-check the live mapping of every index that loads, so a missed type-strip + * can't hide. + */ + private static final Set MULTI_VALUE_DATASETS = + Set.of( + "GAME_OF_THRONES", + "NESTED", + "NESTED_WITH_QUOTES", + "DEEP_NESTED", + "NESTED_WITH_NULLS", + "GRAPH_AIRPORTS", + "ARRAY", + "OTELLOGS"); + + /** Cluster's per-item bulk error when a doc supplies an array to a scalar-mapped field. */ + private static final String MULTI_VALUE_SIGNATURE = "Cannot accept multiple values for field"; + @Override public void init() throws Exception { super.init(); @@ -109,6 +135,12 @@ public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { // unsupported type we're responsible for — not our concern. Skip. continue; } + if (safeToSkipForMultiValueLoad(e, index)) { + // Load failed with the multi-value signature on a known multi-value-array-into-scalar + // dataset (a cardinality limitation, not an unsupported type) — out of scope for the type + // strip, tracked by MULTI_VALUE_FIELD_LOAD. Skip. + continue; + } failures.add( "[" + index.name() @@ -380,6 +412,23 @@ private static boolean safeToSkipForOutOfScopeType(Throwable t, String mapping) return !mappingContainsUnsupportedType(mapping); } + /** + * Safe to skip an index whose load failed, only when BOTH hold: (a) the error is exactly the + * multi-value bulk signature ({@code Cannot accept multiple values for field}), AND (b) the index + * is on the curated {@link #MULTI_VALUE_DATASETS} allowlist. Unlike {@link + * #safeToSkipForOutOfScopeType} this does NOT also require the raw mapping to be free of + * unsupported types: several of these datasets legitimately declare {@code nested} fields that + * the type strip removes at load, while the multi-value failure is on a separate scalar leaf. The + * curated allowlist plus the exact failure signature is the masking guard — an unanticipated + * failure (different message, or a dataset not on the list) is never skipped and surfaces loudly. + */ + private static boolean safeToSkipForMultiValueLoad(Throwable t, Index index) { + String msg = rootMessage(t); + return msg != null + && msg.contains(MULTI_VALUE_SIGNATURE) + && MULTI_VALUE_DATASETS.contains(index.name()); + } + /** True if the raw mapping JSON declares any {@link #UNSUPPORTED} field type at any depth. */ private static boolean mappingContainsUnsupportedType(String mapping) { if (mapping == null || mapping.isEmpty()) { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDedupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDedupCommandIT.java index 6da1268313b..08595da89c5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDedupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteDedupCommandIT.java @@ -5,8 +5,11 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.util.Capability.DEDUP_NONDETERMINISTIC; + import java.io.IOException; import org.opensearch.sql.ppl.DedupCommandIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteDedupCommandIT extends DedupCommandIT { @Override @@ -15,6 +18,11 @@ public void init() throws Exception { enableCalcite(); } + @RequiresCapability( + value = DEDUP_NONDETERMINISTIC, + note = + "consecutive dedup falls back to V2 on the Calcite path, but the AE route has no working" + + " V2 fallback (DEDUP_NONDETERMINISTIC).") @Override public void testConsecutiveDedup() throws IOException { withFallbackEnabled( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java index c374f8bbb29..ced7cf59dd3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java @@ -10,6 +10,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.BINARY_FIELD_STRIPPED; +import static org.opensearch.sql.util.Capability.VALUES_LIMIT_NOT_HONORED; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -21,6 +23,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteMultiValueStatsIT extends PPLIntegTestCase { @@ -169,6 +172,9 @@ public void testListFunctionWithIP() throws IOException { } @Test + @RequiresCapability( + value = BINARY_FIELD_STRIPPED, + note = "binary_value is stripped at load on the AE route (BINARY_FIELD_STRIPPED).") public void testListFunctionWithBinary() throws IOException { JSONObject response = executeQuery( @@ -420,6 +426,11 @@ public void testValuesFunctionWithUnlimitedValues() throws IOException { } @Test + @RequiresCapability( + value = VALUES_LIMIT_NOT_HONORED, + note = + "values() ignores plugins.ppl.values.max.limit on the AE route" + + " (VALUES_LIMIT_NOT_HONORED).") public void testValuesFunctionRespectsConfiguredLimit() throws IOException, InterruptedException { // Test 1: Set limit to 3 and verify only 3 values are returned updateClusterSettings(new ClusterSetting(TRANSIENT, "plugins.ppl.values.max.limit", "3")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java index 6372b818b2b..56eece17ff4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendCommandIT.java @@ -8,6 +8,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.HEAD_WITHOUT_STABLE_SORT; +import static org.opensearch.sql.util.Capability.IP_UDT_BINARY_REPRESENTATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +24,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLAppendCommandIT extends PPLIntegTestCase { @Override @@ -195,6 +198,11 @@ public void testAppendDifferentIndex() throws IOException { } @Test + @RequiresCapability( + value = HEAD_WITHOUT_STABLE_SORT, + note = + "head 5 over the two-branch append has no globally-unique sort key, so the" + + " surviving/ordered rows diverge on the AE route (HEAD_WITHOUT_STABLE_SORT).") public void testAppendWithMergedColumn() throws IOException { JSONObject actual = executeQuery( @@ -258,6 +266,11 @@ public void testAppendSchemaMergeWithTimestampUDT() throws IOException { } @Test + @RequiresCapability( + value = IP_UDT_BINARY_REPRESENTATION, + note = + "cidrmatch over an appended IP column hits the IP-UDT-as-byte[] gap on the AE route" + + " (IP_UDT_BINARY_REPRESENTATION).") public void testAppendSchemaMergeWithIpUDT() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java index 9c94d1e025d..aa096690dde 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinDatetimeFunctionInvalidIT.java @@ -6,12 +6,14 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS_WITH_NULL; +import static org.opensearch.sql.util.Capability.INVALID_DATETIME_ERROR_SHAPE; import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinDatetimeFunctionInvalidIT extends PPLIntegTestCase { @Override @@ -216,6 +218,11 @@ public void testDAYInvalid() { } @Test + @RequiresCapability( + value = INVALID_DATETIME_ERROR_SHAPE, + note = + "dayname/monthname over an invalid datetime literal throws a different error-message" + + " shape on the AE route (INVALID_DATETIME_ERROR_SHAPE).") public void testDAYNAMEInvalid() { Throwable e1 = @@ -760,6 +767,11 @@ public void testMONTH_OF_YEARInvalid() { } @Test + @RequiresCapability( + value = INVALID_DATETIME_ERROR_SHAPE, + note = + "dayname/monthname over an invalid datetime literal throws a different error-message" + + " shape on the AE route (INVALID_DATETIME_ERROR_SHAPE).") public void testMONTHNAMEInvalid() { Throwable e1 = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java index c7651014ea6..4bb83582ca4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionsNullIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; +import static org.opensearch.sql.util.Capability.TIME_TYPE_WIDENED_TO_TIMESTAMP; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalcitePPLBuiltinFunctionsNullIT extends PPLIntegTestCase { @Override @@ -887,6 +889,11 @@ public void testTimeToSecNull() throws IOException { } @Test + @RequiresCapability( + value = TIME_TYPE_WIDENED_TO_TIMESTAMP, + note = + "the TIME field reads back as TIMESTAMP on the AE route, defeating TIMEDIFF's [TIME,TIME]" + + " signature (TIME_TYPE_WIDENED_TO_TIMESTAMP).") public void testTimediffNull() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java index 1b090580d94..2641fe6b73c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java @@ -166,15 +166,18 @@ public void testSortAgeAndFieldsAge() throws IOException { @Test public void testSortAgeAndFieldsNameAge() throws IOException { + // firstname is a unique secondary key so the age=36 tie has a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject actual = executeQuery( - String.format("source=%s | sort - age | fields firstname, age", TEST_INDEX_BANK)); + String.format( + "source=%s | sort - age, firstname | fields firstname, age", TEST_INDEX_BANK)); verifySchema(actual, schema("firstname", "string"), schema("age", "int")); verifyDataRowsInOrder( actual, rows("Virginia", 39), - rows("Hattie", 36), rows("Elinor", 36), + rows("Hattie", 36), rows("Dillard", 34), rows("Dale", 33), rows("Amber JOHnny", 32), @@ -201,15 +204,17 @@ public void testSortAgeNameAndFieldsNameAge() throws IOException { @Test public void testSortWithNullValue() throws IOException { + // firstname is a unique secondary key so the three null-balance rows have a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject result = executeQuery( String.format( - "source=%s | sort balance | fields firstname, balance", + "source=%s | sort balance, firstname | fields firstname, balance", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifyDataRowsInOrder( result, - rows("Hattie", null), rows("Elinor", null), + rows("Hattie", null), rows("Virginia", null), rows("Dale", 4180), rows("Nanette", 32838), @@ -316,9 +321,12 @@ public void testSortWithStrCast() throws IOException { @Test public void testSortWithAutoCast() throws IOException { + // firstname is a unique secondary key so the age=36 tie has a deterministic, + // engine-independent order (sort is otherwise free to order ties differently per engine). JSONObject result = executeQuery( - String.format("source=%s | sort AUTO(age) | fields firstname, age", TEST_INDEX_BANK)); + String.format( + "source=%s | sort AUTO(age), firstname | fields firstname, age", TEST_INDEX_BANK)); verifySchema(result, schema("firstname", "string"), schema("age", "int")); verifyDataRowsInOrder( result, @@ -326,8 +334,8 @@ public void testSortWithAutoCast() throws IOException { rows("Amber JOHnny", 32), rows("Dale", 33), rows("Dillard", 34), - rows("Hattie", 36), rows("Elinor", 36), + rows("Hattie", 36), rows("Virginia", 39)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java index 5c381bb5346..fb88e9be73b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATA; +import static org.opensearch.sql.util.Capability.WILDCARD_COLUMN_ORDER; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -18,6 +19,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class CalciteReverseCommandIT extends PPLIntegTestCase { @@ -230,6 +232,11 @@ public void testReverseWithTimestampAndExplicitSort() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWithReverse() throws IOException { // Test that reverse is ignored when used directly after streamstats // streamstats maintains order via __stream_seq__, but this field is projected out @@ -259,6 +266,11 @@ public void testStreamstatsWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWindowWithReverse() throws IOException { // Test that reverse is ignored after streamstats with window JSONObject result = @@ -286,6 +298,11 @@ public void testStreamstatsWindowWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsByWithReverse() throws IOException { // Test that reverse is effective after streamstats with partitioning (by clause). // Backtracking finds the __stream_seq__ sort from streamstats and reverses its order. @@ -314,6 +331,11 @@ public void testStreamstatsByWithReverse() throws IOException { } @Test + @RequiresCapability( + value = WILDCARD_COLUMN_ORDER, + note = + "streamstats carries all source columns through; the AE route returns them in a different" + + " order (WILDCARD_COLUMN_ORDER).") public void testStreamstatsWithSortThenReverse() throws IOException { // Test that reverse works when there's an explicit sort after streamstats // The explicit sort creates a collation that reverse can detect and reverse diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java index 3e01ba9f9d7..0c6f0250807 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.Capability.LUCENE_PUSHDOWN_EXPLAIN; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyOrder; @@ -16,6 +17,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.ppl.SortCommandIT; +import org.opensearch.sql.util.RequiresCapability; public class CalciteSortCommandIT extends SortCommandIT { @Override @@ -80,6 +82,11 @@ public void testPushdownSortCastExpression() throws IOException { } @Test + @RequiresCapability( + value = LUCENE_PUSHDOWN_EXPLAIN, + note = + "asserts a Lucene SORT-> pushdown fragment in the explain plan, absent on the AE route" + + " (LUCENE_PUSHDOWN_EXPLAIN).") public void testPushdownSortCastToDoubleExpression() throws IOException { // Similar to query: 'source=%s | sort num(age)'. But left query doesn't output casted column. String ppl = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java index 593e56b467c..225d5f259fb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java @@ -31,7 +31,12 @@ public void init() throws Exception { super.init(); enableCalcite(); loadIndex(Index.NESTED_SIMPLE); - loadIndex(Index.DEEP_NESTED); + // deep_nested has a multi-value array for the scalar-mapped `accounts.id` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init(). The + // only test that queries deep_nested is already excluded on the AE route (nested fields). + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.DEEP_NESTED); + } loadIndex(Index.CASCADED_NESTED); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java index 56e52610aba..2d87fe536fc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/CastFunctionIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY_WITH_NULL; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.IP_UDT_BINARY_REPRESENTATION; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -24,6 +25,7 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.util.RequiresCapability; public class CastFunctionIT extends PPLIntegTestCase { @Override @@ -403,6 +405,11 @@ public void testCastTimestamp() throws IOException { } @Test + @RequiresCapability( + value = IP_UDT_BINARY_REPRESENTATION, + note = + "cast(... as IP) sees the IP UDT as BINARY on the AE route" + + " (IP_UDT_BINARY_REPRESENTATION).") public void testCastToIP() throws IOException { // Test casting IP to IP type JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index 4d755c1ab77..2ab3ce6dad6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -6,6 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.CROSS_INDEX_OBJECT_LEAF_MERGE; +import static org.opensearch.sql.util.Capability.ID_METADATA; +import static org.opensearch.sql.util.Capability.INDEX_METADATA; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.columnPattern; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -22,6 +26,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.util.RequiresCapability; public class FieldsCommandIT extends PPLIntegTestCase { @@ -89,6 +94,9 @@ public void testSpecialDataTypes(String querySource) throws IOException { } @Test + @RequiresCapability( + value = {ID_METADATA, INDEX_METADATA}, + note = "queries _id and _index, which parquet-backed scans don't expose on the AE route.") public void testMetadataFields() throws IOException { // Test basic metadata fields JSONObject basicResult = @@ -128,6 +136,11 @@ public void testMetadataFieldsWithEvalError() { } @Test + @RequiresCapability( + value = {CROSS_INDEX_OBJECT_LEAF_MERGE, NESTED_FIELDS}, + note = + "an object leaf present in only one merge_test_* index is FIELD_NOT_FOUND on the AE" + + " route; also reads a nested array field.") public void testMergedObjectFields() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java index bc98c312fb6..a4410095e4a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.Capability.TEXT_KEYWORD_PUSHDOWN_REWRITE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; @@ -13,6 +14,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class LikeQueryIT extends PPLIntegTestCase { @@ -97,6 +99,11 @@ public void test_like_on_text_field_with_greater_than_one_word() throws IOExcept } @Test + @RequiresCapability( + value = TEXT_KEYWORD_PUSHDOWN_REWRITE, + note = + "like() over a text+keyword field doesn't rewrite to .keyword in the explain plan on the" + + " AE route (TEXT_KEYWORD_PUSHDOWN_REWRITE).") public void test_convert_field_text_to_keyword() throws IOException { String query = "source=" diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index 6cd5063b5a0..6df60f68a7b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.Capability.RAND_SEED_UNSUPPORTED; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -17,6 +18,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.sql.util.RequiresCapability; public class MathematicalFunctionIT extends PPLIntegTestCase { @@ -429,6 +431,9 @@ public void testPi() throws IOException { } @Test + @RequiresCapability( + value = RAND_SEED_UNSUPPORTED, + note = "rand(seed) is rejected on the AE route (RAND_SEED_UNSUPPORTED).") public void testRand() throws IOException { JSONObject result = executeQuery(String.format("source=%s | eval f = rand() | fields f", TEST_INDEX_BANK)); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index a9001f5c995..76275b3ed72 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; +import static org.opensearch.sql.util.Capability.HEAD_WITHOUT_STABLE_SORT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyOrder; @@ -23,6 +24,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class SortCommandIT extends PPLIntegTestCase { @@ -313,6 +315,11 @@ public void testSortAllDefaultFields() throws IOException { } @Test + @RequiresCapability( + value = HEAD_WITHOUT_STABLE_SORT, + note = + "head 2 without a stable sort returns a non-deterministic row set on the AE route" + + " (HEAD_WITHOUT_STABLE_SORT).") public void testHeadThenSort() throws IOException { JSONObject result = executeQuery(String.format("source=%s | head 2 | sort age | fields age", TEST_INDEX_BANK)); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java index dbb54505453..65b9e26a851 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java @@ -31,7 +31,12 @@ public void init() throws Exception { super.init(); loadIndex(Index.ACCOUNT); loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.GAME_OF_THRONES); + // game_of_thrones has a multi-value array for the scalar-mapped `titles` field, which the + // parquet store rejects at bulk load; skip it on the AE route so it doesn't abort init() for + // the rest of the suite. No test in this class hierarchy queries game_of_thrones. + if (!isAnalyticsParquetIndicesEnabled()) { + loadIndex(Index.GAME_OF_THRONES); + } loadIndex(Index.DATETIME); } diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index b7e95bfc4f1..1ca2b9adee7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -364,7 +364,112 @@ public enum Capability { */ UNIX_TIMESTAMP_SUBSECOND( "unix_timestamp() drops sub-second precision on the analytics-engine route (returns whole" - + " seconds), whereas the v2/Calcite path preserves the fractional seconds."); + + " seconds), whereas the v2/Calcite path preserves the fractional seconds."), + + /** + * The {@code _index} (and {@code _id}) metadata field is not exposed on the analytics-engine + * route — parquet-backed scans surface only mapped document fields, so a query referencing {@code + * _index} fails with {@code FIELD_NOT_FOUND}. Sibling of {@link #ID_METADATA}. + */ + INDEX_METADATA( + "The analytics-engine route doesn't expose the _index metadata field (parquet-backed scans" + + " surface only mapped document fields)."), + + /** + * An {@code object} leaf sub-field present in only some member indices of a wildcard/alias source + * resolves to {@code FIELD_NOT_FOUND} on the analytics-engine route (e.g. {@code machine.os2} + * mapped only in one of two {@code merge_test_*} indices), whereas the v2/Calcite path merges the + * schemas and returns the leaf with nulls for the indices that lack it. + */ + CROSS_INDEX_OBJECT_LEAF_MERGE( + "An object leaf field present in only some wildcard member indices resolves to" + + " FIELD_NOT_FOUND on the analytics-engine route, whereas the v2/Calcite path merges the" + + " schemas."), + + /** + * {@code dayname}/{@code monthname} (and similar) over an invalid datetime literal throw a + * different error-message shape on the analytics-engine route ({@code timestamp:... yyyy-MM-dd + * HH:mm:ss[.SSSSSSSSS]}) than the v2/Calcite path ({@code date:... yyyy-MM-dd}): the route parses + * the literal through the TIMESTAMP path, so a different parser produces the message. Both + * engines correctly reject the input; only the message text diverges. + */ + INVALID_DATETIME_ERROR_SHAPE( + "An invalid datetime literal throws a different error-message shape on the analytics-engine" + + " route (timestamp/yyyy-MM-dd HH:mm:ss[...]) than the v2/Calcite path" + + " (date/yyyy-MM-dd); both engines reject the input, only the message differs."), + + /** + * Seeded {@code RAND(seed)} is unsupported on the analytics-engine route (rejected with {@code + * Seeded RAND(seed) is not supported on the analytics-engine route}); DataFusion has no + * deterministic seeded RAND equivalent. {@code RAND()} without a seed works. + */ + RAND_SEED_UNSUPPORTED( + "Seeded RAND(seed) is unsupported on the analytics-engine route (DataFusion has no" + + " deterministic seeded RAND); RAND() without a seed works."), + + /** + * The IP user-defined type is materialized as a raw {@code BINARY}/{@code byte[]} column on the + * analytics-engine route, so operations that need its IP type — {@code cast(... as IP)}, {@code + * cidrmatch} over an appended/merged IP column — fail ({@code Cannot convert BINARY to IP} / + * {@code unsupported object class [B}). The v2/Calcite path keeps the column typed IP. + */ + IP_UDT_BINARY_REPRESENTATION( + "The IP user-defined type is materialized as a raw BINARY/byte[] column on the" + + " analytics-engine route, so cast(... as IP) and cidrmatch over an IP column fail; the" + + " v2/Calcite path keeps the column typed IP."), + + /** + * A {@code TIME}-typed field is presented as {@code TIMESTAMP} on the analytics-engine route, so + * a function with a {@code TIME}-only signature (e.g. {@code TIMEDIFF} expects {@code [TIME, + * TIME]}) rejects it with a type error ({@code expects {[TIME,TIME]}, but got + * [TIMESTAMP,TIMESTAMP]}). The v2/Calcite path preserves the {@code TIME} type and the function + * accepts it. + */ + TIME_TYPE_WIDENED_TO_TIMESTAMP( + "A TIME-typed field is presented as TIMESTAMP on the analytics-engine route, so functions" + + " with a TIME-only signature (e.g. TIMEDIFF) reject it with a type error; the" + + " v2/Calcite path preserves the TIME type."), + + /** + * {@code binary}-typed fields are stripped from the dataset at load on the analytics-engine route + * (the parquet/composite store can't hold them), so a query referencing a binary field resolves + * to {@code FIELD_NOT_FOUND}. + */ + BINARY_FIELD_STRIPPED( + "binary-typed fields are stripped from the dataset at load on the analytics-engine route, so" + + " queries referencing a binary field resolve to FIELD_NOT_FOUND."), + + /** + * The {@code plugins.ppl.values.max.limit} cap on {@code values()}/{@code list()} is not honored + * on the analytics-engine route: {@code PplAggregateCallRewriter} emits no sort/limit for these + * aggregates, so all unique values are returned regardless of the configured limit. + */ + VALUES_LIMIT_NOT_HONORED( + "The plugins.ppl.values.max.limit cap on values()/list() is not honored on the" + + " analytics-engine route (the aggregate rewriter emits no limit), so all unique values" + + " are returned."), + + /** + * {@code like()} over a {@code text}+{@code keyword} field does not rewrite the filter to the + * {@code .keyword} sub-field in the explain plan on the analytics-engine route (the DataFusion + * scan has no Lucene term-pushdown to rewrite to), so a test asserting the plan contains {@code + * .keyword} fails. The v2/Calcite-over-Lucene path performs the pushdown rewrite. + */ + TEXT_KEYWORD_PUSHDOWN_REWRITE( + "like() over a text+keyword field doesn't rewrite to the .keyword sub-field in the explain" + + " plan on the analytics-engine route (no Lucene term-pushdown), whereas the v2/Calcite" + + " path does."), + + /** + * A test that asserts a Lucene-specific pushdown fragment in the explain plan (e.g. the {@code + * SORT->[...]} sort-pushdown JSON) can't pass on the analytics-engine route: the DataFusion scan + * produces a different plan shape with no Lucene pushdown fragment. The query results are + * correct; only the plan-text assertion diverges. + */ + LUCENE_PUSHDOWN_EXPLAIN( + "A test asserting a Lucene-specific pushdown fragment in the explain plan (e.g. SORT->[...])" + + " can't pass on the analytics-engine route: the DataFusion scan produces a different" + + " plan with no Lucene pushdown fragment."); private final String reason; From f6b6baacc5437944e5c5f1d6f12dd7bee1e2fe4c Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 19 Jun 2026 14:06:04 -0700 Subject: [PATCH 10/41] [Enhancement] Reject unsupported output formats on the analytics-engine route with a 4xx (#5570) Signed-off-by: Jialiang Liang --- .../org/opensearch/sql/plugin/SQLPlugin.java | 8 +++ .../rest/AnalyticsEngineFormatSupport.java | 45 +++++++++++++ .../transport/TransportPPLQueryAction.java | 8 +++ .../AnalyticsEngineFormatSupportTest.java | 64 +++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java create mode 100644 plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index ab8b923e3ae..e1278aa75e8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -107,6 +107,7 @@ import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; +import org.opensearch.sql.plugin.rest.AnalyticsEngineFormatSupport; import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder; import org.opensearch.sql.plugin.rest.RestPPLGrammarAction; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; @@ -271,6 +272,13 @@ public void onFailure(Exception e) { } }); } else { + // Analytics route only emits JSON; reject unsupported formats (e.g. csv) with a 4xx. + try { + AnalyticsEngineFormatSupport.validateFormat(sqlRequest.format()); + } catch (Exception e) { + RestSqlAction.handleException(channel, e); + return true; + } unifiedQueryHandler.execute( sqlRequest.getQuery(), QueryType.SQL, diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java new file mode 100644 index 00000000000..b4ac9dd0678 --- /dev/null +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupport.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import java.util.Locale; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; +import org.opensearch.sql.protocol.response.format.Format; + +/** + * Guards the analytics-engine query route, which only produces JSON. Rejects the alternate output + * formats (csv/raw/viz) the route does not implement, instead of silently answering with JSON. + */ +public final class AnalyticsEngineFormatSupport { + + private AnalyticsEngineFormatSupport() {} + + /** + * Throw an {@link ErrorReport} if the requested output format is unsupported on the analytics + * engine. JSON/JDBC (the default) is supported; csv/raw/viz are not. + */ + public static void validateFormat(Format format) { + // JDBC (the default) is the JSON contract the analytics route emits. + if (format == Format.JDBC) { + return; + } + throw ErrorReport.wrap( + new IllegalArgumentException( + String.format( + Locale.ROOT, + "response in %s format is not supported on the analytics engine", + format.getFormatName()))) + .code(ErrorCode.UNSUPPORTED_OPERATION) + .stage(QueryProcessingStage.ANALYZING) + .location("while selecting the response format for the analytics engine") + .suggestion( + "The analytics engine only returns JSON. Remove the 'format' parameter, or run this" + + " query against a non-analytics index to use the requested format.") + .build(); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 171ac0a57e7..973f00c54cb 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -40,6 +40,7 @@ import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; +import org.opensearch.sql.plugin.rest.AnalyticsEngineFormatSupport; import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder; import org.opensearch.sql.plugin.rest.RestUnifiedQueryAction; import org.opensearch.sql.ppl.PPLService; @@ -185,6 +186,13 @@ protected void doExecute( task, createExplainResponseListener(transformedRequest, clearingListener)); } else { + // Analytics route only emits JSON; reject unsupported formats (e.g. csv) with a 4xx. + try { + AnalyticsEngineFormatSupport.validateFormat(format(transformedRequest)); + } catch (Exception e) { + clearingListener.onFailure(e); + return; + } unifiedQueryHandler.execute( transformedRequest.getRequest(), QueryType.PPL, diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java new file mode 100644 index 00000000000..e7de2267e78 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/AnalyticsEngineFormatSupportTest.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.rest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.protocol.response.format.Format; + +/** + * Verifies {@link AnalyticsEngineFormatSupport#validateFormat(Format)} accepts the JSON/JDBC + * default the analytics route emits and rejects unsupported output formats (csv/raw/viz) with a + * structured {@link ErrorReport} so the REST layer returns a clean 4xx instead of silently + * returning JSON. + */ +public class AnalyticsEngineFormatSupportTest { + + @Test + public void jdbcFormatIsAccepted() { + // No exception expected — JDBC is the JSON contract the analytics route emits. + AnalyticsEngineFormatSupport.validateFormat(Format.JDBC); + } + + @Test + public void csvFormatIsRejectedAsUnsupportedOperation() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.CSV)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + assertTrue(report.getMessage().toLowerCase().contains("csv")); + assertTrue(report.getMessage().contains("analytics engine")); + assertNotNull(report.getSuggestion()); + } + + @Test + public void rawFormatIsRejected() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.RAW)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + assertTrue(report.getMessage().toLowerCase().contains("raw")); + } + + @Test + public void vizFormatIsRejected() { + ErrorReport report = + assertThrows(() -> AnalyticsEngineFormatSupport.validateFormat(Format.VIZ)); + assertEquals(ErrorCode.UNSUPPORTED_OPERATION, report.getCode()); + } + + private static ErrorReport assertThrows(Runnable runnable) { + try { + runnable.run(); + } catch (ErrorReport e) { + return e; + } + throw new AssertionError("Expected ErrorReport to be thrown, but nothing was thrown"); + } +} From b2fd2685e96bf23f3a0ee772f2f43e46c926a711 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 19 Jun 2026 15:29:08 -0700 Subject: [PATCH 11/41] allow partial pushdown for semi-scripted predicates (#5565) Signed-off-by: Simeon Widdis --- .../CalcitePartialFilterPushdownIT.java | 68 +++++++++++++++++++ .../opensearch/request/PredicateAnalyzer.java | 2 +- .../request/PredicateAnalyzerTest.java | 44 ++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java new file mode 100644 index 00000000000..7921ecf6d55 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePartialFilterPushdownIT.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Tests partial filter pushdown when some predicates can be pushed natively and others require + * script evaluation. + * + *

Regression test for issue where LIKE on text fields (without .keyword subfield) caused entire + * AND filter to fall back to script, preventing timestamp range pushdown. + */ +public class CalcitePartialFilterPushdownIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.LOGS); + } + + @Test + public void testTimestampRangePushesWithUnpushableLike() throws IOException { + // message is text field without .keyword — LIKE on it requires script evaluation + // @timestamp is date field — range should push natively despite LIKE failing + String query = + String.format( + "source=%s | where `@timestamp` >= '2023-01-01' and `@timestamp` < '2023-01-04' " + + "and LIKE(message, '%%failed%%') | stats count()", + TEST_INDEX_LOGS); + + JSONObject result = executeQuery(query); + + // Just verify query executes and returns reasonable results + // The key regression is that this doesn't do a full table scan + verifySchema(result, schema("count()", "bigint")); + // Should find "Database connection failed" in the date range + verifyDataRows(result, rows(1L)); + } + + @Test + public void testMultipleUnpushablePredicatesInAnd() throws IOException { + // Both LIKE conditions are on text field, but timestamp should still push + String query = + String.format( + "source=%s | where `@timestamp` >= '2023-01-01' and LIKE(message, '%%space%%') " + + "and LIKE(message, '%%low%%') | stats count()", + TEST_INDEX_LOGS); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("count()", "bigint")); + // Should find "Disk space low" + verifyDataRows(result, rows(1L)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 67a31a7f56b..476e0018fcd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1394,7 +1394,7 @@ public QueryExpression like(LiteralExpression literal, boolean caseSensitive) { .caseInsensitive(!caseSensitive); return this; } - throw new UnsupportedOperationException("Like query is not supported for text field"); + throw new PredicateAnalyzerException("Like query is not supported for text field"); } @Override diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 041063f62a0..01c5e6108ef 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -1649,4 +1649,48 @@ void notIsFalse_generatesOnlyMustNotTerm() throws ExpressionNotAnalyzableExcepti """, result.toString()); } + + @Test + void andWithUnpushableLike_partiallyPushesOtherPredicates() + throws ExpressionNotAnalyzableException { + // field3 (c) is text without .keyword → LIKE throws PredicateAnalyzerException + // field4 (d) is date → timestamp range should push as RangeQueryBuilder + final RelDataType rowType = + builder + .getTypeFactory() + .builder() + .kind(StructKind.FULLY_QUALIFIED) + .add("a", typeFactory.createSqlType(SqlTypeName.INTEGER)) + .add("b", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("c", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("d", typeFactory.createUDT(ExprUDT.EXPR_TIMESTAMP)) + .add("e", typeFactory.createSqlType(SqlTypeName.BOOLEAN)) + .build(); + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + RexInputRef field3 = builder.makeInputRef(typeFactory.createSqlType(SqlTypeName.VARCHAR), 2); + RexNode likeCall = + builder.makeCall( + SqlStdOperatorTable.LIKE, field3, stringLiteral, builder.makeLiteral("\\")); + RexNode rangeCall = + builder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, field4, dateTimeLiteral); + RexNode andCall = builder.makeCall(SqlStdOperatorTable.AND, rangeCall, likeCall); + + QueryBuilder result = PredicateAnalyzer.analyze(andCall, schema, fieldTypes, rowType, cluster); + + // Should be a BoolQueryBuilder with range in must[] and LIKE as script + assertInstanceOf(BoolQueryBuilder.class, result); + BoolQueryBuilder boolQuery = (BoolQueryBuilder) result; + assertEquals(2, boolQuery.must().size()); + + // First must clause should be the range query (pushable) + QueryBuilder firstMust = boolQuery.must().get(0); + assertInstanceOf(RangeQueryBuilder.class, firstMust); + RangeQueryBuilder rangeQuery = (RangeQueryBuilder) firstMust; + assertEquals("d", rangeQuery.fieldName()); + + // Second must clause should be script query (unpushable LIKE) + QueryBuilder secondMust = boolQuery.must().get(1); + assertInstanceOf(ScriptQueryBuilder.class, secondMust); + } } From 4c4166bbbe5de7d38d6053cc79dec9cb320db6fa Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Sat, 20 Jun 2026 00:02:01 -0700 Subject: [PATCH 12/41] [Error Enhancement] Fix NPE when rex sits inside appendcol subsearch for Analytic Engine (#5574) Signed-off-by: Jialiang Liang --- .../main/java/org/opensearch/sql/ast/tree/Rex.java | 2 +- .../sql/calcite/remote/CalcitePPLAppendcolIT.java | 14 ++++++++++++++ .../sql/ppl/calcite/CalcitePPLRexTest.java | 10 ++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java b/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java index c3b1d975ac1..4b0e2188617 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Rex.java @@ -79,7 +79,7 @@ public Rex attach(UnresolvedPlan child) { @Override public List getChild() { - return ImmutableList.of(child); + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); } @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java index 877c10947b8..c6fc62fae74 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAppendcolIT.java @@ -81,4 +81,18 @@ public void testAppendColOverride() throws IOException { rows("F", "DE", 101, null), rows("F", "FL", 310, null)); } + + /** Verifies that rex can be used as the first command of an appendcol subsearch. */ + @Test + public void testAppendColWithRexInSubsearch() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats count() as cnt by gender" + + " | appendcol [ rex field=email '^(?[^@]+)@.*' | fields user ]" + + " | head 2", + TEST_INDEX_ACCOUNT)); + verifySchema( + actual, schema("gender", "string"), schema("cnt", "bigint"), schema("user", "string")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java index 619cb26b64a..2275f9352de 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRexTest.java @@ -306,4 +306,14 @@ public void testRexWithMaxMatchAndOffsetField() { + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + /** Verifies that rex plans correctly when it appears as the first command of a subsearch. */ + @Test + public void testRexInsideSubsearch() { + String ppl = + "source=EMP | stats count() as base_c by JOB" + + " | appendcol [ rex field=ENAME '^(?[A-Z])' | fields ENAME, first ]"; + RelNode root = getRelNode(ppl); + org.junit.Assert.assertNotNull(root); + } } From 5392d62931df5cd5472b920291abca5d70d304ab Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 22 Jun 2026 09:03:29 -0700 Subject: [PATCH 13/41] [Error Enhancement] Fix NPE on case() with incompatible branch types (#5575) Signed-off-by: Jialiang Liang --- .../sql/calcite/CalciteRexNodeVisitor.java | 14 +++++++++++++- .../remote/CalcitePPLCaseFunctionIT.java | 17 +++++++++++++++++ .../ppl/calcite/CalcitePPLCaseFunctionTest.java | 16 ++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 44c6d87da12..3c37a11ba5b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -835,6 +835,7 @@ public RexNode visitCast(Cast node, CalcitePlanContext context) { @Override public RexNode visitCase(Case node, CalcitePlanContext context) { List caseOperands = new ArrayList<>(); + List resultTypes = new ArrayList<>(); for (When when : node.getWhenClauses()) { RexNode condition = analyze(when.getCondition(), context); if (!SqlTypeUtil.isBoolean(condition.getType())) { @@ -843,11 +844,22 @@ public RexNode visitCase(Case node, CalcitePlanContext context) { "Condition expected a boolean type, but got %s", condition.getType())); } caseOperands.add(condition); - caseOperands.add(analyze(when.getResult(), context)); + RexNode result = analyze(when.getResult(), context); + caseOperands.add(result); + resultTypes.add(result.getType()); } RexNode elseExpr = node.getElseClause().map(e -> analyze(e, context)).orElse(context.relBuilder.literal(null)); caseOperands.add(elseExpr); + resultTypes.add(elseExpr.getType()); + + // Pre-validate the THEN/ELSE result types so an unsupertyped mix surfaces as a clean + // 400 here instead of an opaque NPE deep in Calcite's makeCall return-type inference. + RelDataType commonType = context.rexBuilder.getTypeFactory().leastRestrictive(resultTypes); + if (commonType == null) { + throw new ExpressionEvaluationException( + StringUtils.format("case branches must have a common type, but got %s", resultTypes)); + } return context.rexBuilder.makeCall(SqlStdOperatorTable.CASE, caseOperands); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index bc9d4388d5b..8bba907e2e2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -537,4 +537,21 @@ public void testNestedCaseAggWithAutoDateHistogram() throws IOException { schema("flags", "bigint")); verifyNumOfRows(actual2, 32); } + + /** Case with no common branch supertype must return a clean 4xx, not a 500. */ + @Test + public void testCaseWithIncompatibleBranchTypesRejectsCleanly() { + org.opensearch.client.ResponseException e = + org.junit.Assert.assertThrows( + org.opensearch.client.ResponseException.class, + () -> + executeQuery( + String.format( + "source=%s | eval x = case(age > 30, 'old', age > 20, 1 else 0.0) | fields" + + " x", + TEST_INDEX_BANK))); + org.junit.Assert.assertTrue( + "expected 400 status, got: " + e.getMessage(), + e.getMessage().contains("status line [HTTP/1.1 400")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java index 5cc257b0b0e..2788f7a9cfe 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java @@ -5,9 +5,13 @@ package org.opensearch.sql.ppl.calcite; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + import org.apache.calcite.rel.RelNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Test; +import org.opensearch.sql.exception.ExpressionEvaluationException; public class CalcitePPLCaseFunctionTest extends CalcitePPLAbstractTest { @@ -101,4 +105,16 @@ public void testCaseWhenInSubquery() { + "FROM `scott`.`EMP`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + /** Case branches with no common supertype must be rejected cleanly, not NPE. */ + @Test + public void testCaseWithIncompatibleBranchTypesRejectsCleanly() { + String ppl = + "source=EMP | eval x = case(DEPTNO > 20, 'big'," + " DEPTNO > 10, 1 else 0.0) | fields x"; + ExpressionEvaluationException e = + assertThrows(ExpressionEvaluationException.class, () -> getRelNode(ppl)); + assertTrue( + "expected message to list incompatible types, got: " + e.getMessage(), + e.getMessage().contains("case branches must have a common type")); + } } From 0b96cfc3f8c8126b46e71e9e78d57d3017182575 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 24 Jun 2026 10:48:37 -0700 Subject: [PATCH 14/41] fix lychee (#5451) Signed-off-by: Simeon Widdis --- .lycheeignore | 12 ++++++++++++ README.md | 2 -- async-query-core/README.md | 4 ++-- benchmarks/README.md | 6 +++--- docs/attributions.md | 2 +- docs/dev/intro-v3-architecture.md | 2 +- docs/dev/intro-v3-engine.md | 2 +- docs/dev/opensearch-relevancy-search.md | 4 ++-- docs/dev/testing-doctest.md | 2 +- .../opensearch-sql.release-notes-2.10.0.0.md | 2 +- .../opensearch-sql.release-notes-2.11.0.0.md | 2 +- 11 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 .lycheeignore diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 00000000000..335800e6047 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,12 @@ +# example opensearch/dashboards ports +http://localhost:5601 +http://localhost:9200 + +# sites that block scraping +https://hg.openjdk.org/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/ +https://www.linkedin.com/in/*/ + +# we have many pull requests links due to changelogs, skip these as they're unlikely to break and it saves API usage +https://github.com/opendistro-for-elasticsearch/sql/pull/* +https://github.com/opensearch-project/sql/pull/* + diff --git a/README.md b/README.md index 4a7e1e5ec9e..4059fd68ac2 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,6 @@ The following projects are related to the SQL plugin, but stored in the differen | [![untriaged open][untriaged-badge]][untriaged-link] | | [![nolabel open][nolabel-badge]][nolabel-link] | -[dco-badge]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml/badge.svg -[dco-badge-link]: https://github.com/opensearch-project/sql/actions/workflows/dco.yml [link-check-badge]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml/badge.svg [link-check-link]: https://github.com/opensearch-project/sql/actions/workflows/link-checker.yml [bwc-tests-badge]: https://img.shields.io/badge/BWC%20tests-in%20progress-yellow diff --git a/async-query-core/README.md b/async-query-core/README.md index 08301c024d7..f6d08ad7103 100644 --- a/async-query-core/README.md +++ b/async-query-core/README.md @@ -16,7 +16,7 @@ Following is the list of extension points where the consumer of the library need - Data store interface - [AsyncQueryJobMetadataStorageService](src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java) - - [SessionStorageService](java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) + - [SessionStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) - [StatementStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java) - [FlintIndexMetadataService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java) - [FlintIndexStateModelService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java) @@ -39,4 +39,4 @@ To update the grammar files, update `build.gradle` file (in `downloadG4Files` ta ``` ./gradlew async-query-core:downloadG4Files ``` -This will overwrite the files under `src/main/antlr`. \ No newline at end of file +This will overwrite the files under `src/main/antlr`. diff --git a/benchmarks/README.md b/benchmarks/README.md index 6a720c7201c..7467e37f2f0 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ # OpenSearch SQL/PPL Microbenchmark Suite -This directory contains the microbenchmark suite of OpenSearch SQL/PPL. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/). +This directory contains the microbenchmark suite of OpenSearch SQL/PPL. It relies on [JMH](https://openjdk.java.net/projects/code-tools/jmh/). ## Purpose @@ -25,6 +25,6 @@ Run specific benchmarks using the `-Pjmh.includes` parameter: ## Adding Microbenchmarks -Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the [JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). +Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the [JMH samples](https://hg.openjdk.org/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). -In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. \ No newline at end of file +In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. diff --git a/docs/attributions.md b/docs/attributions.md index cc16c35bae0..b5eb089181b 100644 --- a/docs/attributions.md +++ b/docs/attributions.md @@ -22,7 +22,7 @@ Apart from the problems we identified earlier, we made significant improvement i 1. *Integration Test*: We migrated all integrate tests to standard OpenSearch IT framework which spins up in-memory cluster for testing. Now all test cases treat plugin code as blackbox and verify functionality from externally. 2. *New JDBC Driver*: We developed our own JDBC driver without any dependency on Elasticsearch proprietary code. - [sql-jdbc](https://github.com/opensearch-project/sql/tree/main/sql-jdbc) + [sql-jdbc](https://github.com/opensearch-project/sql-jdbc) 3. *Better Hash JOIN*: Block Hash Join implementation with circuit break mechanism protect your OpenSearch memory. Performance testing showed our implementation is 1.5 ~ 2x better than old hash join in terms of throughput and latency and much lower error rate under heavy pressure. 4. *Query Planner*: Logical and physical planner was added to support JOIN query in efficient and extendible way. 5. *PartiQL Compatibility*: we are partially compatible with PartiQL specification which allows for query involved in nested JSON documents. diff --git a/docs/dev/intro-v3-architecture.md b/docs/dev/intro-v3-architecture.md index fc5bf3237d1..4f51fd1016b 100644 --- a/docs/dev/intro-v3-architecture.md +++ b/docs/dev/intro-v3-architecture.md @@ -24,7 +24,7 @@ PPL is specifically designed to simplify tasks in observability and security ana The current PPL engine (shared with SQL v2 engine) is built with custom components, including a parser, analyzer, optimizer, and relies heavily on OpenSearch DSL capabilities to execute query plans. By aligning its syntax and concepts with familiar languages like Splunk SPL and SQL, we aim to streamline migration for users from these backgrounds, allowing them to adopt PPL with minimal effort. The lack of comprehensive ability is a critical blocker for Splunk-to-OpenSearch migrations. We added ~20 new commands in PPL-on-Spark, but there are still dozens of command gaps to be filled. Not to mention that there are still a large number of functions to be implemented. ### 2.2 Lack of Unified PPL Experience -The PPL language is currently inconsistent across [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4). There are a lot of new commands added in PPL-on-Spark, such as `join`, `lookup` and `subsearch` are not yet supported in PPL-on-OpenSearch. As more and more new commands and functions are implemented in PPL-on-Spark, this gap will continue to widen. +The PPL language is currently inconsistent across [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/sql/blob/main/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4). There are a lot of new commands added in PPL-on-Spark, such as `join`, `lookup` and `subsearch` are not yet supported in PPL-on-OpenSearch. As more and more new commands and functions are implemented in PPL-on-Spark, this gap will continue to widen. ### 2.3 Lack of mature query optimizer Although the v2 engine framework comes with an optimizer class, it only has a few pushdown optimization rules and lacks of mature optimization rules and cost-based optimizer like those found in traditional databases. Query performance and scalability are core to PPL's design, enabling it to efficiently handle high-performance queries and scale to support large datasets and complex queries. diff --git a/docs/dev/intro-v3-engine.md b/docs/dev/intro-v3-engine.md index fd73cd5c8e1..2d11f30aeaa 100644 --- a/docs/dev/intro-v3-engine.md +++ b/docs/dev/intro-v3-engine.md @@ -89,5 +89,5 @@ If you're interested in the new query engine, please find more details in [V3 Ar The following items are on our roadmap with high priority: - Resolve the [V3 limitation](#33-limitations). - Advancing pushdown optimization and benchmarking -- Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4) +- Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/sql/blob/main/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4) - Support more DSL aggregation diff --git a/docs/dev/opensearch-relevancy-search.md b/docs/dev/opensearch-relevancy-search.md index 9b25cc757dd..94a63e9ed41 100644 --- a/docs/dev/opensearch-relevancy-search.md +++ b/docs/dev/opensearch-relevancy-search.md @@ -119,7 +119,7 @@ Besides, one of the query engine architecture tenets is to keep the job of every #### Option B: Create a new query plan node dedicated for the search features. -The diagram below is simplified with only the logical plan and physical plan sections and leaves out others. Please check out [OpenSearch SQL Engine Architecture](https://github.com/opensearch-project/sql/blob/main/docs/dev/Architecture.md) for the complete architecture of the query engine. +The diagram below is simplified with only the logical plan and physical plan sections and leaves out others. Please check out [OpenSearch SQL Engine Architecture](https://github.com/opensearch-project/sql/blob/main/docs/dev/intro-architecture.md) for the complete architecture of the query engine. ![relevance-Page-2 (4)](https://user-images.githubusercontent.com/33583073/129938534-28fa4845-4246-4707-9519-e68c9e86d174.png) @@ -291,4 +291,4 @@ All the code changes should be test driven. The pull requests should include uni ### A2. Search flow in search engine -![relevance](https://user-images.githubusercontent.com/33583073/129938659-5b49f43d-a83f-47d5-be5b-937b1c96e5bc.png) \ No newline at end of file +![relevance](https://user-images.githubusercontent.com/33583073/129938659-5b49f43d-a83f-47d5-be5b-937b1c96e5bc.png) diff --git a/docs/dev/testing-doctest.md b/docs/dev/testing-doctest.md index e8b88f19b0f..86fee86bd64 100644 --- a/docs/dev/testing-doctest.md +++ b/docs/dev/testing-doctest.md @@ -192,7 +192,7 @@ Doctest is relying on the console/command line to run code examples in documenta * https://github.com/crate/crate/blob/master/docs/general/dql/selects.rst -Similar to CrateDB using it’s CLI “crash”, we can make use of our own [SQL-CLI](https://github.com/opensearch-project/sql/tree/main/sql-cli) +Similar to CrateDB using it’s CLI “crash”, we can make use of our own [SQL-CLI](https://github.com/opensearch-project/sql-cli) To support PPL, we need to add PPL support to SQL-CLI. Since PPL and SQL expose similar http endpoint for query and share similar response format. The update won’t be much of work. diff --git a/release-notes/opensearch-sql.release-notes-2.10.0.0.md b/release-notes/opensearch-sql.release-notes-2.10.0.0.md index f1a730ce46b..67ce2c09275 100644 --- a/release-notes/opensearch-sql.release-notes-2.10.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.10.0.0.md @@ -40,4 +40,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.10.0 * [Backport 2.x] [spotless] Removes Checkstyle in favor of spotless by @MitchellGale in https://github.com/opensearch-project/sql/pull/2018 * [Backport 2.x] [Spotless] Entire project running spotless by @MitchellGale in https://github.com/opensearch-project/sql/pull/2016 --- -**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.10.0.0 \ No newline at end of file +**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...2.10.0.0 diff --git a/release-notes/opensearch-sql.release-notes-2.11.0.0.md b/release-notes/opensearch-sql.release-notes-2.11.0.0.md index a560d5c8dd7..a517b86bc48 100644 --- a/release-notes/opensearch-sql.release-notes-2.11.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.11.0.0.md @@ -52,4 +52,4 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.11.0 * bump okio to 3.4.0 by @joshuali925 in https://github.com/opensearch-project/sql/pull/2047 --- -**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...v.2.11.0.0 \ No newline at end of file +**Full Changelog**: https://github.com/opensearch-project/sql/compare/2.3.0.0...2.11.0.0 From e99aff00a418596e17777146843b5bb1ce498e3c Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:21:53 -0700 Subject: [PATCH 15/41] Stabilize CalciteStreamstatsCommandIT on the analytics-engine route (#5582) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate the streamstats ITs that diverge on the analytics-engine route (parquet-backed composite store, DataFusion backend) so the route runs green, while keeping every test active on the v2/Calcite path. Mechanism follows the established capability-gating pattern (#5560): an in-test @RequiresCapability(...) annotation plus a matching integTestRemote excludeTestsMatching entry; both are no-ops on the v2 route. Triaged single-shard and multi-shard (num_shards=3) analytics runs against the v2 baseline. Three groups: - DOC_MUTATION (4 tests): testStreamstatsGlobalWithNull, testStreamstatsGlobalWithNullBucket, testStreamstatsResetWithNull, testStreamstatsResetWithNullBucket seed state via PUT+DELETE. Doc-level DELETE is unsupported on the parquet store, and same-_id PUT is append-only, so the leaked doc inflated the row counts of every sibling test that reads the shared index. Gated with the same DOC_MUTATION capability the three existing mutation tests already carry. - CHAINED_STREAMSTATS_BY (4 tests): chaining two streamstats where an upstream stage partitions `by` a group emits a ROW_NUMBER() sequence column from each stage; the Substrait converter names both physical columns identically, so the stacked schema has a duplicate/ambiguous field name (500) or, for chained window streamstats, non-deterministic values. The Calcite logical plan is correct; the alias is lost in Substrait conversion. Fails single- and multi-shard. - STREAMSTATS_SORT_NOT_HONORED (1 test): testStreamstatsAndSort. The window is computed over the backend scan order, ignoring a preceding `| sort` (the OVER clause carries no explicit ORDER BY), so the per-row aggregates diverge from the v2/Calcite path. Pass rate on the single-shard analytics route, CalciteStreamstatsCommandIT: | metric | before | after | |-----------|--------|-------| | tests run | 47 | 42 | | failures | 12 | 0 | | skipped | 3 | 7 | (The before-failures count is inflated by the DOC_MUTATION leak described above; the four leaking tests plus their downstream row-count victims all clear once gated.) v2 route (:integTest): 47 run, 0 failed, 0 skipped — gates are no-ops off the analytics route. Twelve further tests fail only on the multi-shard route (they pass single-shard) due to cross-shard fragment-order non-determinism in the streamstats window gather; that is an engine-side gap and is left unchanged here rather than gated, to avoid skipping passing tests on the single-shard route. Signed-off-by: Kai Huang --- integ-test/build.gradle | 14 ++++++++ .../remote/CalciteStreamstatsCommandIT.java | 11 +++++++ .../org/opensearch/sql/util/Capability.java | 32 ++++++++++++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d4780f90ea9..17b88a901de 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1328,6 +1328,20 @@ task integTestRemote(type: RestIntegTestTask) { // === Excludes: asserts a Lucene pushdown fragment absent on the AE route === excludeTestsMatching '*CalciteSortCommandIT.testPushdownSortCastToDoubleExpression' + + // === Excludes: CalciteStreamstatsCommandIT route divergences === + // Each test also carries an in-test @RequiresCapability(...) recording the reason. + // - CHAINED_STREAMSTATS_BY: chaining two streamstats where an upstream stage has `by` + // emits two ROW_NUMBER() sequence columns the Substrait converter names identically, + // so the stacked schema has a duplicate/ambiguous field name (500) or, for chained + // window streamstats, non-deterministic values. Fails single- and multi-shard. + excludeTestsMatching '*CalciteStreamstatsCommandIT.testMultipleStreamstats' + excludeTestsMatching '*CalciteStreamstatsCommandIT.testMultipleStreamstatsWithWindow' + excludeTestsMatching '*CalciteStreamstatsCommandIT.testMultipleStreamstatsWithNull1' + excludeTestsMatching '*CalciteStreamstatsCommandIT.testMultipleStreamstatsWithEval' + // - STREAMSTATS_SORT_NOT_HONORED: streamstats computes its window over the backend scan + // order, ignoring a preceding `| sort` (the OVER clause has no explicit ORDER BY). + excludeTestsMatching '*CalciteStreamstatsCommandIT.testStreamstatsAndSort' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java index 43ede1606bc..e70812e3c3b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java @@ -6,7 +6,9 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.CHAINED_STREAMSTATS_BY; import static org.opensearch.sql.util.Capability.DOC_MUTATION; +import static org.opensearch.sql.util.Capability.STREAMSTATS_SORT_NOT_HONORED; import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; @@ -558,6 +560,7 @@ public void testStreamstatsGlobal() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsGlobalWithNull() throws IOException { final int docId = 7; Request insertRequest = @@ -613,6 +616,7 @@ public void testStreamstatsGlobalWithNull() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsGlobalWithNullBucket() throws IOException { final int docId = 7; Request insertRequest = @@ -718,6 +722,7 @@ public void testStreamstatsReset() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsResetWithNull() throws IOException { final int docId = 7; Request insertRequest = @@ -773,6 +778,7 @@ public void testStreamstatsResetWithNull() throws IOException { } @Test + @RequiresCapability(DOC_MUTATION) public void testStreamstatsResetWithNullBucket() throws IOException { final int docId = 7; Request insertRequest = @@ -845,6 +851,7 @@ public void testUnsupportedWindowFunctions() { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstats() throws IOException { JSONObject actual = executeQuery( @@ -863,6 +870,7 @@ public void testMultipleStreamstats() throws IOException { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithWindow() throws IOException { // Test case from GitHub issue #4800: chained streamstats with window=2 JSONObject actual = @@ -899,6 +907,7 @@ public void testMultipleStreamstatsWithWindow() throws IOException { // causing Calcite's RelDecorrelator to fail on duplicate correlate references. @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithNull1() throws IOException { JSONObject actual = executeQuery( @@ -1008,6 +1017,7 @@ public void testStreamstatsAndEventstats() throws IOException { } @Test + @RequiresCapability(STREAMSTATS_SORT_NOT_HONORED) public void testStreamstatsAndSort() throws IOException { JSONObject actual = executeQuery( @@ -1074,6 +1084,7 @@ public void testWhereInWithStreamstatsSubquery() throws IOException { } @Test + @RequiresCapability(CHAINED_STREAMSTATS_BY) public void testMultipleStreamstatsWithEval() throws IOException { JSONObject actual = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index 1ca2b9adee7..3bac06e7fca 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -469,7 +469,37 @@ public enum Capability { LUCENE_PUSHDOWN_EXPLAIN( "A test asserting a Lucene-specific pushdown fragment in the explain plan (e.g. SORT->[...])" + " can't pass on the analytics-engine route: the DataFusion scan produces a different" - + " plan with no Lucene pushdown fragment."); + + " plan with no Lucene pushdown fragment."), + + /** + * Chaining two {@code streamstats} commands where an upstream stage partitions {@code by} a group + * fails on the analytics-engine route. Each {@code streamstats ... by} stage projects a {@code + * ROW_NUMBER() OVER ()} sequence column to order its window; the Calcite plan aliases these + * distinctly ({@code __stream_seq__}), but the Substrait converter names both physical columns + * after the operator ({@code "row_number() ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"}), + * so the stacked schema has a duplicate/ambiguous field name. Verified: it surfaces as a 500 + * ({@code Schema contains duplicate unqualified field name ...} / streaming-fragment failure) or, + * for chained {@code window} streamstats, non-deterministic window values. A single {@code + * streamstats by} (or a chain where only the final stage has {@code by}) works. + */ + CHAINED_STREAMSTATS_BY( + "Chaining two streamstats where an upstream stage partitions by a group fails on the" + + " analytics-engine route: both stages emit a ROW_NUMBER() sequence column the Substrait" + + " converter names identically, producing a duplicate/ambiguous field name (500) or" + + " non-deterministic window values."), + + /** + * {@code streamstats} computes its running/window aggregate over the backend scan order on the + * analytics-engine route, ignoring a preceding {@code | sort}. The {@code OVER} clause carries no + * explicit {@code ORDER BY} (streamstats orders by encounter order by design), so DataFusion + * evaluates the window in scan order rather than the sorted order the v2/Calcite path honors. + * Verified: {@code sort age | streamstats window=2 avg(age)} yields window values computed in + * insertion order, not age order, so the per-row aggregates diverge. + */ + STREAMSTATS_SORT_NOT_HONORED( + "streamstats computes its window over the backend scan order on the analytics-engine route," + + " ignoring a preceding | sort (the OVER clause has no explicit ORDER BY), so the window" + + " values diverge from the v2/Calcite path which honors the sort."); private final String reason; From cb10516b43d60822f6220c309cdb7c3734d1d33f Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 25 Jun 2026 09:54:53 -0700 Subject: [PATCH 16/41] [BugFix] Bump get-ci-image-tag.yml ref to SHA-pinned opensearch-build commit to unblock CI (#5583) Signed-off-by: Eric Wei --- .github/workflows/analytics-engine-compat.yml | 2 +- .github/workflows/integ-tests-with-security.yml | 2 +- .github/workflows/issue-dedupe.yml | 4 ++-- .github/workflows/pr_review.yml | 4 ++-- .github/workflows/sql-pitest.yml | 2 +- .github/workflows/sql-test-and-build-workflow.yml | 2 +- .github/workflows/sql-test-workflow.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/analytics-engine-compat.yml b/.github/workflows/analytics-engine-compat.yml index f1e722b0579..e39adb0a311 100644 --- a/.github/workflows/analytics-engine-compat.yml +++ b/.github/workflows/analytics-engine-compat.yml @@ -15,7 +15,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 5ef979e2875..10d66bc30a9 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -13,7 +13,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/issue-dedupe.yml b/.github/workflows/issue-dedupe.yml index ee0b162e772..609d356eb29 100644 --- a/.github/workflows/issue-dedupe.yml +++ b/.github/workflows/issue-dedupe.yml @@ -23,7 +23,7 @@ on: jobs: detect: if: (github.event_name == 'issues' && github.event.issue.user.type != 'Bot') || (github.event_name == 'workflow_dispatch' && inputs.job == 'detect') - uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-detect.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main permissions: contents: read issues: write @@ -36,7 +36,7 @@ jobs: auto-close: if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.job == 'auto-close') - uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/issue-dedupe-autoclose.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main permissions: issues: write with: diff --git a/.github/workflows/pr_review.yml b/.github/workflows/pr_review.yml index ccafed6754f..e351989a11d 100644 --- a/.github/workflows/pr_review.yml +++ b/.github/workflows/pr_review.yml @@ -6,7 +6,7 @@ on: jobs: Code-Diff-Analyzer: - uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/code-diff-analyzer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main if: github.repository == 'opensearch-project/sql' permissions: id-token: write # github oidc to assume aws roles @@ -18,7 +18,7 @@ jobs: update_pr_comment_with_analyzer_report: true Code-Diff-Reviewer: - uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/code-diff-reviewer.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main needs: Code-Diff-Analyzer if: github.repository == 'opensearch-project/sql' permissions: diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index 8c0fbd305fe..2085e3afc97 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -12,7 +12,7 @@ run-name: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 2b8f2e25794..6e24e2838fd 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -20,7 +20,7 @@ on: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index ec23758e126..990cb45010c 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -12,7 +12,7 @@ run-name: jobs: Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@c2498b758c08fb7bc48476509a5fc1b8dd5f7493 # main + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch From cc65d75a96fcf3c3073253977a2ad56d2de41e04 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Thu, 25 Jun 2026 12:03:22 -0700 Subject: [PATCH 17/41] Fix SQL IT test queries, assertions, and data for engine-agnostic compatibility (#5584) * test(integ-test): fix engine-agnostic IT queries Stabilize non-deterministic GROUP BY results with ORDER BY, normalize LENGTH() case, and use ANSI positional GROUP BY. Correctness fixes that apply regardless of execution engine. Signed-off-by: Chen Dai * test(integ-test): relax schema matcher for analytics engine Signed-off-by: Chen Dai --------- Signed-off-by: Chen Dai --- .../sql/legacy/OrdinalAliasRewriterIT.java | 24 +++++++++-------- .../sql/legacy/PrettyFormatResponseIT.java | 6 ++++- .../sql/legacy/TypeInformationIT.java | 4 +-- .../org/opensearch/sql/sql/ConditionalIT.java | 20 ++++---------- .../org/opensearch/sql/util/MatcherUtils.java | 27 +++++++++++++++++++ .../resources/game_of_thrones_complex.json | 14 +++++----- 6 files changed, 59 insertions(+), 36 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java index caea2aa7c66..92bbe365e84 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OrdinalAliasRewriterIT.java @@ -25,13 +25,13 @@ public void simpleGroupByOrdinal() { String expected = executeQuery( StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY lastname LIMIT 3", + "SELECT lastname FROM %s AS b GROUP BY lastname ORDER BY lastname LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT lastname FROM %s AS b GROUP BY 1 LIMIT 3", + "SELECT lastname FROM %s AS b GROUP BY 1 ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -43,13 +43,14 @@ public void multipleGroupByOrdinal() { executeQuery( StringUtils.format( "SELECT lastname, firstname, age FROM %s AS b GROUP BY firstname, age, lastname" - + " LIMIT 3", + + " ORDER BY lastname, firstname, age LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1 LIMIT 3", + "SELECT lastname, firstname, age FROM %s AS b GROUP BY 2, 3, 1" + + " ORDER BY 1, 2, 3 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -60,13 +61,13 @@ public void selectFieldiWithBacticksGroupByOrdinal() { String expected = executeQuery( StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY `lastname` LIMIT 3", + "SELECT `lastname` FROM %s AS b GROUP BY `lastname` ORDER BY `lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `lastname` FROM %s AS b GROUP BY 1 LIMIT 3", + "SELECT `lastname` FROM %s AS b GROUP BY 1 ORDER BY 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -78,13 +79,14 @@ public void selectFieldiWithBacticksAndTableAliasGroupByOrdinal() { executeQuery( StringUtils.format( "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY `age`," - + " `b`.`lastname` , firstname LIMIT 10", + + " `b`.`lastname` , firstname ORDER BY `b`.`lastname`, `age` LIMIT 10", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 LIMIT 10", + "SELECT `b`.`lastname`, `age`, firstname FROM %s AS b GROUP BY 2, 1, 3 ORDER BY 1," + + " 2 LIMIT 10", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); @@ -166,14 +168,14 @@ public void selectFieldiWithBacticksAndTableAliasOrderByOrdinalAndNull() { executeQuery( StringUtils.format( "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY `b`.`lastname` IS NOT NULL DESC," - + " age is NULL LIMIT 3", + + " age is NULL, `b`.`lastname` LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); String actual = executeQuery( StringUtils.format( - "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL" - + " LIMIT 3", + "SELECT `b`.`lastname`, age FROM %s AS b ORDER BY 1 IS NOT NULL DESC, 2 IS NULL," + + " 1 LIMIT 3", TestsConstants.TEST_INDEX_ACCOUNT), "jdbc"); assertThat(actual, equalTo(expected)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index 6745d90d50b..b4218db6abf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -80,7 +80,11 @@ protected void init() throws Exception { loadIndex(Index.ACCOUNT); loadIndex(Index.PHRASE); loadIndex(Index.GAME_OF_THRONES); - loadIndex(Index.NESTED); + // Skip on the analytics-engine route, where the parquet store rejects nested_objects' + // multi-value array in the scalar-mapped myNum field at bulk load. + if (!TestUtils.AnalyticsIndexConfig.isEnabled()) { + loadIndex(Index.NESTED); + } } @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java index 421aae9622b..d6f238fbff1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TypeInformationIT.java @@ -96,11 +96,11 @@ public void testLengthWithTextFieldReturnsInt() { public void testLengthWithGroupByExpr() { JSONObject response = executeJdbcRequest( - "SELECT Length(firstname) FROM " + "SELECT LENGTH(firstname) FROM " + TestsConstants.TEST_INDEX_ACCOUNT + " GROUP BY LENGTH(firstname) LIMIT 5"); - verifySchema(response, schema("Length(firstname)", null, "integer")); + verifySchema(response, schema("LENGTH(firstname)", null, "integer")); } /* diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java index 1f8aba29dea..05e6af5acbc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java @@ -46,12 +46,8 @@ public void init() throws Exception { public void ifnullShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest( - "SELECT IFNULL(lastname, 'unknown') AS name FROM " - + TEST_INDEX_ACCOUNT - + " GROUP BY name"); - assertEquals("IFNULL(lastname, 'unknown')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + "SELECT IFNULL(lastname, 'unknown') FROM " + TEST_INDEX_ACCOUNT + " GROUP BY 1"); + verifySchema(response, schema("IFNULL(lastname, 'unknown')", null, "keyword")); } @Test @@ -109,9 +105,7 @@ public void ifnullWithMissingInputTest() { public void nullifShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT NULLIF(lastname, 'unknown') AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("NULLIF(lastname, 'unknown')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + verifySchema(response, schema("NULLIF(lastname, 'unknown')", "name", "keyword")); } @Test @@ -152,9 +146,7 @@ public void nullifWithNullInputTest() { public void isnullShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT ISNULL(lastname) AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("ISNULL(lastname)", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("boolean", response.query("/schema/0/type")); + verifySchema(response, schema("ISNULL(lastname)", "name", "boolean")); } @Ignore( @@ -208,9 +200,7 @@ public void isnullWithMathExpr() throws IOException { public void ifShouldPassJDBC() throws IOException { JSONObject response = executeJdbcRequest("SELECT IF(2 > 0, 'hello', 'world') AS name FROM " + TEST_INDEX_ACCOUNT); - assertEquals("IF(2 > 0, 'hello', 'world')", response.query("/schema/0/name")); - assertEquals("name", response.query("/schema/0/alias")); - assertEquals("keyword", response.query("/schema/0/type")); + verifySchema(response, schema("IF(2 > 0, 'hello', 'world')", "name", "keyword")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index dc9ebc30de5..4fb8bd28129 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -27,6 +27,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; @@ -39,6 +40,7 @@ import org.json.JSONObject; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.sql.legacy.TestUtils; import org.opensearch.sql.utils.YamlFormatter; public class MatcherUtils { @@ -275,6 +277,9 @@ public static TypeSafeMatcher schema(String expectedName, String exp public static TypeSafeMatcher schema( String expectedName, String expectedAlias, String expectedType) { return new TypeSafeMatcher() { + private static final Set NUMERIC_TYPES = Set.of("integer", "long", "float", "double"); + private static final Set STRING_TYPES = Set.of("keyword", "text", "string"); + @Override public void describeTo(Description description) { description.appendText( @@ -287,10 +292,32 @@ protected boolean matchesSafely(JSONObject jsonObject) { String actualName = (String) jsonObject.query("/name"); String actualAlias = (String) jsonObject.query("/alias"); String actualType = (String) jsonObject.query("/type"); + + if (TestUtils.AnalyticsIndexConfig.isEnabled()) { + // The analytics-engine route promotes the alias to the name (SQL-standard) and unifies + // keyword/text/string and the numeric types, so relax name and type matching for it only. + boolean nameMatches = + expectedName.equals(actualName) + || (!Strings.isNullOrEmpty(expectedAlias) && expectedAlias.equals(actualName)); + boolean typeMatches = + expectedType.equals(actualType) || isCompatibleType(expectedType, actualType); + return nameMatches && typeMatches; + } + return expectedName.equals(actualName) && (Strings.isNullOrEmpty(expectedAlias) || expectedAlias.equals(actualAlias)) && expectedType.equals(actualType); } + + private boolean isCompatibleType(String expected, String actual) { + if (expected == null || actual == null) { + return false; + } + String e = expected.toLowerCase(); + String a = actual.toLowerCase(); + return (NUMERIC_TYPES.contains(e) && NUMERIC_TYPES.contains(a)) + || (STRING_TYPES.contains(e) && STRING_TYPES.contains(a)); + } }; } diff --git a/integ-test/src/test/resources/game_of_thrones_complex.json b/integ-test/src/test/resources/game_of_thrones_complex.json index 240344e25eb..7be567b6de9 100644 --- a/integ-test/src/test/resources/game_of_thrones_complex.json +++ b/integ-test/src/test/resources/game_of_thrones_complex.json @@ -1,14 +1,14 @@ {"index":{"_id":"1"}} -{"name":{"firstname":"Daenerys","lastname":"Targaryen","ofHerName":1},"nickname":"Daenerys \"Stormborn\"","house":"Targaryen","gender":"F","parents":{"father":"Aerys" , "mother":"Rhaella"},"titles":["motherOfDragons","queenOfTheAndals","breakerOfChains","Khaleesi"]} +{"name": {"firstname": "Daenerys", "lastname": "Targaryen", "ofHerName": 1}, "nickname": "Daenerys \"Stormborn\"", "house": "Targaryen", "gender": "F", "parents": {"father": "Aerys", "mother": "Rhaella"}} {"index":{"_id":"2"}} -{"name":{"firstname":"Eddard","lastname":"Stark","ofHisName":1},"house":"Stark", "parents":{"father":"Rickard" , "mother":"Lyarra"} ,"gender":"M","titles":["lordOfWinterfell","wardenOfTheNorth","handOfTheKing"]} +{"name": {"firstname": "Eddard", "lastname": "Stark", "ofHisName": 1}, "house": "Stark", "parents": {"father": "Rickard", "mother": "Lyarra"}, "gender": "M"} {"index":{"_id":"3"}} -{"name":{"firstname":"Brandon","lastname":"Stark","ofHisName":4},"house":"Stark","parents":{"father":"Eddard","mother":"Catelyn"},"gender":"M","titles":["princeOfWinterfell"],"@wolf":"Summer"} +{"name": {"firstname": "Brandon", "lastname": "Stark", "ofHisName": 4}, "house": "Stark", "parents": {"father": "Eddard", "mother": "Catelyn"}, "gender": "M", "@wolf": "Summer"} {"index":{"_id":"4"}} -{"name":{"firstname":"Jaime","lastname":"Lannister","ofHisName":1},"gender":"M","house":"Lannister","parents":{"father":"Tywin","mother":"Joanna"},"titles":["kingSlayer","lordCommanderOfTheKingsguard","Ser"]} +{"name": {"firstname": "Jaime", "lastname": "Lannister", "ofHisName": 1}, "gender": "M", "house": "Lannister", "parents": {"father": "Tywin", "mother": "Joanna"}} {"index":{"_id":"5"}} -{"words":"fireAndBlood","hname":"Targaryen","sigil":"Dragon","seat":"Dragonstone"} +{"words": "fireAndBlood", "hname": "Targaryen", "sigil": "Dragon", "seat": "Dragonstone"} {"index":{"_id":"6"}} -{"words":"winterIsComing" , "hname":"Stark","sigil":"direwolf","seat":"Winterfell"} +{"words": "winterIsComing", "hname": "Stark", "sigil": "direwolf", "seat": "Winterfell"} {"index":{"_id":"7"}} -{"words":"hearMeRoar" , "hname":"Lannister","sigil":"lion","seat":"CasterlyRock"} +{"words": "hearMeRoar", "hname": "Lannister", "sigil": "lion", "seat": "CasterlyRock"} From 0a4d40e31d4411fb7a32a8eb73b5acc1f552117c Mon Sep 17 00:00:00 2001 From: Radhakrishnan Pachyappan Date: Fri, 26 Jun 2026 00:38:26 +0530 Subject: [PATCH 18/41] Fix ClassCastException in PPL multisearch on indexes with @timestamp alias field (#5577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #5533. When `@timestamp` is defined as a field-type alias in the index mapping, multisearch queries threw: ClassCastException: RelCompositeTrait cannot be cast to RelCollation Root cause: `reIndexCollations()` in `CalciteLogicalIndexScan` and `pushDownSort()` in `AbstractCalciteIndexScan` both called `RelTraitSet.plus()` to update the collation trait on a scan node. `plus()` *composes* traits — if the trait set already contains a `RelCollation`, it merges the old and new collations into a `RelCompositeTrait`. Calcite's `RelTraitSet.getCollation()` then does an unchecked cast `(RelCollation) getTrait(...)` which fails at runtime for `RelCompositeTrait`. The `@timestamp` alias path specifically triggers this because `wrapProjectForAliasFields()` adds a project on top of each sub-scan which is later pushed back down via `pushDownProject()`. `pushDownProject()` calls `reIndexCollations()` to remap field indices inside an existing collation — but re-using `plus()` here composes the existing sort collation with the re-indexed one, producing the bad composite. Fix: use `RelTraitSet.replace()` in both locations. `replace()` substitutes the collation trait in-place regardless of what was there before, which is the correct semantics for "this scan is now sorted by these columns". Added a regression IT (`testMultisearchWithTimestampAliasFieldDoesNotThrow`) that runs a multisearch against `TEST_INDEX_ALIAS`, whose mapping defines `@timestamp` as an alias for `original_date`. Signed-off-by: Radhakrishnan Pachyappan --- .../remote/CalciteMultisearchCommandIT.java | 41 +++++++++++++++++++ .../scan/AbstractCalciteIndexScan.java | 2 +- .../storage/scan/CalciteLogicalIndexScan.java | 2 +- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java index 10cc7ffd459..383ae5e400f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.Capability.MULTISEARCH_COLUMN_ORDER; import static org.opensearch.sql.util.Capability.MULTISEARCH_SAME_INDEX_CONFLATION; @@ -12,6 +13,7 @@ import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; import java.io.IOException; import org.json.JSONObject; @@ -31,6 +33,7 @@ public void init() throws Exception { loadIndex(Index.TIME_TEST_DATA); loadIndex(Index.TIME_TEST_DATA2); loadIndex(Index.LOCATIONS_TYPE_CONFLICT); + loadIndex(Index.DATA_TYPE_ALIAS); } @Test @@ -462,4 +465,42 @@ public void testMultisearchTypeConflictWithStats() { .getMessage() .contains("Unable to process column 'age' due to incompatible types:")); } + + /** + * Regression test for GitHub issue #5533. When {@code @timestamp} is defined as a field-type + * alias in the index mapping, multisearch used to throw: + * + * 

ClassCastException: RelCompositeTrait cannot be cast to RelCollation
+ * + *

Root cause: {@code reIndexCollations()} and {@code pushDownSort()} both used {@code + * RelTraitSet.plus()} which composes collation traits into a {@link + * org.apache.calcite.rel.RelCompositeTrait} when a collation is already present. Calcite's {@code + * RelTraitSet.getCollation()} then fails with a ClassCastException. Fixed by using {@code + * RelTraitSet.replace()} instead to always replace the collation trait. + */ + @Test + public void testMultisearchWithTimestampAliasFieldDoesNotThrow() throws IOException { + // alias-typed fields are stripped when loading indices in analytics-engine parquet mode, + // so @timestamp does not exist in TEST_INDEX_ALIAS on that route. + assumeFalse( + "alias-typed fields are stripped in analytics-engine parquet mode;" + + " @timestamp won't exist in TEST_INDEX_ALIAS on that route.", + isAnalyticsParquetIndicesEnabled()); + // TEST_INDEX_ALIAS has @timestamp defined as an alias field pointing to original_date. + // Running multisearch on such an index used to crash with ClassCastException. + JSONObject result = + executeQuery( + String.format( + "| multisearch " + + "[search source=%s | where original_col > 1 | fields original_col," + + " @timestamp] " + + "[search source=%s | where original_col = 1 | fields original_col," + + " @timestamp]", + TEST_INDEX_ALIAS, TEST_INDEX_ALIAS)); + + verifySchemaInOrder( + result, schema("original_col", null, "int"), schema("@timestamp", null, "timestamp")); + // 2 rows from original_col > 1, 1 row from original_col = 1 + assertEquals(3, result.getInt("total")); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 609a5aaa92f..52d64fb5a73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -323,7 +323,7 @@ && isAnyCollationNameInAggregators(collationNames)) { // aggregators. return null; } - RelTraitSet traitsWithCollations = getTraitSet().plus(RelCollations.of(collations)); + RelTraitSet traitsWithCollations = getTraitSet().replace(RelCollations.of(collations)); PushDownContext pushDownContextWithoutSort = this.pushDownContext.cloneWithoutSort(); AbstractAction action; Object digest; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 740801ff418..2017437e7bd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -321,7 +321,7 @@ private RelTraitSet reIndexCollations(List selectedColumns) { collation -> collation.withFieldIndex(selectedColumns.indexOf(collation.getFieldIndex()))) .collect(Collectors.toList()); - newTraitSet = getTraitSet().plus(RelCollations.of(newCollations)); + newTraitSet = getTraitSet().replace(RelCollations.of(newCollations)); } else { newTraitSet = getTraitSet(); } From 1dc92d66c409fe53c8cdf8d9224f93e05c7bf4ff Mon Sep 17 00:00:00 2001 From: Michael Oviedo Date: Thu, 25 Jun 2026 14:46:48 -0700 Subject: [PATCH 19/41] Return 4xx instead of 500 for unsupported window functions (#5587) Window functions outside WINDOW_FUNC_MAPPING (e.g. RANK) used to escape the AE route as HTTP 500 because the throw site emitted a raw UnsupportedOperationException, which UnifiedQueryPlanner rethrows unchanged. Switching to CalciteUnsupportedException lets the existing 4xx wrapper added in #5569 normalize it to SemanticCheckException. Repro: SELECT RegionID, COUNT(*) AS cnt, RANK() OVER (ORDER BY COUNT(*) DESC) AS rnk FROM clickbench GROUP BY RegionID LIMIT 5 Signed-off-by: Michael Oviedo --- .../opensearch/sql/api/UnifiedQueryPlannerTest.java | 12 ++++++++++++ .../sql/calcite/CalciteRexNodeVisitor.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java index bb2d1e4a53f..008121e8377 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -159,6 +159,18 @@ public void unsupportedFeatureIsRethrownAsSemanticCheckException() { .assertErrorMessageContains("unsupported in Calcite"); } + @Test + public void unsupportedWindowFunctionIsRethrownAsSemanticCheckException() { + // Window functions outside WINDOW_FUNC_MAPPING reach + // CalciteRexNodeVisitor#visitWindowFunction's + // orElseThrow. The throw site emits CalciteUnsupportedException so this path normalizes to a + // 4xx SemanticCheckException rather than escaping as a 500. + givenInvalidQuery("source = catalog.employees | eventstats rank()") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteUnsupportedException.class) + .assertErrorMessageContains("Unexpected window function: rank"); + } + @Test public void assertionErrorIsWrappedAsSemanticCheckException() { // Remove when the underlying Calcite assertion is fixed. diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 3c37a11ba5b..849b615f970 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -705,7 +705,7 @@ public RexNode visitWindowFunction(WindowFunction node, CalcitePlanContext conte node.getWindowFrame()); }) .orElseThrow( - () -> new UnsupportedOperationException("Unexpected window function: " + funcName)); + () -> new CalciteUnsupportedException("Unexpected window function: " + funcName)); } private List translateOrderKeys( From 2e9114b1a46472bb091fce3f9fd22d87b38a68fa Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Mon, 29 Jun 2026 11:55:56 -0700 Subject: [PATCH 20/41] fix: window functions with ORDER BY/LIMIT on unified SQL path (#5592) This PR fixes SQL window functions used with ORDER BY / LIMIT, which produced wrong plans for Analytics Engine). Because fixing the shared AstBuilder directly would impact the SQL V2 engine (V2 requires the top operator to be a Project), the fix is implemented in the extended AST builder of the unified query API only. Signed-off-by: Chen Dai --- .../sql/api/parser/SqlV2QueryParser.java | 86 +++++++++++++++++++ .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 69 +++++++++++++++ .../opensearch/sql/sql/parser/AstBuilder.java | 4 +- .../sql/sql/parser/AstSortBuilder.java | 4 +- 4 files changed, 159 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java index d60ef7ed4ec..1078c58bcc1 100644 --- a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java +++ b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java @@ -10,25 +10,36 @@ import static org.opensearch.sql.ast.dsl.AstDSL.join; import static org.opensearch.sql.ast.dsl.AstDSL.union; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.antlr.v4.runtime.tree.ParseTree; +import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.ast.tree.Join.JoinType; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.Sort; +import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExistsSubqueryExpressionAtomContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FromClauseContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.InSubqueryPredicateContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.JoinClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.OrderByClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.QuerySpecificationContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.UnionSelectContext; import org.opensearch.sql.sql.parser.AstBuilder; import org.opensearch.sql.sql.parser.AstExpressionBuilder; +import org.opensearch.sql.sql.parser.AstSortBuilder; import org.opensearch.sql.sql.parser.AstStatementBuilder; +import org.opensearch.sql.sql.parser.context.QuerySpecification; /** SQL query parser that produces {@link UnresolvedPlan} using the V2 ANTLR grammar. */ public class SqlV2QueryParser implements UnifiedQueryParser { @@ -62,6 +73,53 @@ private static class ExtendedAstBuilder extends AstBuilder { super(query); } + @Override + public UnresolvedPlan visitQuerySpecification(QuerySpecificationContext queryContext) { + if (!hasWindowFunctionInProjectList(queryContext)) { + return super.visitQuerySpecification(queryContext); + } + + context.push(); + context.peek().collect(queryContext, query); + Project project = (Project) visit(queryContext.selectClause()); + UnresolvedPlan result = project.attach(visit(queryContext.fromClause())); + + // Window output must be computed before ORDER BY/LIMIT, so build Limit(Sort(Project(from))) + OrderByClauseContext orderByClause = queryContext.fromClause().orderByClause(); + if (orderByClause != null) { + result = new ExtendedAstSortBuilder(context.peek()).visit(orderByClause).attach(result); + } + if (queryContext.limitClause() != null) { + result = visit(queryContext.limitClause()).attach(result); + } + + context.pop(); + return result; + } + + @Override + public UnresolvedPlan visitFromClause(FromClauseContext ctx) { + UnresolvedPlan from = super.visitFromClause(ctx); + if (hasWindowFunctionInProjectList(context.peek()) && from instanceof Sort sort) { + // Drop the ORDER BY Sort for window queries; it is re-attached above the Project + return sort.getChild().get(0); + } + return from; + } + + private boolean hasWindowFunctionInProjectList(QuerySpecificationContext queryContext) { + if (queryContext.fromClause() == null) { + return false; + } + QuerySpecification probe = new QuerySpecification(); + probe.collect(queryContext, query); + return hasWindowFunctionInProjectList(probe); + } + + private static boolean hasWindowFunctionInProjectList(QuerySpecification querySpec) { + return querySpec.getSelectItems().stream().anyMatch(item -> item instanceof WindowFunction); + } + @Override protected AstExpressionBuilder createExpressionBuilder() { return new ExtendedAstExpressionBuilder(); @@ -114,4 +172,32 @@ public UnresolvedExpression visitExistsSubqueryExpressionAtom( } } } + + /** + * Keeps an ORDER BY window-alias as a column reference (Sort is above the Project) to avoid a + * second RexOver. + */ + private static class ExtendedAstSortBuilder extends AstSortBuilder { + + ExtendedAstSortBuilder(QuerySpecification querySpec) { + super(querySpec); + } + + @Override + public UnresolvedPlan visitOrderByClause(OrderByClauseContext ctx) { + List fields = new ArrayList<>(); + List items = querySpec.getOrderByItems(); + List options = querySpec.getOrderByOptions(); + for (int i = 0; i < items.size(); i++) { + UnresolvedExpression item = items.get(i); + UnresolvedExpression sortKey = + (querySpec.isSelectAlias(item) + && querySpec.getSelectItemByAlias(item) instanceof WindowFunction) + ? item + : querySpec.replaceIfAliasOrOrdinal(item); + fields.add(new Field(sortKey, createSortArguments(options.get(i)))); + } + return new Sort(fields); + } + } } diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 8538ebbae61..064eff32d76 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -547,4 +547,73 @@ SELECT LENGTH(name) FROM catalog.employees GROUP BY LENGTH(name) ORDER BY LENGTH LogicalTableScan(table=[[catalog, employees]]) """); } + + @Test + public void testWindowOverGroupByWithLimit() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testWindowOverGroupByOrderByWindowAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department ORDER BY rn LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testWindowOverGroupByOrderByWindowAliasWithoutLimit() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn + FROM catalog.employees GROUP BY department ORDER BY rn + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testMultipleWindowFunctionsOrderByWindowAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt, ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS rn, + ROW_NUMBER() OVER (ORDER BY department) AS rn2 + FROM catalog.employees GROUP BY department ORDER BY rn LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$2], dir0=[ASC-nulls-first], fetch=[3]) + LogicalProject(department=[$0], cnt=[$1], rn=[ROW_NUMBER() OVER (ORDER BY $1 DESC NULLS FIRST)], rn2=[ROW_NUMBER() OVER (ORDER BY $0 NULLS FIRST)]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } } diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index aaed2ba5ec2..641ef0d39ca 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -54,13 +54,13 @@ public class AstBuilder extends OpenSearchSQLParserBaseVisitor { private final AstExpressionBuilder expressionBuilder; /** Parsing context stack that contains context for current query parsing. */ - private final ParsingContext context = new ParsingContext(); + protected final ParsingContext context = new ParsingContext(); /** * SQL query to get original token text. This is necessary because token.getText() returns text * without whitespaces or other characters discarded by lexer. */ - private final String query; + protected final String query; public AstBuilder(String query) { this.query = query; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java index 2594709f4f4..1647bc9ee9e 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstSortBuilder.java @@ -33,7 +33,7 @@ @RequiredArgsConstructor public class AstSortBuilder extends OpenSearchSQLParserBaseVisitor { - private final QuerySpecification querySpec; + protected final QuerySpecification querySpec; @Override public UnresolvedPlan visitOrderByClause(OrderByClauseContext ctx) { @@ -57,7 +57,7 @@ private List createSortFields() { * Argument "asc" is required. Argument "nullFirst" is optional and determined by Analyzer later * if absent. */ - private List createSortArguments(SortOption option) { + protected List createSortArguments(SortOption option) { SortOrder sortOrder = option.getSortOrder(); NullOrder nullOrder = option.getNullOrder(); ImmutableList.Builder args = ImmutableList.builder(); From 1caa63939169cfdb6c3ace3de94bb711d24acb63 Mon Sep 17 00:00:00 2001 From: Finn Date: Tue, 30 Jun 2026 09:34:25 -0700 Subject: [PATCH 21/41] Fix multi-index FGAC routing and add bypass regression tests (#5581) SQL plugin routing fix: - RestUnifiedQueryAction.isAnalyticsIndex() now splits comma-separated index names and checks each independently. Routes to analytics engine only if ALL indices are composite. Previously, the joined string 'idx1,idx2' was looked up as a single index in cluster metadata, causing multi-index queries to fall through to the legacy pipeline. Regression tests: - testPPLMultiIndexDeniedWhenSecondIndexUnauthorized - testPPLMultiIndexDeniedWithBackticksAuthorizedFirst - testPPLMultiIndexDeniedWithUnauthorizedFirst - testPPLMultiIndexAllowedWhenAllAuthorized Also fixes plugin install order (composite-engine before backends). Signed-off-by: Finnegan Carroll Signed-off-by: Finn Carroll --- integ-test/build.gradle | 2 +- .../security/AnalyticsEngineSecurityIT.java | 133 ++++++++++++++++++ .../plugin/rest/RestUnifiedQueryAction.java | 24 +++- 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 17b88a901de..1435d1d499d 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -502,11 +502,11 @@ testClusters.analyticsEngineSecurityIT { plugin(getJobSchedulerPlugin()) plugin(getArrowBasePlugin()) plugin(getArrowFlightRpcPlugin()) + plugin(getCompositeEnginePlugin()) plugin(getAnalyticsEnginePlugin()) plugin(getAnalyticsBackendLucenePlugin()) plugin(getAnalyticsBackendDatafusionPlugin()) plugin(getParquetDataFormatPlugin()) - plugin(getCompositeEnginePlugin()) plugin ":opensearch-sql-plugin" // Arrow Flight / streaming transport requirements jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' diff --git a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java index 626a7658dab..2b4d346fa0d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java @@ -393,6 +393,94 @@ public void testPPLQueryWithWildcardIndexPartialAccessDenied() throws IOExceptio assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- Multi-index comma-separated source tests (FGAC bypass regression) --- + + @Test + public void testPPLMultiIndexDeniedWhenSecondIndexUnauthorized() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + TEST_INDEX + ", " + FORBIDDEN_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexDeniedWithBackticksAuthorizedFirst() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = `" + TEST_INDEX + "`, `" + FORBIDDEN_INDEX + "` | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexDeniedWithUnauthorizedFirst() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + FORBIDDEN_INDEX + ", " + TEST_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLMultiIndexAllowedWhenAllAuthorized() throws IOException { + try { + JSONObject result = + executePPLAsUser( + "source = " + TEST_INDEX + ", " + TEST_INDEX_2 + " | fields name, age", + WILDCARD_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) when all indices are authorized", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + // --- Edge cases: malformed comma-separated source patterns --- + + @Test + public void testPPLDoubleCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser( + "source = " + TEST_INDEX + ",," + FORBIDDEN_INDEX + " | fields name, age", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLLeadingCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser("source = ," + TEST_INDEX + " | fields name, age", ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLTrailingCommaRejected() throws IOException { + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executePPLAsUser("source = " + TEST_INDEX + ", | fields name, age", ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + @Test public void testSQLQueryAllowedForAuthorizedUser() throws IOException { try { @@ -441,6 +529,51 @@ public void testSQLQueryDeniedWithSearchPermissionOnly() throws IOException { assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- SQL multi-index syntax validation --- + + @Test + public void testSQLMultiIndexCommaInFromRejected() throws IOException { + // SQL FROM "idx1,idx2" — comma inside identifier, should be rejected as syntax error + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT name, age FROM " + TEST_INDEX + "," + FORBIDDEN_INDEX + " LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testSQLMultiIndexCrossJoinRejected() throws IOException { + // SQL cross join syntax — should not bypass FGAC + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT a.name FROM " + TEST_INDEX + " a, " + FORBIDDEN_INDEX + " b LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testSQLMultiIndexJoinRejected() throws IOException { + // Explicit JOIN — should not bypass FGAC + ResponseException e = + assertThrows( + ResponseException.class, + () -> + executeSQLAsUser( + "SELECT a.name FROM " + + TEST_INDEX + + " a JOIN " + + FORBIDDEN_INDEX + + " b ON a.name = b.name LIMIT 3", + ALLOWED_USER)); + assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + } + /** Executes a PPL query via the production SQL plugin endpoint (/_plugins/_ppl). */ private JSONObject executePPLAsUser(String query, String username) throws IOException { Request request = new Request("POST", "/_plugins/_ppl"); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 84ee147d34a..2e810033a9f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -122,13 +122,35 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { try (UnifiedQueryContext context = buildParsingContext(queryType)) { return extractIndexName(query, queryType, context) .map(this::stripSchemaPrefix) - .map(this::isPluggableDataformatIndex) + .map(this::allIndicesArePluggableDataformat) .orElse(false); } catch (Exception e) { return false; } } + /** + * Checks if all indices in a (possibly comma-separated) index expression are pluggable + * dataformat. For multi-index queries (source=idx1,idx2), each index is checked independently. + * Returns true only if every index is composite — mixed or all-lucene returns false. + */ + private boolean allIndicesArePluggableDataformat(String indexExpression) { + String[] indices = indexExpression.split(","); + if (indices.length == 0) { + return false; + } + for (String idx : indices) { + String trimmed = idx.trim(); + if (trimmed.isEmpty()) { + continue; + } + if (!isPluggableDataformatIndex(trimmed)) { + return false; + } + } + return true; + } + private static boolean isSystemCatalog(String name) { return SystemIndexUtils.isSystemIndex(name) || SystemIndexUtils.DATASOURCES_TABLE_NAME.equals(name); From cfd4b6294fcc0c9802e911716d94fa0991364e54 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Tue, 30 Jun 2026 11:28:26 -0700 Subject: [PATCH 22/41] Gate analytics engine incompatible IT tests with capability matrix (#5585) * test(integ-test): gate analytics-engine excludes via @RequiresCapability Migrate the analytics-engine IT exclusions to method/class-level @RequiresCapability annotations so each test self-skips when the analytics engine is active, instead of relying on build.gradle excludes. Add coarse capabilities (backend: vector/geopoint/identifier, untyped NULL literal, filtered aggregate; frontend: response format, pagination/cursor, prepared statement, legacy method query, query error, explain format, function type compat) and reuse existing capabilities where they fit. SQLCorrectnessIT is outside the SQLIntegTestCase hierarchy, so it calls BackendCapabilities.requireCapability directly. Signed-off-by: Chen Dai * test(integ-test): skip index cleanup when client is null Signed-off-by: Chen Dai --------- Signed-off-by: Chen Dai --- .../sql/legacy/CsvFormatResponseIT.java | 3 + .../org/opensearch/sql/legacy/CursorIT.java | 3 + .../org/opensearch/sql/legacy/JdbcTestIT.java | 6 ++ .../sql/legacy/JoinAliasWriterRuleIT.java | 16 ++++ .../sql/legacy/MalformedQueryIT.java | 4 + .../opensearch/sql/legacy/MethodQueryIT.java | 8 ++ .../sql/legacy/ObjectFieldSelectIT.java | 3 + .../sql/legacy/PointInTimeLeakIT.java | 3 + .../sql/legacy/PrettyFormatResponseIT.java | 16 ++++ .../sql/legacy/SQLIntegTestCase.java | 5 ++ .../sql/legacy/SqlLegacyEngineSanityIT.java | 3 + .../sql/ppl/MathematicalFunctionIT.java | 2 + .../opensearch/sql/ppl/SystemFunctionIT.java | 2 + .../opensearch/sql/ppl/TextFunctionIT.java | 3 + .../org/opensearch/sql/sql/AggregationIT.java | 21 ++++++ .../sql/sql/ComplexTimestampQueryIT.java | 5 ++ .../org/opensearch/sql/sql/ConditionalIT.java | 3 + .../org/opensearch/sql/sql/CsvFormatIT.java | 3 + .../opensearch/sql/sql/DateTimeFormatsIT.java | 5 ++ .../sql/sql/DateTimeFunctionIT.java | 8 ++ .../opensearch/sql/sql/ExistsPushdownIT.java | 4 + .../opensearch/sql/sql/GeopointFormatsIT.java | 4 + .../sql/sql/HighlightFunctionIT.java | 3 + .../org/opensearch/sql/sql/IdentifierIT.java | 9 +++ .../sql/sql/LegacyAPICompatibilityIT.java | 3 + .../org/opensearch/sql/sql/LikeQueryIT.java | 3 + .../java/org/opensearch/sql/sql/MatchIT.java | 7 ++ .../sql/sql/MathematicalFunctionIT.java | 3 + .../org/opensearch/sql/sql/MultiMatchIT.java | 5 ++ .../java/org/opensearch/sql/sql/NestedIT.java | 3 + .../org/opensearch/sql/sql/NullLiteralIT.java | 5 ++ .../sql/sql/PaginationBlackboxIT.java | 4 + .../sql/sql/PaginationFallbackIT.java | 3 + .../sql/sql/PaginationFilterIT.java | 4 + .../org/opensearch/sql/sql/PaginationIT.java | 3 + .../sql/sql/PaginationWindowIT.java | 3 + .../sql/sql/PreparedStatementIT.java | 4 + .../org/opensearch/sql/sql/QueryStringIT.java | 3 + .../opensearch/sql/sql/QueryValidationIT.java | 3 + .../org/opensearch/sql/sql/RawFormatIT.java | 3 + .../opensearch/sql/sql/SQLCorrectnessIT.java | 4 + .../org/opensearch/sql/sql/ScoreQueryIT.java | 3 + .../sql/sql/SimpleQueryStringIT.java | 3 + .../sql/sql/StandalonePaginationIT.java | 3 + .../opensearch/sql/sql/SystemFunctionIT.java | 5 ++ .../opensearch/sql/sql/TextFunctionIT.java | 3 + .../sql/sql/VectorSearchExecutionIT.java | 3 + .../sql/sql/VectorSearchExplainIT.java | 4 + .../opensearch/sql/sql/VectorSearchIT.java | 3 + .../sql/sql/VectorSearchSubqueryIT.java | 3 + .../opensearch/sql/sql/WindowFunctionIT.java | 5 ++ .../org/opensearch/sql/util/Capability.java | 74 ++++++++++++++++++- 52 files changed, 313 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index b75da57c571..bd5b928aebe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -20,6 +20,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import java.io.IOException; import java.util.ArrayList; @@ -39,8 +40,10 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.executor.csv.CSVResult; +import org.opensearch.sql.util.RequiresCapability; /** Tests to cover requests with "?format=csv" parameter */ +@RequiresCapability(RESPONSE_FORMAT) public class CsvFormatResponseIT extends SQLIntegTestCase { private boolean flatOption = false; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java index 5dea06b7634..0b0a510cb3d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import static org.opensearch.sql.util.TestUtils.verifyIsV2Cursor; import java.io.IOException; @@ -27,7 +28,9 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PAGINATION_CURSOR) public class CursorIT extends SQLIntegTestCase { private static final String CURSOR = "cursor"; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 4ad88c632ba..a0d74cd5977 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -7,6 +7,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import java.io.IOException; import org.json.JSONArray; @@ -14,6 +16,7 @@ import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; public class JdbcTestIT extends SQLIntegTestCase { @@ -25,6 +28,7 @@ protected void init() throws Exception { loadIndex(Index.WEBLOG); } + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentilesQuery() { JSONObject response = executeJdbcRequest( @@ -43,6 +47,7 @@ public void testPercentilesQuery() { // https://github.com/opensearch-project/sql/issues/537 @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testSlowQuery() throws IOException { // set slow log threshold = 0s updateClusterSettings(new ClusterSetting(PERSISTENT, "plugins.sql.slowlog", "0")); @@ -89,6 +94,7 @@ public void testDivisionInQuery() { assertThat(response.getJSONArray("datarows").getJSONArray(0).getDouble(0), equalTo(16827.0)); } + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testGroupByInQuery() { JSONObject response = executeJdbcRequest( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java index 3933338f0a6..63d37b3ed6a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JoinAliasWriterRuleIT.java @@ -6,6 +6,8 @@ package org.opensearch.sql.legacy; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.sql.util.Capability.EXPLAIN_FORMAT; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import java.io.IOException; import org.junit.Ignore; @@ -13,6 +15,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.opensearch.client.ResponseException; +import org.opensearch.sql.util.RequiresCapability; /** Test cases for writing missing join table aliases. */ public class JoinAliasWriterRuleIT extends SQLIntegTestCase { @@ -26,6 +29,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void noTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -45,6 +49,7 @@ public void noTableAliasNoCommonColumns() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void oneTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -61,6 +66,7 @@ public void oneTableAliasNoCommonColumns() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void bothTableAliasNoCommonColumns() throws IOException { sameExplain( query( @@ -78,6 +84,7 @@ public void bothTableAliasNoCommonColumns() throws IOException { @Test @Ignore + @RequiresCapability(EXPLAIN_FORMAT) public void tableNamesWithTypeName() throws IOException { sameExplain( query( @@ -98,6 +105,7 @@ public void tableNamesWithTypeName() throws IOException { @Ignore @Test + @RequiresCapability(EXPLAIN_FORMAT) public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { sameExplain( query( @@ -114,6 +122,7 @@ public void tableNamesWithTypeNameExplicitTableAlias() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void actualTableNameAsAliasOnColumnFields() throws IOException { sameExplain( query( @@ -130,6 +139,7 @@ public void actualTableNameAsAliasOnColumnFields() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { sameExplain( query( @@ -150,6 +160,7 @@ public void actualTableNameAsAliasOnColumnFieldsTwo() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void columnsWithTableAliasNotAffected() throws IOException { sameExplain( query( @@ -166,6 +177,7 @@ public void columnsWithTableAliasNotAffected() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void commonColumnWithoutTableAliasDifferentTables() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Field name [firstname] is ambiguous"); @@ -179,6 +191,7 @@ public void commonColumnWithoutTableAliasDifferentTables() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Not unique table/alias: [opensearch-sql_test_index_bank]"); @@ -192,6 +205,7 @@ public void sameTablesNoAliasAndNoAliasOnColumns() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException { exception.expect(ResponseException.class); exception.expectMessage("Not unique table/alias: [opensearch-sql_test_index_bank]"); @@ -206,6 +220,7 @@ public void sameTablesNoAliasWithTableNameAsAliasOnColumns() throws IOException } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void sameTablesWithExplicitAliasOnFirst() throws IOException { sameExplain( query( @@ -221,6 +236,7 @@ public void sameTablesWithExplicitAliasOnFirst() throws IOException { } @Test + @RequiresCapability(EXPLAIN_FORMAT) public void sameTablesWithExplicitAliasOnSecond() throws IOException { sameExplain( query( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java index 84b60fdabd7..bf49f239465 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MalformedQueryIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.legacy; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; + import java.io.IOException; import java.util.Locale; import org.apache.hc.core5.http.ParseException; @@ -12,8 +14,10 @@ import org.json.JSONObject; import org.junit.Assert; import org.opensearch.client.ResponseException; +import org.opensearch.sql.util.RequiresCapability; /** Tests for clean handling of various types of invalid queries */ +@RequiresCapability(QUERY_ERROR_MESSAGE) public class MalformedQueryIT extends SQLIntegTestCase { @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index 7589304af0a..4f39b8d4efa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -7,12 +7,15 @@ import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.EXPLAIN_FORMAT; +import static org.opensearch.sql.util.Capability.LEGACY_METHOD_QUERY; import java.io.IOException; import java.util.Locale; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; /** * 定製方法查詢． @@ -41,6 +44,7 @@ protected void init() throws Exception { * @throws IOException */ @Test + @RequiresCapability(EXPLAIN_FORMAT) public void queryTest() throws IOException { final String result = explainQuery( @@ -70,6 +74,7 @@ public void queryTest() throws IOException { * @throws IOException */ @Test + @RequiresCapability(EXPLAIN_FORMAT) public void matchQueryTest() throws IOException { final String result = explainQuery( @@ -153,6 +158,7 @@ public void scoreQueryTest() throws IOException { } @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void regexpQueryTest() throws IOException { final String result = explainQuery( @@ -168,6 +174,7 @@ public void regexpQueryTest() throws IOException { } @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void negativeRegexpQueryTest() throws IOException { final String result = explainQuery( @@ -198,6 +205,7 @@ public void negativeRegexpQueryTest() throws IOException { * @throws IOException */ @Test + @RequiresCapability(LEGACY_METHOD_QUERY) public void wildcardQueryTest() throws IOException { final String result = explainQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java index aadd79469db..4f64cdb704d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED; +import static org.opensearch.sql.util.Capability.STRUCT_PARENT_FIELD; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -16,11 +17,13 @@ import org.junit.Test; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** * Integration test for OpenSearch object field (and nested field). This class is focused on simple * SELECT-FROM query to ensure right column number and value is returned. */ +@RequiresCapability(STRUCT_PARENT_FIELD) public class ObjectFieldSelectIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java index ab0f196ce33..9c4513a973a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PointInTimeLeakIT.java @@ -7,6 +7,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import org.json.JSONArray; @@ -17,12 +18,14 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** * Integration test verifying PIT contexts are created only when needed and properly cleaned up. * * @see Issue #5002 */ +@RequiresCapability(PAGINATION_CURSOR) public class PointInTimeLeakIT extends SQLIntegTestCase { private static final String TEST_INDEX = "test-logs-2025.01.01"; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index b4218db6abf..7e4950a5fab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -10,6 +10,9 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.opensearch.sql.util.Capability.DYNAMIC_STRING_NO_KEYWORD; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import com.google.common.collect.Sets; import java.io.IOException; @@ -27,6 +30,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.sql.util.RequiresCapability; /** * @@ -97,6 +101,7 @@ protected Request getSqlRequest(String request, boolean explain) { // This is testing a deprecated feature @Test + @RequiresCapability(RESPONSE_FORMAT) public void wrongIndexType() throws IOException { String type = "wrongType"; try { @@ -167,18 +172,21 @@ public void selectScore() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithoutFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s", regularFields, fields("message", "comment")); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s e, e.message m", regularFields, messageFields); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT * FROM %s e, e.message m, e.comment c", @@ -186,12 +194,14 @@ public void selectAllFromNestedWithMultipleFieldsInFrom() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectAllNestedFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT m.* FROM %s e, e.message m", messageFields); } @Test + @RequiresCapability(NESTED_FIELDS) public void selectSpecificRegularFieldAndAllFromNestedWithFieldInFrom() throws IOException { assertNestedFieldQueryResultContainsColumnsAndData( "SELECT e.someField, m.* FROM %s e, e.message m", fields("someField"), messageFields); @@ -218,6 +228,7 @@ private Set fields(String... fieldNames) { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectNestedFields() throws IOException { JSONObject response = executeQuery( @@ -237,6 +248,7 @@ public void selectNestedFields() throws IOException { } @Test + @RequiresCapability(NESTED_FIELDS) public void selectNestedFieldWithWildcard() throws IOException { JSONObject response = executeQuery( @@ -456,6 +468,7 @@ public void aggregationFunctionInHaving() throws IOException { // TEST_INDEX_ACCOUNT); // } @Test + @RequiresCapability(RESPONSE_FORMAT) public void fieldsWithAlias() throws IOException { JSONObject response = executeQuery( @@ -506,6 +519,7 @@ public void joinQuery() throws IOException { } @Test + @RequiresCapability(RESPONSE_FORMAT) public void joinQueryWithAlias() throws IOException { JSONObject response = executeQuery( @@ -565,6 +579,7 @@ public void joinQuerySelectOnlyOnOneTable() throws Exception { } @Test + @RequiresCapability(DYNAMIC_STRING_NO_KEYWORD) public void fieldOrder() throws IOException { final String[] expectedFields = {"age", "firstname", "address", "gender", "email"}; @@ -574,6 +589,7 @@ public void fieldOrder() throws IOException { } @Test + @RequiresCapability(DYNAMIC_STRING_NO_KEYWORD) public void fieldOrderOther() throws IOException { final String[] expectedFields = {"email", "firstname", "age", "gender", "address"}; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index ef05923b060..05a670de393 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -139,6 +139,11 @@ public static void dumpCoverage() { */ @AfterClass public static void cleanUpIndices() throws IOException { + // No client when every test in the class was skipped (e.g. @RequiresCapability on the AE + // route). + if (client() == null) { + return; + } if (System.getProperty("tests.rest.bwcsuite") == null) { wipeAllOpenSearchIndices(); wipeAllClusterSettings(); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java index b2d22808a49..1c92cee74a9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SqlLegacyEngineSanityIT.java @@ -7,12 +7,14 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE; +import static org.opensearch.sql.util.Capability.TEXT_FIELD_EXACT_MATCH; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; +import org.opensearch.sql.util.RequiresCapability; /** * Sanity tests for the legacy SQL engine. Many legacy integration tests (JoinIT, SubqueryIT, @@ -38,6 +40,7 @@ public void testInnerJoinFallback() throws IOException { } @Test + @RequiresCapability(TEXT_FIELD_EXACT_MATCH) public void testLeftJoinFallback() throws IOException { JSONObject result = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index 6df60f68a7b..42f69010270 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.Capability.RAND_SEED_UNSUPPORTED; import static org.opensearch.sql.util.MatcherUtils.closeTo; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -71,6 +72,7 @@ public void testDivideFunction() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testCeil() throws IOException { JSONObject result = executeQuery(String.format("source=%s | eval f = ceil(age) | fields f", TEST_INDEX_BANK)); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index 276ab01da4c..00eb26b43dd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; import static org.opensearch.sql.util.Capability.SCALED_FLOAT_TYPE; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -28,6 +29,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void typeof_sql_types() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java index 7fe360d8844..bc1b60eebac 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -14,6 +15,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.sql.util.RequiresCapability; public class TextFunctionIT extends PPLIntegTestCase { @Override @@ -78,6 +80,7 @@ void verifyRegexQuery(String pattern, Boolean outputRow1, Boolean outputRow2, Bo } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testRegexp() throws IOException { if (isCalciteEnabled()) { verifyRegexQuery("hello", true, false, true); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java index 0966c63e0d6..66122db4aa9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java @@ -7,6 +7,9 @@ import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FILTERED_AGGREGATE; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verify; @@ -26,6 +29,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class AggregationIT extends SQLIntegTestCase { @Override @@ -37,6 +41,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredAggregatePushDown() throws IOException { JSONObject response = executeQuery("SELECT COUNT(*) FILTER(WHERE age > 35) FROM " + TEST_INDEX_BANK); @@ -45,6 +50,7 @@ public void testFilteredAggregatePushDown() throws IOException { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredAggregateNotPushDown() throws IOException { JSONObject response = executeQuery( @@ -221,6 +227,7 @@ public void testPushDownAggregationOnNullNumericValuesReturnsNull() throws IOExc } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateTimeValuesFromTableReturnsNull() throws IOException { var response = executeQuery( @@ -236,6 +243,7 @@ public void testPushDownAggregationOnNullDateTimeValuesFromTableReturnsNull() th } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateValuesReturnsNull() throws IOException { var response = executeQuery( @@ -252,6 +260,7 @@ public void testPushDownAggregationOnNullDateValuesReturnsNull() throws IOExcept } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullTimeValuesReturnsNull() throws IOException { var response = executeQuery( @@ -268,6 +277,7 @@ public void testPushDownAggregationOnNullTimeValuesReturnsNull() throws IOExcept } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullTimeStampValuesReturnsNull() throws IOException { var response = executeQuery( @@ -284,6 +294,7 @@ public void testPushDownAggregationOnNullTimeStampValuesReturnsNull() throws IOE } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testPushDownAggregationOnNullDateTimeValuesReturnsNull() throws IOException { var response = executeQuery( @@ -473,6 +484,7 @@ public void testMaxDatePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDatePushedDown() throws IOException { var response = executeQuery(String.format("SELECT avg(date0)" + " from %s", TEST_INDEX_CALCS)); verifySchema(response, schema("avg(date0)", null, "date")); @@ -500,6 +512,7 @@ public void testMaxDateTimePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateTimePushedDown() throws IOException { var response = executeQuery( @@ -524,6 +537,7 @@ public void testMaxTimePushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimePushedDown() throws IOException { var response = executeQuery(String.format("SELECT avg(time1)" + " from %s", TEST_INDEX_CALCS)); verifySchema(response, schema("avg(time1)", null, "time")); @@ -551,6 +565,7 @@ public void testMaxTimeStampPushedDown() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeStampPushedDown() throws IOException { var response = executeQuery( @@ -581,6 +596,7 @@ public void testMaxDateInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateInMemory() throws IOException { var response = executeQuery( @@ -625,6 +641,7 @@ public void testMaxDateTimeInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgDateTimeInMemory() throws IOException { var response = executeQuery( @@ -662,6 +679,7 @@ public void testMaxTimeInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeInMemory() throws IOException { var response = executeQuery( @@ -702,6 +720,7 @@ public void testMaxTimeStampInMemory() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testAvgTimeStampInMemory() throws IOException { var response = executeQuery( @@ -725,6 +744,7 @@ public void testPercentilePushedDown() throws IOException { } @Test + @RequiresCapability(FILTERED_AGGREGATE) public void testFilteredPercentilePushDown() throws IOException { JSONObject response = executeQuery( @@ -735,6 +755,7 @@ public void testFilteredPercentilePushDown() throws IOException { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileGroupByPushDown() throws IOException { var response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java index c0eb800c103..bb33ab891a6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ComplexTimestampQueryIT.java @@ -7,6 +7,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_TIME_NESTED; +import static org.opensearch.sql.util.Capability.ID_METADATA; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import java.io.IOException; import java.util.Locale; @@ -16,6 +18,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class ComplexTimestampQueryIT extends SQLIntegTestCase { @Override @@ -26,6 +29,7 @@ protected void init() throws Exception { /** See: 3159 */ @Test + @RequiresCapability(ID_METADATA) public void joinWithTimestampFieldsSchema() throws IOException { String query = String.format( @@ -100,6 +104,7 @@ public void nonJoinTimestampComparison() throws IOException { /** See: 1545 */ @Test + @RequiresCapability(NESTED_FIELDS) public void selectDatetimeWithNested() throws IOException { String query = String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java index 05e6af5acbc..88c1bd54ce0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.hitAny; import static org.opensearch.sql.util.MatcherUtils.kvInt; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -32,6 +33,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchHits; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class ConditionalIT extends SQLIntegTestCase { @@ -51,6 +53,7 @@ public void ifnullShouldPassJDBC() throws IOException { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void ifnullWithNullInputTest() { JSONObject response = new JSONObject( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java index d400ad646fc..34850e7e5bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; import static org.opensearch.sql.protocol.response.format.CsvResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; @@ -16,7 +17,9 @@ import org.opensearch.client.Response; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(RESPONSE_FORMAT) public class CsvFormatIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index a24775a9755..1e44a62fa24 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -23,6 +24,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class DateTimeFormatsIT extends SQLIntegTestCase { @@ -34,6 +36,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testReadingDateFormats() throws IOException { String query = String.format( @@ -50,6 +53,7 @@ public void testReadingDateFormats() throws IOException { } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateFormatsWithOr() throws IOException { String query = String.format("SELECT yyyy-MM-dd_OR_epoch_millis FROM %s", TEST_INDEX_DATE_FORMATS); @@ -153,6 +157,7 @@ public void testDateNanosWithFormats() { @Test @SneakyThrows + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateNanosWithFunctions() { // in memory funcs String query = diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index e5132eb02f1..2435256fcb5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -8,6 +8,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.DATETIME_FORMAT_RENDERING; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -31,6 +33,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class DateTimeFunctionIT extends SQLIntegTestCase { @@ -1170,6 +1173,7 @@ void verifyDateFormat(String date, String type, String format, String formatted) } @Test + @RequiresCapability(DATETIME_FORMAT_RENDERING) public void testDateFormat() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; String timestampFormat = @@ -1397,6 +1401,7 @@ protected JSONObject executeQuery(String query) throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testDateStringAsTimestamp() throws IOException { JSONObject result = executeQuery("select {timestamp '2025-07-10'} as t"); verifySchema(result, schema("{timestamp '2025-07-10'}", "t", "timestamp")); @@ -1404,6 +1409,7 @@ public void testDateStringAsTimestamp() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testTimestampBracket() throws IOException { JSONObject result = executeQuery("select {timestamp '2020-09-16 17:30:00'}"); verifySchema(result, schema("{timestamp '2020-09-16 17:30:00'}", null, "timestamp")); @@ -1423,6 +1429,7 @@ public void testTimestampBracket() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testTimeBracket() throws IOException { JSONObject result = executeQuery("select {time '17:30:00'}"); verifySchema(result, schema("{time '17:30:00'}", null, "time")); @@ -1461,6 +1468,7 @@ private void compareBrackets(String query1, String query2, String timestamp) thr } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testBracketedEquivalent() throws IOException { compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00"); compareBrackets("timestamp", "ts", "2020-09-16 17:30:00"); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java index 08ceb8c35f9..c3a6a4c5ea3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ExistsPushdownIT.java @@ -5,12 +5,15 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.LUCENE_PUSHDOWN_EXPLAIN; + import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Explain-plan integration tests asserting that {@code IS NOT NULL} / {@code IS NULL} predicates @@ -22,6 +25,7 @@ * with a single {@code must_not[exists]} child for {@code IS NULL}. This matches what downstream * tooling, serverless / AOSS, and the Calcite path already produce. */ +@RequiresCapability(LUCENE_PUSHDOWN_EXPLAIN) public class ExistsPushdownIT extends SQLIntegTestCase { // Anchored on the surrounding `sourceBuilder=...`, `pitId=` tokens in OpenSearchRequest's diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java index 68dc9a18f34..fc79618fc07 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/GeopointFormatsIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.GEOPOINT_TYPE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +18,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class GeopointFormatsIT extends SQLIntegTestCase { @@ -26,6 +28,7 @@ public void init() throws Exception { } @Test + @RequiresCapability(GEOPOINT_TYPE) public void testReadingGeopoints() throws IOException { String query = String.format("SELECT point FROM %s LIMIT 5", Index.GEOPOINTS.getName()); JSONObject result = executeJdbcRequest(query); @@ -41,6 +44,7 @@ public void testReadingGeopoints() throws IOException { public static final double TOLERANCE = 1E-5; + @RequiresCapability(GEOPOINT_TYPE) public void testReadingGeoHash() throws IOException { String query = String.format("SELECT point FROM %s WHERE _id='6'", Index.GEOPOINTS.getName()); JSONObject result = executeJdbcRequest(query); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java index d0f890526b7..245b87f0d41 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/HighlightFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,7 +18,9 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public class HighlightFunctionIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java index ce866dc3bdf..50ddc0324d2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/IdentifierIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.IDENTIFIER_RESOLUTION; +import static org.opensearch.sql.util.Capability.ID_METADATA; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -17,6 +19,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** Integration tests for identifiers including index and field name symbol. */ public class IdentifierIT extends SQLIntegTestCase { @@ -56,6 +59,7 @@ public void testSpecialFieldName() throws IOException { } @Test + @RequiresCapability(IDENTIFIER_RESOLUTION) public void testMultipleQueriesWithSpecialIndexNames() throws IOException { createIndexWithOneDoc("test.one", "test.two"); queryAndAssertTheDoc("SELECT * FROM test.one"); @@ -63,6 +67,7 @@ public void testMultipleQueriesWithSpecialIndexNames() throws IOException { } @Test + @RequiresCapability(IDENTIFIER_RESOLUTION) public void testDoubleUnderscoreIdentifierTest() throws IOException { new Index("test.twounderscores").addDoc("{\"__age\": 30}"); final JSONObject result = @@ -73,6 +78,7 @@ public void testDoubleUnderscoreIdentifierTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierTest() throws IOException { // create an index, but the contents doesn't matter String id = "12345"; @@ -94,6 +100,7 @@ public void testMetafieldIdentifierTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierRoutingSelectTest() throws IOException { // create an index, but the contents doesn't really matter String index = "test.routing_select"; @@ -132,6 +139,7 @@ public void testMetafieldIdentifierRoutingSelectTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierRoutingFilterTest() throws IOException { // create an index, but the contents doesn't really matter String index = "test.routing_filter"; @@ -172,6 +180,7 @@ public void testMetafieldIdentifierRoutingFilterTest() throws IOException { } @Test + @RequiresCapability(ID_METADATA) public void testMetafieldIdentifierWithAliasTest() throws IOException { // create an index, but the contents doesn't matter String id = "99999"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java index 155d9002aed..de804555587 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/LegacyAPICompatibilityIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.plugin.rest.RestQuerySettingsAction.SETTINGS_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import org.json.JSONObject; @@ -17,6 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; /** For backward compatibility, check if legacy API endpoints are accessible. */ public class LegacyAPICompatibilityIT extends SQLIntegTestCase { @@ -50,6 +52,7 @@ public void explain() { } @Test + @RequiresCapability(PAGINATION_CURSOR) public void closeCursor() throws IOException { String sql = StringUtils.format("SELECT firstname FROM %s WHERE balance > 100", TEST_INDEX_ACCOUNT); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java index 118dd9849b4..6e5451192d8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.Capability.TEXT_KEYWORD_PUSHDOWN_REWRITE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class LikeQueryIT extends SQLIntegTestCase { @Override @@ -145,6 +147,7 @@ public void test_like_on_text_field_with_greater_than_one_word() throws IOExcept } @Test + @RequiresCapability(TEXT_KEYWORD_PUSHDOWN_REWRITE) public void test_convert_field_text_to_keyword() throws IOException { String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE '*'"; String result = explainQuery(query); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java index 5bde838e190..68f6a0070f7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java @@ -7,6 +7,8 @@ import static org.hamcrest.Matchers.containsString; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -20,6 +22,7 @@ import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; import org.opensearch.sql.legacy.utils.StringUtils; +import org.opensearch.sql.util.RequiresCapability; public class MatchIT extends SQLIntegTestCase { @Override @@ -46,6 +49,7 @@ public void match_in_having() throws IOException { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match(invalid, 'Bates')", TEST_INDEX_ACCOUNT); @@ -61,6 +65,7 @@ public void missing_field_test() { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_quoted_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match('invalid', 'Bates')", TEST_INDEX_ACCOUNT); @@ -76,6 +81,7 @@ public void missing_quoted_field_test() { } @Test + @RequiresCapability(QUERY_ERROR_MESSAGE) public void missing_backtick_field_test() { String query = StringUtils.format("SELECT * FROM %s WHERE match(`invalid`, 'Bates')", TEST_INDEX_ACCOUNT); @@ -182,6 +188,7 @@ public void match_alternate_syntaxes_return_the_same_results() throws IOExceptio } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void matchPhraseQueryTest() throws IOException { final String result = explainQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index b7f2ced5fbd..d345643a89e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -22,6 +23,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class MathematicalFunctionIT extends SQLIntegTestCase { @@ -40,6 +42,7 @@ public void testPI() throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testCeil() throws IOException { JSONObject result = executeQuery("select ceil(0)"); verifySchema(result, schema("ceil(0)", null, "long")); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java index 0bc091b0d20..1bb40b41c8b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class MultiMatchIT extends SQLIntegTestCase { @Override @@ -53,6 +55,7 @@ public void test_all_params() { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void verify_wildcard_test() { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE multi_match(['Tags'], 'taste')"; JSONObject result1 = executeJdbcRequest(query1); @@ -141,6 +144,7 @@ public void test_all_params_multimatchquery_alternate_parameter_syntax() { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void multi_match_alternate_syntax() throws IOException { String query = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE CreationDate = multi_match('2014-01-22');"; @@ -149,6 +153,7 @@ public void multi_match_alternate_syntax() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void multimatch_alternate_syntax() throws IOException { String query = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE CreationDate = multimatch('2014-01-22');"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 18d93dbb2a0..3bf6ba72a78 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -10,6 +10,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_NULLS; +import static org.opensearch.sql.util.Capability.NESTED_FIELDS; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -27,7 +28,9 @@ import org.junit.Test; import org.junit.jupiter.api.Disabled; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(NESTED_FIELDS) public class NestedIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java index f885b6d4e0d..854cae2837d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NullLiteralIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -13,6 +14,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * This manual IT for NULL literal cannot be replaced with comparison test because other database @@ -22,6 +24,7 @@ public class NullLiteralIT extends SQLIntegTestCase { @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralSchema() { verifySchema( query("SELECT NULL, ABS(NULL), 1 + NULL, NULL + 1.0"), @@ -32,6 +35,7 @@ public void testNullLiteralSchema() { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralInOperator() { verifyDataRows(query("SELECT NULL = NULL, NULL AND TRUE"), rows(null, null)); } @@ -42,6 +46,7 @@ public void testNullLiteralInFunction() { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void testNullLiteralInInterval() { verifyDataRows( query("SELECT INTERVAL NULL DAY, INTERVAL 60 * 60 * 24 * (NULL - FLOOR(NULL)) SECOND"), diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java index 84289d8f57e..02a627c72c4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationBlackboxIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; + import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import java.io.IOException; @@ -17,10 +19,12 @@ import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; // This class has only one test case, because it is parametrized and takes significant time @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class PaginationBlackboxIT extends SQLIntegTestCase { private final Index index; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java index dfb0bb2080e..675bbde2c7a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java @@ -7,14 +7,17 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import static org.opensearch.sql.util.TestUtils.verifyIsV1Cursor; import static org.opensearch.sql.util.TestUtils.verifyIsV2Cursor; import java.io.IOException; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationFallbackIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java index 9a945ec86f5..221e67c910b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationFilterIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; + import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import java.io.IOException; @@ -19,6 +21,7 @@ import org.junit.jupiter.api.DisplayNameGenerator; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Test pagination with `WHERE` clause using a parametrized test. See constructor {@link @@ -26,6 +29,7 @@ * #STATEMENT_TO_NUM_OF_PAGES} to see how these parameters are generated. */ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class PaginationFilterIT extends SQLIntegTestCase { /** diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java index 938a5d1fcd5..c94a81cd50a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import lombok.SneakyThrows; @@ -21,8 +22,10 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.TestUtils; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java index 4c387e2c171..a0d6ee5b318 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PaginationWindowIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import java.io.IOException; import java.util.ArrayList; @@ -15,7 +16,9 @@ import org.junit.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PAGINATION_CURSOR) public class PaginationWindowIT extends SQLIntegTestCase { @Override public void init() throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java index 8200f64b668..0a156c9dba5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/PreparedStatementIT.java @@ -5,11 +5,15 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PREPARED_STATEMENT; + import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(PREPARED_STATEMENT) public class PreparedStatementIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java index 3d4e08b4cdc..2e42413b170 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java @@ -6,11 +6,13 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class QueryStringIT extends SQLIntegTestCase { @Override @@ -70,6 +72,7 @@ public void all_params_test() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void wildcard_test() throws IOException { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE query_string(['Tags'], 'taste')"; JSONObject result1 = executeJdbcRequest(query1); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 2cdfc67d228..9c4df756792 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -8,6 +8,7 @@ import static org.hamcrest.Matchers.is; import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.QUERY_ERROR_MESSAGE; import static org.opensearch.sql.util.MatcherUtils.featureValueOf; import java.io.IOException; @@ -22,11 +23,13 @@ import org.opensearch.client.ResponseException; import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * The query validation IT only covers test for error cases that not doable in comparison test. For * all other tests, comparison test should be favored over manual written test like this. */ +@RequiresCapability(QUERY_ERROR_MESSAGE) public class QueryValidationIT extends SQLIntegTestCase { @Rule public final ExpectedException exceptionRule = ExpectedException.none(); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java index 0f085a1cdeb..33f891b22e6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_RAW_SANITIZE; import static org.opensearch.sql.protocol.response.format.RawResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.RESPONSE_FORMAT; import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; @@ -17,7 +18,9 @@ import org.opensearch.client.Response; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(RESPONSE_FORMAT) public class RawFormatIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java index 6056a1c4168..72941ee9e09 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SQLCorrectnessIT.java @@ -5,6 +5,9 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.BackendCapabilities.requireCapability; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; + import com.google.common.io.Resources; import java.io.IOException; import java.nio.file.Files; @@ -27,6 +30,7 @@ protected void init() throws Exception { @Test public void runAllTests() throws Exception { + requireCapability(UNTYPED_NULL_LITERAL); verifyQueries(EXPR_TEST_DIR, expr -> "SELECT " + expr); verifyQueries(QUERY_TEST_DIR, Function.identity()); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java index a1f71dcf6c4..4244daf9f38 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ScoreQueryIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataAddressRows; @@ -18,7 +19,9 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; +@RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public class ScoreQueryIT extends SQLIntegTestCase { @Override protected void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java index 8742dedbc70..05bfa483b6f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.Capability.FULLTEXT_RELEVANCE_FUNC; import java.io.IOException; import org.json.JSONObject; @@ -14,6 +15,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class SimpleQueryStringIT extends SQLIntegTestCase { @Override @@ -63,6 +65,7 @@ public void test_all_params() throws IOException { } @Test + @RequiresCapability(FULLTEXT_RELEVANCE_FUNC) public void verify_wildcard_test() throws IOException { String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + " WHERE simple_query_string(['Tags'], 'taste')"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java index 01daded897d..4dbc348cb8a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/StandalonePaginationIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.executor.QueryType.SQL; import static org.opensearch.sql.ppl.StandaloneIT.getDataSourceMetadataStorage; import static org.opensearch.sql.ppl.StandaloneIT.getDataSourceUserRoleHelper; +import static org.opensearch.sql.util.Capability.PAGINATION_CURSOR; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -51,9 +52,11 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.sql.util.InternalRestHighLevelClient; +import org.opensearch.sql.util.RequiresCapability; import org.opensearch.sql.util.StandaloneModule; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +@RequiresCapability(PAGINATION_CURSOR) public class StandalonePaginationIT extends SQLIntegTestCase { private QueryService queryService; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 7129d058c00..b5504b9ecf1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -7,12 +7,15 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; +import static org.opensearch.sql.util.Capability.UNTYPED_NULL_LITERAL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class SystemFunctionIT extends SQLIntegTestCase { @@ -23,6 +26,7 @@ protected void init() throws Exception { } @Test + @RequiresCapability(UNTYPED_NULL_LITERAL) public void typeof_sql_types() { JSONObject response = executeJdbcRequest( @@ -40,6 +44,7 @@ public void typeof_sql_types() { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void typeof_opensearch_types() { JSONObject response = executeJdbcRequest( diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java index 314132fed0a..a793be2debf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.Capability.FUNCTION_TYPE_COMPAT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -20,6 +21,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; public class TextFunctionIT extends SQLIntegTestCase { @@ -47,6 +49,7 @@ void verifyQueryWithNullOutput(String query, String type) throws IOException { } @Test + @RequiresCapability(FUNCTION_TYPE_COMPAT) public void testRegexp() throws IOException { verifyQuery("'a' regexp 'b'", "integer", 0); verifyQuery("'a' regexp '.*'", "integer", 1); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java index 36e78567d54..401732fff56 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExecutionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; import static org.opensearch.sql.util.TestUtils.isIndexExist; import static org.opensearch.sql.util.TestUtils.performRequest; @@ -20,6 +21,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.util.RequiresCapability; /** * Happy-path execution tests for the vectorSearch() SQL table function. These tests run an actual @@ -31,6 +33,7 @@ * is absent. Run locally against a cluster that has opensearch-knn installed. Provisioning k-NN in * CI is a separate follow-up. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchExecutionIT extends SQLIntegTestCase { private static final String TEST_INDEX = "vector_exec_test"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java index 8719189b13a..b9efa7e0fb4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchExplainIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; + import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -15,12 +17,14 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Explain-plan integration tests for vectorSearch SQL table function. These tests verify DSL * push-down shape via _explain. They do NOT require the k-NN plugin since _explain only parses and * plans the query without executing it against a knn index. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchExplainIT extends SQLIntegTestCase { // Matches WrapperQueryBuilder's base64 payload in explain JSON. The explain output escapes diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 8ae3167b40b..093ea58e489 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import java.io.IOException; import org.junit.Assume; @@ -15,12 +16,14 @@ import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for vectorSearch SQL table function — validation and error paths. These tests * verify that invalid inputs are rejected with clear error messages. Explain-plan DSL shape tests * live in {@link VectorSearchExplainIT}. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java index 04346f87a76..2f894dbaed2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchSubqueryIT.java @@ -6,12 +6,14 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.sql.util.Capability.VECTOR_SEARCH; import java.io.IOException; import org.junit.Test; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; /** * Integration tests for vectorSearch() used inside subqueries. Locks in the rejection of outer @@ -22,6 +24,7 @@ *

Uses _explain-only plus error-path queries, so the k-NN plugin is not required — the planner * validation fires during planning, before any k-NN execution. */ +@RequiresCapability(VECTOR_SEARCH) public class VectorSearchSubqueryIT extends SQLIntegTestCase { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java index 95c1f7433dc..ad248b98d62 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/WindowFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.util.Capability.PERCENTILE_APPROXIMATE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; @@ -13,6 +14,7 @@ import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; +import org.opensearch.sql.util.RequiresCapability; public class WindowFunctionIT extends SQLIntegTestCase { @@ -125,6 +127,7 @@ public void testDistinctCountPartition() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileOverNull() { JSONObject response = new JSONObject( @@ -145,6 +148,7 @@ public void testPercentileOverNull() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentileOver() { JSONObject response = new JSONObject( @@ -165,6 +169,7 @@ public void testPercentileOver() { } @Test + @RequiresCapability(PERCENTILE_APPROXIMATE) public void testPercentilePartition() { JSONObject response = new JSONObject( diff --git a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java index 3bac06e7fca..1d4067f1414 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/Capability.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/Capability.java @@ -499,7 +499,79 @@ public enum Capability { STREAMSTATS_SORT_NOT_HONORED( "streamstats computes its window over the backend scan order on the analytics-engine route," + " ignoring a preceding | sort (the OVER clause has no explicit ORDER BY), so the window" - + " values diverge from the v2/Calcite path which honors the sort."); + + " values diverge from the v2/Calcite path which honors the sort."), + + // Capabilities below were migrated from build.gradle analytics-engine excludes (commit 1ccf431). + // BACKEND layer: divergence rooted in the AE/DataFusion execution + composite/parquet storage. + + /** BACKEND: kNN/vector search has no analytics-engine (DataFusion) backend. */ + VECTOR_SEARCH( + "Vector/kNN search is unsupported on the analytics-engine route: DataFusion has no kNN" + + " backend."), + + /** BACKEND: geo_point fields are not held by the composite/parquet store. */ + GEOPOINT_TYPE( + "geo_point fields are unsupported on the analytics-engine route: the composite/parquet store" + + " does not hold the geo_point type."), + + /** + * BACKEND: dotted index names and underscore-prefixed identifiers don't resolve on the AE route. + */ + IDENTIFIER_RESOLUTION( + "Dotted index names and underscore-prefixed field identifiers don't resolve on the" + + " analytics-engine route."), + + // FRONTEND layer: divergence rooted in the Calcite parser/planner replacing the V2 engine. + + /** FRONTEND: CSV/raw/pretty response formatters are not wired in the Calcite path. */ + RESPONSE_FORMAT( + "CSV/raw/pretty response formats are not produced by the Calcite path used by the" + + " analytics-engine route."), + + /** FRONTEND: stateful pagination/cursor/PIT is not implemented in the Calcite path. */ + PAGINATION_CURSOR( + "Pagination, cursor, and point-in-time are unsupported on the analytics-engine route: the" + + " Calcite path has no stateful cursor."), + + /** FRONTEND: JDBC prepared statements are not implemented in the Calcite path. */ + PREPARED_STATEMENT( + "Prepared statements are unsupported on the analytics-engine route (Calcite path)."), + + /** + * FRONTEND: legacy method-query syntax (regexp_query/wildcard_query) is not in the Calcite + * grammar. + */ + LEGACY_METHOD_QUERY( + "Legacy method-query syntax (regexp_query/wildcard_query/query/matchquery) is not in the" + + " Calcite grammar used by the analytics-engine route."), + + /** FRONTEND: error/validation message text differs under the Calcite path. */ + QUERY_ERROR_MESSAGE( + "Query validation and error-message text differ on the analytics-engine route (Calcite" + + " produces different wording for the same semantic error)."), + + /** FRONTEND: explain output is a Calcite plan, not the V2 OpenSearch DSL text. */ + EXPLAIN_FORMAT( + "Explain output differs on the analytics-engine route: the Calcite path emits a different" + + " plan shape than the V2 OpenSearch DSL text the test asserts."), + + /** + * FRONTEND: Calcite function return types/signatures differ from V2 (CEIL, REGEXP, typeof, AVG). + */ + FUNCTION_TYPE_COMPAT( + "Function return types and signatures differ on the analytics-engine route: Calcite uses" + + " standard SQL types (e.g. CEIL->double, REGEXP->boolean, typeof ANSI names, AVG" + + " rejects temporal) where V2 used OpenSearch-specific behavior."), + + /** BACKEND: untyped NULL literal in a no-FROM query can't be serialized to Substrait. */ + UNTYPED_NULL_LITERAL( + "An untyped NULL literal in a no-FROM query (SELECT NULL, NULL in operators/intervals," + + " typeof(NULL)) can't be serialized to Substrait on the analytics-engine route."), + + /** BACKEND: FILTER(WHERE) on aggregates can't be executed via Substrait streaming. */ + FILTERED_AGGREGATE( + "FILTER(WHERE) on aggregates can't be executed on the analytics-engine route: the Substrait" + + " streaming path doesn't support filtered aggregates."); private final String reason; From e4f29d53503403ac5ba4de5949e3667687f25d58 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 30 Jun 2026 22:53:46 -0700 Subject: [PATCH 23/41] Suggest fields for 'field not found' errors (#5402) Signed-off-by: Simeon Widdis --- .../sql/common/utils/StringUtils.java | 81 +++++++++++++++++++ .../sql/ast/expression/QualifiedName.java | 23 +++++- .../sql/calcite/QualifiedNameResolver.java | 33 ++++++-- .../sql/common/utils/StringUtilsTest.java | 46 +++++++++++ docs/user/ppl/cmd/mvcombine.md | 4 +- .../sql/calcite/remote/CalcitePPLBasicIT.java | 14 ++++ .../sql/ppl/parser/AstExpressionBuilder.java | 14 +++- 7 files changed, 203 insertions(+), 12 deletions(-) diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index 4b7752a9de5..a3f55fcd231 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -6,8 +6,10 @@ package org.opensearch.sql.common.utils; import com.google.common.base.Strings; +import java.util.Collection; import java.util.IllegalFormatException; import java.util.Locale; +import java.util.Optional; public class StringUtils { /** @@ -96,4 +98,83 @@ public static String format(final String format, Object... args) { private static boolean isQuoted(String text, String mark) { return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); } + + /** + * Calculates the Levenshtein distance between two strings. + * + * @param s1 first string + * @param s2 second string + * @return the Levenshtein distance between s1 and s2 + */ + public static int levenshteinDistance(String s1, String s2) { + if (s1 == null || s2 == null) { + return Integer.MAX_VALUE; + } + if (s1.equals(s2)) { + return 0; + } + + int len1 = s1.length(); + int len2 = s2.length(); + + if (len1 == 0) { + return len2; + } + if (len2 == 0) { + return len1; + } + + int[] prev = new int[len2 + 1]; + int[] curr = new int[len2 + 1]; + + for (int j = 0; j <= len2; j++) { + prev[j] = j; + } + + for (int i = 1; i <= len1; i++) { + curr[0] = i; + for (int j = 1; j <= len2; j++) { + int cost = (s1.charAt(i - 1) == s2.charAt(j - 1)) ? 0 : 1; + curr[j] = Math.min(Math.min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost); + } + int[] temp = prev; + prev = curr; + curr = temp; + } + + return prev[len2]; + } + + /** + * Finds the closest match to a target string from a collection of candidates using Levenshtein + * distance. Returns empty if no candidates are provided or if the best match distance is too + * large. + * + * @param target the string to match against + * @param candidates the collection of candidate strings + * @return the closest match, or empty if no good match is found + */ + public static Optional findClosestMatch(String target, Collection candidates) { + if (target == null || candidates == null || candidates.isEmpty()) { + return Optional.empty(); + } + + String bestMatch = null; + int bestDistance = Integer.MAX_VALUE; + + for (String candidate : candidates) { + int distance = levenshteinDistance(target.toLowerCase(), candidate.toLowerCase()); + if (distance < bestDistance) { + bestDistance = distance; + bestMatch = candidate; + } + } + + // Only return a suggestion if the distance is reasonable + if (bestMatch != null && bestDistance <= Math.max(4, target.length() / 2)) { + return Optional.of(bestMatch); + } + + return Optional.empty(); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java index 84fb486702a..3290c438cd5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/QualifiedName.java @@ -20,22 +20,41 @@ import org.opensearch.sql.ast.AbstractNodeVisitor; @Getter -@EqualsAndHashCode(callSuper = false) +@EqualsAndHashCode( + callSuper = false, + exclude = {"line", "column"}) public class QualifiedName extends UnresolvedExpression { public static final String DELIMITER = "."; private final List parts; + private final Integer line; + + private final Integer column; + public QualifiedName(String name) { - this.parts = Collections.singletonList(name); + this(Collections.singletonList(name), null, null); } /** QualifiedName Constructor. */ public QualifiedName(Iterable parts) { + this(parts, null, null); + } + + /** + * Constructor with source position. + * + * @param parts The parts of the qualified name + * @param line Line number (1-based), null if not available + * @param column Column position (0-based), null if not available + */ + public QualifiedName(Iterable parts, Integer line, Integer column) { List partsList = StreamSupport.stream(parts.spliterator(), false).collect(toList()); if (partsList.isEmpty()) { throw new IllegalArgumentException("parts is empty"); } this.parts = partsList; + this.line = line; + this.column = column; } /** Construct {@link QualifiedName} from list of string. */ diff --git a/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java index dba881b3fc3..c75c0829b5f 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java +++ b/core/src/main/java/org/opensearch/sql/calcite/QualifiedNameResolver.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -18,6 +19,7 @@ import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; @@ -58,7 +60,7 @@ private static RexNode resolveInJoinCondition( return resolveFieldWithAlias(nameNode, context, 2) .or(() -> resolveFieldWithoutAlias(nameNode, context, 2)) - .orElseThrow(() -> getNotFoundException(nameNode)); + .orElseThrow(() -> getNotFoundException(nameNode, context)); } /** Resolves qualified name in non-join condition context. */ @@ -91,7 +93,7 @@ private static RexNode resolveInNonJoinCondition( return resolveCorrelationField(nameNode, context) .or(() -> replaceWithNullLiteralInCoalesce(context)) - .orElseThrow(() -> getNotFoundException(nameNode)); + .orElseThrow(() -> getNotFoundException(nameNode, context)); } private static String joinParts(List parts, int start, int length) { @@ -327,10 +329,27 @@ private static Optional replaceWithNullLiteralInCoalesce(CalcitePlanCon return Optional.empty(); } - private static ErrorReport getNotFoundException(QualifiedName node) { - return ErrorReport.wrap( - new IllegalArgumentException(String.format("Field [%s] not found.", node.toString()))) - .code(ErrorCode.FIELD_NOT_FOUND) - .build(); + private static ErrorReport getNotFoundException(QualifiedName node, CalcitePlanContext context) { + // Collect all available fields from the current context + List availableFields = context.relBuilder.peek().getRowType().getFieldNames(); + + ErrorReport.Builder builder = + ErrorReport.wrap( + new IllegalArgumentException( + String.format("Field [%s] not found.", node.toString()))) + .code(ErrorCode.FIELD_NOT_FOUND) + .context("requested_field", node.toString()) + .context("available_fields", availableFields); + + // Add a suggestion based on Levenshtein distance + StringUtils.findClosestMatch(node.toString(), availableFields) + .ifPresent(suggestion -> builder.suggestion("Did you mean: " + suggestion)); + + // Add source position if available (populated by PPL parser) + if (node.getLine() != null && node.getColumn() != null) { + builder.context("query_pos", Map.of("line", node.getLine(), "column", node.getColumn())); + } + + return builder.build(); } } diff --git a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java index 2a2c9de63ab..b47ec1861fe 100644 --- a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java @@ -6,8 +6,11 @@ package org.opensearch.sql.common.utils; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.common.utils.StringUtils.unquoteText; +import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.Test; class StringUtilsTest { @@ -46,4 +49,47 @@ void unquoteTest() { assertEquals("hel\"lo", unquoteText("\"hel\"lo\"")); assertEquals("hel\\'\\lo", unquoteText("'hel\\\\''\\\\lo'")); } + + @Test + void levenshteinDistanceTest() { + assertEquals(0, StringUtils.levenshteinDistance("test", "test")); + assertEquals(1, StringUtils.levenshteinDistance("test", "text")); + assertEquals(1, StringUtils.levenshteinDistance("test", "tst")); + assertEquals(3, StringUtils.levenshteinDistance("kitten", "sitting")); + assertEquals(4, StringUtils.levenshteinDistance("hello", "world")); + assertEquals(4, StringUtils.levenshteinDistance("test", "")); + assertEquals(4, StringUtils.levenshteinDistance("", "test")); + assertEquals(0, StringUtils.levenshteinDistance("", "")); + } + + @Test + void findClosestMatchTest() { + List fields = List.of("name", "age", "email", "address", "phone"); + + // Exact match or close typo + Optional match = StringUtils.findClosestMatch("nam", fields); + assertTrue(match.isPresent()); + assertEquals("name", match.get()); + + match = StringUtils.findClosestMatch("emal", fields); + assertTrue(match.isPresent()); + assertEquals("email", match.get()); + + match = StringUtils.findClosestMatch("addres", fields); + assertTrue(match.isPresent()); + assertEquals("address", match.get()); + + // Case insensitive + match = StringUtils.findClosestMatch("NAME", fields); + assertTrue(match.isPresent()); + assertEquals("name", match.get()); + + // Too far off - should not match (longer string with many edits) + match = StringUtils.findClosestMatch("xyzabc", fields); + assertTrue(match.isEmpty()); + + // Empty candidates + match = StringUtils.findClosestMatch("test", List.of()); + assertTrue(match.isEmpty()); + } } diff --git a/docs/user/ppl/cmd/mvcombine.md b/docs/user/ppl/cmd/mvcombine.md index bd8aaf12976..9b3e07399f2 100644 --- a/docs/user/ppl/cmd/mvcombine.md +++ b/docs/user/ppl/cmd/mvcombine.md @@ -102,10 +102,12 @@ source=mvcombine_data The query returns the following error: ```text -{'context': {'stage': 'analyzing', 'stage_description': 'Parsing and validating the query'}, 'reason': 'Field [does_not_exist] not found.', 'details': 'Field [does_not_exist] not found.', 'location': ['while preparing and validating the query plan'], 'code': 'FIELD_NOT_FOUND', 'type': 'IllegalArgumentException'} + {'context': {'stage_description': 'Parsing and validating the query', 'stage': 'analyzing', 'requested_field': 'does_not_exist', 'available_fields': ['packets_str', 'bytes', 'case', 'letters', 'ip', 'tags', '_id', '_index', '_score', '_maxscore', '_sort', '_routing'], 'query_pos': {'column': 34, 'line': 1}}, 'reason': 'Field [does_not_exist] not found.', 'details': 'Field [does_not_exist] not found.', 'location': ['while preparing and validating the query plan'], 'code': 'FIELD_NOT_FOUND', 'type': 'IllegalArgumentException'} Error: Query returned no data ``` +======= + ## Related commands - [`nomv`](nomv.md) -- Converts a multivalue field into a single-value string diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 1a826266f19..d17d151d7ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -126,6 +126,20 @@ public void testFieldsShouldBeCaseSensitive() { verifyErrorMessageContains(e, "Field [NAME] not found."); } + @Test + public void testFieldNotFoundWithSuggestion() { + // Typo: "nam" instead of "name" + Throwable e = + assertThrowsWithReplace( + IllegalStateException.class, () -> executeQuery("source=test | fields nam")); + String stack = org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e); + verifyErrorMessageContains(e, "Field [nam] not found."); + // Verify suggestion based on Levenshtein distance + verifyErrorMessageContains(e, "Did you mean: name"); + // Verify available fields are listed + verifyErrorMessageContains(e, "available_fields"); + } + @Test public void testFilterQuery1() throws IOException { JSONObject actual = executeQuery("source=test | where age = 30 | fields name, age"); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 77d5c77a635..7d2f3d9056b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -903,11 +903,21 @@ public UnresolvedExpression visitMaxOption(OpenSearchPPLParser.MaxOptionContext } public QualifiedName visitIdentifiers(List ctx) { - return new QualifiedName( + List parts = ctx.stream() .map(RuleContext::getText) .map(StringUtils::unquoteIdentifier) - .collect(Collectors.toList())); + .collect(Collectors.toList()); + + // Capture source position from the first identifier for error reporting + if (!ctx.isEmpty()) { + ParserRuleContext first = ctx.get(0); + int line = first.getStart().getLine(); + int column = first.getStart().getCharPositionInLine(); + return new QualifiedName(parts, line, column); + } + + return new QualifiedName(parts); } private List singleFieldRelevanceArguments( From 87cfe99f397aa96f6df7ab6210725b85eaabbb69 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 1 Jul 2026 07:19:07 -0700 Subject: [PATCH 24/41] [BugFix] Return 400 instead of 500 on vectorSearch() arg-count mismatch (#5589) A vectorSearch() call with more arguments than its resolved signature declares (for example a duplicate named argument such as table='x', table='x') crashed with an unchecked IndexOutOfBoundsException surfaced as HTTP 500. The resolver returns a fixed-arity signature regardless of how many arguments it was called with, so castArguments looped over the supplied arguments while indexing into the shorter resolved-type list and ran off the end. Guard the argument count against the resolved signature in castArguments and throw an ExpressionEvaluationException, which maps to a clean 400, before any indexing. This protects every custom function resolver, not only vectorSearch(). Signed-off-by: Eric Wei --- .../function/BuiltinFunctionRepository.java | 20 ++++++- .../BuiltinFunctionRepositoryTest.java | 57 ++++++++++++++++++ .../opensearch/sql/sql/VectorSearchIT.java | 20 +++++++ ...VectorSearchTableFunctionResolverTest.java | 59 +++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 79ea58b8608..1e19050f6e3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -162,7 +162,7 @@ private Optional resolve( || sourceTypes.equals(targetTypes)) { return Optional.of(funcBuilder); } - return Optional.of(castArguments(sourceTypes, targetTypes, funcBuilder)); + return Optional.of(castArguments(functionName, sourceTypes, targetTypes, funcBuilder)); } else { return Optional.empty(); } @@ -175,8 +175,24 @@ private Optional resolve( * this case, wrap F and return equal(BOOL, cast_to_bool(STRING)). */ private FunctionBuilder castArguments( - List sourceTypes, List targetTypes, FunctionBuilder funcBuilder) { + FunctionName funcName, + List sourceTypes, + List targetTypes, + FunctionBuilder funcBuilder) { return (fp, arguments) -> { + // A resolver may return a fixed-arity resolved signature regardless of how many arguments it + // was called with (e.g. a table function called with a duplicate named argument). When more + // arguments are supplied than the resolved signature declares, the cast loop would index past + // the end of the resolved type list and throw an unchecked IndexOutOfBoundsException that + // surfaces as HTTP 500. Reject that overflow with a clean ExpressionEvaluationException. + // Under-arity is left to the resolver's own validation, which produces a more specific + // message; it cannot overflow the resolved type list. + if (arguments.size() > targetTypes.size()) { + throw new ExpressionEvaluationException( + String.format( + "Expected %d arguments for function %s but received %d", + targetTypes.size(), funcName.getFunctionName(), arguments.size())); + } List argsCasted = new ArrayList<>(); for (int i = 0; i < arguments.size(); i++) { Expression arg = arguments.get(i); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java index 237477050dc..7d8f2d2ac61 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/BuiltinFunctionRepositoryTest.java @@ -181,6 +181,63 @@ void resolve_should_throw_exception_for_unsupported_conversion() { assertEquals(error.getMessage(), "Type conversion to type STRUCT is not supported"); } + @Test + @DisplayName("resolve should not crash when resolved signature is shorter than the arguments") + void resolve_should_not_index_past_resolved_signature() { + // A custom FunctionResolver (e.g. vectorSearch) may return a fixed-arity resolved signature + // regardless of the number of arguments it was called with. When more arguments are supplied + // than the resolved signature declares (e.g. a duplicate named argument), castArguments must + // not index past the end of the resolved type list. Before the fix this threw an unchecked + // IndexOutOfBoundsException, surfacing as an HTTP 500 instead of the resolver's clean 400. + when(mockFunctionName.getFunctionName()).thenReturn("mock"); + + FunctionName funcName = mockFunctionName; + FunctionSignature sourceSignature = + new FunctionSignature(funcName, ImmutableList.of(STRING, STRING)); + FunctionSignature resolvedSignature = new FunctionSignature(funcName, ImmutableList.of(STRING)); + + DefaultFunctionResolver funcResolver = mock(DefaultFunctionResolver.class); + FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + when(mockMap.containsKey(eq(funcName))).thenReturn(true); + when(mockMap.get(eq(funcName))).thenReturn(funcResolver); + when(funcResolver.resolve(any())).thenReturn(Pair.of(resolvedSignature, funcBuilder)); + + ExpressionEvaluationException error = + assertThrows( + ExpressionEvaluationException.class, + () -> + repo.resolve(Collections.emptyList(), sourceSignature) + .apply(functionProperties, ImmutableList.of(mockExpression, mockExpression))); + assertEquals("Expected 1 arguments for function mock but received 2", error.getMessage()); + } + + @Test + @DisplayName("resolve should defer under-arity to the builder rather than the cast guard") + void resolve_should_not_intercept_under_arity() { + // When fewer arguments are supplied than the resolved signature declares, the cast loop cannot + // index past the resolved type list, so there is no crash to guard against. The overflow guard + // must not fire here; the call must reach the resolved builder so a resolver's own (more + // specific) arity validation still runs. + FunctionName funcName = mockFunctionName; + FunctionSignature sourceSignature = new FunctionSignature(funcName, ImmutableList.of(STRING)); + FunctionSignature resolvedSignature = + new FunctionSignature(funcName, ImmutableList.of(STRING, STRING)); + + DefaultFunctionResolver funcResolver = mock(DefaultFunctionResolver.class); + FunctionBuilder funcBuilder = mock(FunctionBuilder.class); + when(mockMap.containsKey(eq(funcName))).thenReturn(true); + when(mockMap.get(eq(funcName))).thenReturn(funcResolver); + when(funcResolver.resolve(any())).thenReturn(Pair.of(resolvedSignature, funcBuilder)); + when(funcBuilder.apply(eq(functionProperties), any())) + .thenReturn(new FakeFunctionExpression(funcName, ImmutableList.of(mockExpression))); + + repo.resolve(Collections.emptyList(), sourceSignature) + .apply(functionProperties, ImmutableList.of(mockExpression)); + + // The builder is reached (overflow guard did not short-circuit under-arity). + verify(funcBuilder).apply(eq(functionProperties), any()); + } + @Test @DisplayName("resolve unregistered function should throw exception") void resolve_unregistered() { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index 093ea58e489..f3c3046f41e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -113,6 +113,26 @@ public void testEmptyVectorRejects() throws IOException { assertThat(ex.getMessage(), containsString("must not be empty")); } + @Test + public void testDuplicateNamedArgumentRejectsWith400() throws IOException { + // A duplicate named argument on top of all four distinct names sends five arguments to the + // resolver, exceeding its fixed four-arg signature. This previously crashed with an unchecked + // IndexOutOfBoundsException surfaced as HTTP 500; it must now be a clean 400 with a + // user-facing message. + ResponseException ex = + expectThrows( + ResponseException.class, + () -> + executeQuery( + "SELECT v._id FROM vectorSearch(table='t', table='t', field='f', " + + "vector='[1.0]', option='k=5') AS v")); + + assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); + // The five-argument case is caught by the repository arity guard before the resolver's + // duplicate-name check, so the message reports the argument-count mismatch. + assertThat(ex.getMessage(), containsString("Expected 4 arguments")); + } + @Test public void testInvalidFieldNameRejects() throws IOException { ResponseException ex = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java index c6fece7bf32..afa4e9e725e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/VectorSearchTableFunctionResolverTest.java @@ -21,6 +21,7 @@ import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.FunctionBuilder; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.expression.function.FunctionProperties; @@ -182,6 +183,64 @@ void resolve_rejectsDuplicateNamedArgument() { assertTrue(ex.getMessage().contains("table")); } + @Test + void resolverArityGuardRejectsExtraNamedArgument() { + // The resolver's own builder runs validateArguments() first, so applying it directly to a + // duplicate named argument plus all four required arguments (five total) throws a clean + // ExpressionEvaluationException. This covers the resolver's arity guard only; the + // repository-level cast path that originally crashed is covered by + // resolveThroughRepositoryRejectsExtraNamedArgument and BuiltinFunctionRepositoryTest. + VectorSearchTableFunctionResolver resolver = + new VectorSearchTableFunctionResolver(client, settings); + FunctionName functionName = FunctionName.of("vectorsearch"); + List expressions = + List.of( + DSL.namedArgument("table", DSL.literal("a")), + DSL.namedArgument("table", DSL.literal("b")), + DSL.namedArgument("field", DSL.literal("embedding")), + DSL.namedArgument("vector", DSL.literal("[1.0]")), + DSL.namedArgument("option", DSL.literal("k=5"))); + FunctionSignature functionSignature = + new FunctionSignature( + functionName, expressions.stream().map(Expression::type).collect(Collectors.toList())); + FunctionBuilder builder = resolver.resolve(functionSignature).getValue(); + + ExpressionEvaluationException ex = + assertThrows( + ExpressionEvaluationException.class, + () -> builder.apply(functionProperties, expressions)); + assertTrue(ex.getMessage().contains("requires 4 arguments")); + } + + @Test + void resolveThroughRepositoryRejectsExtraNamedArgument() { + // Routes the vectorSearch resolver through BuiltinFunctionRepository the way production does + // (storage-engine resolver registered as a datasource function). A duplicate named argument + // plus all four required arguments gives five arguments against the resolver's fixed four-arg + // signature, which exercises the cast-wrapper path that originally crashed with an unchecked + // IndexOutOfBoundsException (surfacing as HTTP 500). It must now fail with a clean + // ExpressionEvaluationException instead. + VectorSearchTableFunctionResolver resolver = + new VectorSearchTableFunctionResolver(client, settings); + BuiltinFunctionRepository repository = BuiltinFunctionRepository.getInstance(); + List expressions = + List.of( + DSL.namedArgument("table", DSL.literal("a")), + DSL.namedArgument("table", DSL.literal("b")), + DSL.namedArgument("field", DSL.literal("embedding")), + DSL.namedArgument("vector", DSL.literal("[1.0]")), + DSL.namedArgument("option", DSL.literal("k=5"))); + + assertThrows( + ExpressionEvaluationException.class, + () -> + repository.compile( + functionProperties, + List.of(resolver), + FunctionName.of("vectorsearch"), + expressions)); + } + @Test void resolve_rejectsUnknownArgumentName() { VectorSearchTableFunctionResolver resolver = From a71179ea96cefe0082396e63b4e93dadb5fed7f9 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 1 Jul 2026 15:06:19 -0700 Subject: [PATCH 25/41] feat: Add configurable expression depth limit for AST building (#5602) Introduce plugins.query.max_expression_depth (default 1000; 0 to disable) to bound expression nesting depth during AST building, improving robustness for very large or deeply nested SQL/PPL queries. Signed-off-by: Chen Dai --- .../sql/api/UnifiedQueryContext.java | 2 +- .../sql/api/parser/SqlV2QueryParser.java | 19 ++++-- .../api/parser/UnifiedQueryParserTest.java | 35 ++++++++++ .../sql/common/antlr/AstBuildGuard.java | 65 +++++++++++++++++++ .../sql/common/setting/Settings.java | 1 + docs/user/admin/settings.rst | 32 +++++++++ .../opensearch/sql/ppl/QueryValidationIT.java | 44 +++++++++++++ .../opensearch/sql/sql/QueryValidationIT.java | 22 +++++++ .../setting/OpenSearchSettings.java | 16 +++++ .../plugin/config/OpenSearchPluginModule.java | 5 +- .../plugin/rest/RestUnifiedQueryAction.java | 2 + .../opensearch/sql/ppl/parser/AstBuilder.java | 3 +- .../sql/ppl/parser/AstExpressionBuilder.java | 19 ++++++ .../ppl/parser/AstExpressionBuilderTest.java | 42 ++++++++++++ .../org/opensearch/sql/sql/SQLService.java | 10 ++- .../opensearch/sql/sql/parser/AstBuilder.java | 11 +++- .../sql/sql/parser/AstExpressionBuilder.java | 23 +++++++ .../sql/sql/parser/AstBuilderTest.java | 26 ++++++++ .../sql/parser/AstExpressionBuilderTest.java | 39 ++++++++++- 19 files changed, 405 insertions(+), 11 deletions(-) create mode 100644 common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java index 82c7b3cf917..27b76e6e5e4 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java @@ -262,7 +262,7 @@ public UnifiedQueryContext build() { private UnifiedQueryParser createParser(CalcitePlanContext planContext, Settings settings) { return switch (queryType) { case PPL -> new PPLQueryParser(settings); - case SQL -> new SqlV2QueryParser(); + case SQL -> new SqlV2QueryParser(settings); }; } diff --git a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java index 1078c58bcc1..b1f61e732ff 100644 --- a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java +++ b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Not; @@ -26,6 +27,8 @@ import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.antlr.AstBuildGuard; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExistsSubqueryExpressionAtomContext; @@ -42,8 +45,12 @@ import org.opensearch.sql.sql.parser.context.QuerySpecification; /** SQL query parser that produces {@link UnresolvedPlan} using the V2 ANTLR grammar. */ +@RequiredArgsConstructor public class SqlV2QueryParser implements UnifiedQueryParser { + /** Settings containing execution limits and feature flags used by AST builders. */ + private final Settings settings; + /** Reusable ANTLR-based SQL syntax parser. Stateless and thread-safe. */ private final SQLSyntaxParser syntaxParser = new SQLSyntaxParser(); @@ -52,7 +59,7 @@ public UnresolvedPlan parse(String query) { ParseTree cst = syntaxParser.parse(query); AstStatementBuilder astStmtBuilder = new AstStatementBuilder( - new ExtendedAstBuilder(query), + new ExtendedAstBuilder(query, settings), AstStatementBuilder.StatementBuilderContext.builder().build()); Statement statement = cst.accept(astStmtBuilder); @@ -69,8 +76,8 @@ public UnresolvedPlan parse(String query) { */ private static class ExtendedAstBuilder extends AstBuilder { - ExtendedAstBuilder(String query) { - super(query); + ExtendedAstBuilder(String query, Settings settings) { + super(query, settings); } @Override @@ -122,7 +129,7 @@ private static boolean hasWindowFunctionInProjectList(QuerySpecification querySp @Override protected AstExpressionBuilder createExpressionBuilder() { - return new ExtendedAstExpressionBuilder(); + return new ExtendedAstExpressionBuilder(guard); } @Override @@ -157,6 +164,10 @@ public UnresolvedPlan visitUnionSelect(UnionSelectContext ctx) { */ private class ExtendedAstExpressionBuilder extends AstExpressionBuilder { + ExtendedAstExpressionBuilder(AstBuildGuard guard) { + super(guard); + } + @Override public UnresolvedExpression visitInSubqueryPredicate(InSubqueryPredicateContext ctx) { UnresolvedPlan subquery = ExtendedAstBuilder.this.visit(ctx.querySpecification()); diff --git a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java index 1b6b5181aef..bb06e7c7b8b 100644 --- a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java +++ b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java @@ -26,9 +26,12 @@ import static org.opensearch.sql.ast.dsl.AstDSL.relation; import org.junit.Test; +import org.opensearch.sql.api.UnifiedQueryContext; import org.opensearch.sql.api.UnifiedQueryTestBase; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.QueryType; public class UnifiedQueryParserTest extends UnifiedQueryTestBase { @@ -80,6 +83,38 @@ public void testSyntaxErrorThrows() { assertThrows(SyntaxCheckException.class, () -> context.getParser().parse("not a valid query")); } + @Test + public void deeplyNestedExpressionShouldBeRejected() throws Exception { + String key = Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(); + try (UnifiedQueryContext ppl = + UnifiedQueryContext.builder() + .language(QueryType.PPL) + .catalog(DEFAULT_CATALOG, testSchema) + .setting(key, 20) + .build(); + UnifiedQueryContext sql = + UnifiedQueryContext.builder() + .language(QueryType.SQL) + .catalog(DEFAULT_CATALOG, testSchema) + .setting(key, 20) + .build()) { + assertThrows( + IllegalArgumentException.class, + () -> ppl.getParser().parse("source = catalog.employees | where " + orChain(30))); + assertThrows( + IllegalArgumentException.class, + () -> sql.getParser().parse("SELECT * FROM catalog.employees WHERE " + orChain(30))); + } + } + + private String orChain(int terms) { + StringBuilder sb = new StringBuilder("age = 1"); + for (int i = 2; i <= terms; i++) { + sb.append(" or age = ").append(i); + } + return sb.toString(); + } + private void assertEqual(String query, UnresolvedPlan expected) { UnresolvedPlan actual = (UnresolvedPlan) context.getParser().parse(query); assertEquals(expected, actual); diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java b/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java new file mode 100644 index 00000000000..7dd6311a150 --- /dev/null +++ b/common/src/main/java/org/opensearch/sql/common/antlr/AstBuildGuard.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.common.antlr; + +import java.util.function.Supplier; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.utils.StringUtils; + +/** + * Guards AST construction against pathological queries by bounding the depth of parse-tree visitor + * recursion, rejecting over-nested expressions with an {@link IllegalArgumentException} instead of + * letting a StackOverflowError crash the node. + * + *

Holds per-traversal state, so an instance must not be shared across threads. + */ +@RequiredArgsConstructor +public final class AstBuildGuard { + + public static final int DEFAULT_MAX_DEPTH = 1000; + + /** Maximum nesting depth allowed; {@code 0} or less means unlimited. */ + private final int maxDepth; + + /** Live nesting depth of the in-progress traversal; resets to 0 between top-level visits. */ + private int depth = 0; + + public AstBuildGuard() { + this(DEFAULT_MAX_DEPTH); + } + + /** Builds a guard from the configured {@code plugins.query.max_expression_depth} setting. */ + public static AstBuildGuard fromSettings(Settings settings) { + Integer configured = + settings == null ? null : settings.getSettingValue(Settings.Key.MAX_EXPRESSION_DEPTH); + return new AstBuildGuard(configured == null ? DEFAULT_MAX_DEPTH : configured); + } + + /** + * Runs a single AST-build descent under the configured guardrails. + * + * @param visit the visitor descent to execute + * @return the result of {@code visit} + * @throws IllegalArgumentException if a positive maximum depth is configured and would be + * exceeded + */ + public T enforce(Supplier visit) { + if (maxDepth > 0 && depth >= maxDepth) { + throw new IllegalArgumentException( + StringUtils.format( + "Expression nesting depth exceeds the maximum allowed [%d]; simplify the query or" + + " reduce the number of chained conditions.", + maxDepth)); + } + depth++; + try { + return visit.get(); + } finally { + depth--; + } + } +} diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 96fe2e04eea..bf3e65d8741 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -51,6 +51,7 @@ public enum Key { /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), + MAX_EXPRESSION_DEPTH("plugins.query.max_expression_depth"), QUERY_BUCKET_SIZE("plugins.query.buckets"), SEARCH_MAX_BUCKETS("search.max_buckets"), ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"), diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index d8402605a4d..dafa8c84172 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -204,6 +204,38 @@ Result set:: } } +plugins.query.max_expression_depth +================================== + +Version +------- +3.8 + +Description +----------- + +This setting bounds the maximum nesting depth of an expression while a query is parsed into its abstract syntax tree, keeping parsing bounded for very large or deeply nested expressions. A query exceeding the limit is rejected with a 400 error. The default value is 1000. Set it to 0 to disable the limit (unlimited). Here is an example:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.max_expression_depth" : 500 + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "query" : { + "max_expression_depth" : "500" + } + } + } + } + plugins.query.buckets ===================== diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java new file mode 100644 index 00000000000..4e4083ad5ef --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryValidationIT.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import org.junit.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.common.setting.Settings; + +/** PPL counterpart of the SQL {@code QueryValidationIT} for query-level rejection cases. */ +public class QueryValidationIT extends PPLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.ACCOUNT); + } + + @Test + public void deeplyNestedPredicateIsRejectedInsteadOfCrashingNode() throws IOException { + // Lower the limit so a small, safe-to-parse query triggers the guard (a query large enough + // to exhaust the default limit could overflow the ANTLR parser itself before the guard runs). + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), "20")); + try { + StringBuilder predicate = new StringBuilder("age = 1"); + for (int i = 2; i <= 30; i++) { + predicate.append(" or age = ").append(i); + } + executeQuery("source=opensearch-sql_test_index_account | where " + predicate); + fail("Expected ResponseException for an over-nested predicate"); + } catch (ResponseException e) { + assertTrue(e.getMessage().contains("Expression nesting depth exceeds the maximum allowed")); + } finally { + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), null)); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 9c4df756792..e814252cfe7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -22,6 +22,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.ResponseException; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.util.RequiresCapability; @@ -39,6 +40,27 @@ protected void init() throws Exception { loadIndex(Index.ACCOUNT); } + @Test + public void testDeeplyNestedPredicateIsRejectedInsteadOfCrashingNode() throws IOException { + // Lower the limit so a small, safe-to-parse query triggers the guard (a query large enough + // to exhaust the default limit could overflow the ANTLR parser itself before the guard runs). + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), "20")); + try { + StringBuilder predicate = new StringBuilder("age = 1"); + for (int i = 2; i <= 30; i++) { + predicate.append(" OR age = ").append(i); + } + expectResponseException() + .hasStatusCode(BAD_REQUEST) + .containsMessage("Expression nesting depth exceeds the maximum allowed") + .whenExecute("SELECT * FROM opensearch-sql_test_index_account WHERE " + predicate); + } finally { + updateClusterSettings( + new ClusterSetting(TRANSIENT, Settings.Key.MAX_EXPRESSION_DEPTH.getKeyValue(), null)); + } + } + @Ignore( "Will add this validation in analyzer later. This test should be enabled once " + "https://github.com/opensearch-project/sql/issues/910 has been resolved") diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index bd8001f589d..b596c7bc47a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -29,6 +29,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; import org.opensearch.search.aggregations.MultiBucketConsumerService; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.setting.Settings; /** Setting implementation on OpenSearch. */ @@ -201,6 +202,14 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting MAX_EXPRESSION_DEPTH_SETTING = + Setting.intSetting( + Key.MAX_EXPRESSION_DEPTH.getKeyValue(), + AstBuildGuard.DEFAULT_MAX_DEPTH, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + // Set the default value to QUERY_SIZE_LIMIT_SETTING public static final Setting QUERY_BUCKET_SIZE_SETTING = Setting.intSetting( @@ -479,6 +488,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.QUERY_SIZE_LIMIT, QUERY_SIZE_LIMIT_SETTING, new Updater(Key.QUERY_SIZE_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.MAX_EXPRESSION_DEPTH, + MAX_EXPRESSION_DEPTH_SETTING, + new Updater(Key.MAX_EXPRESSION_DEPTH)); register( settingBuilder, clusterSettings, @@ -650,6 +665,7 @@ public static List> pluginSettings() { .add(SQL_ENABLED_SETTING) .add(SQL_SLOWLOG_SETTING) .add(SQL_CURSOR_KEEP_ALIVE_SETTING) + .add(MAX_EXPRESSION_DEPTH_SETTING) .add(PPL_ENABLED_SETTING) .add(PPL_QUERY_TIMEOUT_SETTING) .add(PPL_SYNTAX_LEGACY_PREFERRED_SETTING) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java index d9406935ee5..057c88c9a02 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -106,8 +106,9 @@ public PPLService pplService( } @Provides - public SQLService sqlService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { - return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory); + public SQLService sqlService( + QueryManager queryManager, QueryPlanFactory queryPlanFactory, Settings settings) { + return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory, settings); } /** {@link QueryPlanFactory}. */ diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 2e810033a9f..5efc7b57baa 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -338,6 +338,8 @@ private UnifiedQueryContext.Builder applyClusterOverrides(UnifiedQueryContext.Bu builder, org.opensearch.sql.common.setting.Settings.Key.PPL_REX_MAX_MATCH_LIMIT); forwardClusterSetting( builder, org.opensearch.sql.common.setting.Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + forwardClusterSetting( + builder, org.opensearch.sql.common.setting.Settings.Key.MAX_EXPRESSION_DEPTH); return builder; } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3741137f5a9..f8604a352ce 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -124,6 +124,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; @@ -159,7 +160,7 @@ public AstBuilder(String query) { } public AstBuilder(String query, Settings settings) { - this.expressionBuilder = new AstExpressionBuilder(this); + this.expressionBuilder = new AstExpressionBuilder(this, AstBuildGuard.fromSettings(settings)); this.query = query; this.settings = settings; } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 7d2f3d9056b..8d70de44366 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -22,6 +22,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.dsl.AstDSL; @@ -31,6 +32,7 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; @@ -95,8 +97,25 @@ public class AstExpressionBuilder extends OpenSearchPPLParserBaseVisitor super.visit(tree)); + } + + @Override + public UnresolvedExpression visitChildren(RuleNode node) { + return guard.enforce(() -> super.visitChildren(node)); } /** Eval clause. */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index ce7a120ff56..bf90b8bbbc9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -55,7 +55,9 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import org.antlr.v4.runtime.CommonTokenStream; import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ast.Node; @@ -65,10 +67,50 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.Chart; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.AstBuildGuard; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; +import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; public class AstExpressionBuilderTest extends AstBuilderTest { + + @Test + public void deeplyNestedExpressionShouldBeRejected() { + for (String expr : + List.of( + nest(30, "a = 0", e -> "(" + e + " or a = 1)"), + nest(30, "a = 0", e -> "(" + e + " and a = 1)"), + nest(30, "a", e -> "abs(" + e + ")") + " = 1")) { + assertThrows(IllegalArgumentException.class, () -> parseWithGuard(expr, 20)); + } + } + + @Test + public void shallowExpressionWithinLimitIsAccepted() { + parseWithGuard("a = 0 or a = 1 or a = 2", 20); + } + + private void parseWithGuard(String expr, int maxDepth) { + OpenSearchPPLParser parser = + new OpenSearchPPLParser( + new CommonTokenStream(new OpenSearchPPLLexer(new CaseInsensitiveCharStream(expr)))); + parser.addErrorListener(new SyntaxAnalysisErrorListener()); + parser + .logicalExpression() + .accept(new AstExpressionBuilder(new AstBuilder(expr), new AstBuildGuard(maxDepth))); + } + + private static String nest(int depth, String base, UnaryOperator wrap) { + String expr = base; + for (int i = 0; i < depth; i++) { + expr = wrap.apply(expr); + } + return expr; + } + @Test public void testLogicalNotExpr() { assertEqual( diff --git a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java index d51fa1c898d..9b4cf8c1a37 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/SQLService.java +++ b/sql/src/main/java/org/opensearch/sql/sql/SQLService.java @@ -11,6 +11,7 @@ import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.statement.Statement; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.executor.QueryManager; @@ -32,8 +33,15 @@ public class SQLService { private final QueryPlanFactory queryExecutionFactory; + private final Settings settings; + private final QueryType SQL_QUERY = QueryType.SQL; + public SQLService( + SQLSyntaxParser parser, QueryManager queryManager, QueryPlanFactory queryExecutionFactory) { + this(parser, queryManager, queryExecutionFactory, null); + } + /** * Given {@link SQLQueryRequest}, execute it. Using listener to listen result. * @@ -95,7 +103,7 @@ private AbstractPlan plan( Statement statement = cst.accept( new AstStatementBuilder( - new AstBuilder(request.getQuery()), + new AstBuilder(request.getQuery(), settings), AstStatementBuilder.StatementBuilderContext.builder() .isExplain(isExplainRequest) .fetchSize(request.getFetchSize()) diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index 641ef0d39ca..f1f140fce8e 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -39,7 +39,9 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -62,8 +64,15 @@ public class AstBuilder extends OpenSearchSQLParserBaseVisitor { */ protected final String query; + protected final AstBuildGuard guard; + public AstBuilder(String query) { + this(query, null); + } + + public AstBuilder(String query, Settings settings) { this.query = query; + this.guard = AstBuildGuard.fromSettings(settings); this.expressionBuilder = createExpressionBuilder(); } @@ -290,7 +299,7 @@ protected UnresolvedExpression visitAstExpression(ParseTree tree) { /** Override to provide a custom expression builder (e.g., with subquery support). */ protected AstExpressionBuilder createExpressionBuilder() { - return new AstExpressionBuilder(); + return new AstExpressionBuilder(guard); } private UnresolvedExpression visitSelectItem(SelectElementContext ctx) { diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 33cfb1a56ca..e7510f31b7a 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -79,11 +79,14 @@ import java.util.Optional; import java.util.stream.Collectors; import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.*; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -101,6 +104,26 @@ /** Expression builder to parse text to expression in AST. */ public class AstExpressionBuilder extends OpenSearchSQLParserBaseVisitor { + private final AstBuildGuard guard; + + public AstExpressionBuilder() { + this(new AstBuildGuard()); + } + + public AstExpressionBuilder(AstBuildGuard guard) { + this.guard = guard; + } + + @Override + public UnresolvedExpression visit(ParseTree tree) { + return guard.enforce(() -> super.visit(tree)); + } + + @Override + public UnresolvedExpression visitChildren(RuleNode node) { + return guard.enforce(() -> super.visitChildren(node)); + } + @Override public UnresolvedExpression visitTableName(TableNameContext ctx) { return visit(ctx.qualifiedName()); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 9d504a32bb0..893731e4a10 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Test; @@ -51,6 +52,7 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; @@ -777,4 +779,28 @@ public void exists_subquery_throws_syntax_check_exception() { SyntaxCheckException.class, () -> buildAST("SELECT * FROM t WHERE EXISTS (SELECT 1 FROM t2)")); } + + @Test + public void configured_max_expression_depth_from_settings_is_applied() { + Settings settings = + new Settings() { + @Override + public T getSettingValue(Key key) { + return (T) Integer.valueOf(5); + } + + @Override + public List getSettings() { + return List.of(); + } + }; + StringBuilder where = new StringBuilder("a = 1"); + for (int i = 2; i <= 20; i++) { + where.append(" OR a = ").append(i); + } + String query = "SELECT * FROM t WHERE " + where; + AstBuilder builder = new AstBuilder(query, settings); + assertThrows( + IllegalArgumentException.class, () -> new SQLSyntaxParser().parse(query).accept(builder)); + } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index cb981f6f45f..aba8023b07e 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql.parser; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.and; import static org.opensearch.sql.ast.dsl.AstDSL.between; @@ -35,6 +36,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.HashMap; +import java.util.List; +import java.util.function.UnaryOperator; import java.util.stream.Stream; import org.antlr.v4.runtime.CommonTokenStream; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -47,6 +50,7 @@ import org.opensearch.sql.ast.expression.WindowFrame; import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; +import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLLexer; @@ -827,10 +831,43 @@ public void canBuildInClause() { buildExprAst("age in (abs(20), abs(30))")); } + @Test + public void deeplyNestedExpressionShouldBeRejected() { + AstExpressionBuilder guarded = new AstExpressionBuilder(new AstBuildGuard(20)); + for (String expr : + List.of( + nest(30, "a = 0", e -> "(" + e + " or a = 1)"), + nest(30, "a = 0", e -> "(" + e + " and a = 1)"), + nest(30, "a", e -> "abs(" + e + ")"), + nest(30, "a", e -> "case when " + e + " > 0 then 1 else 0 end"))) { + assertThrows(IllegalArgumentException.class, () -> buildExprAst(expr, guarded)); + } + } + + @Test + public void shallowExpressionWithinLimitIsAccepted() { + AstExpressionBuilder guarded = new AstExpressionBuilder(new AstBuildGuard(20)); + assertEquals( + buildExprAst("a = 0 or a = 1 or a = 2", guarded), + buildExprAst("a = 0 or a = 1 or a = 2", guarded)); + } + + private static String nest(int depth, String base, UnaryOperator wrap) { + String expr = base; + for (int i = 0; i < depth; i++) { + expr = wrap.apply(expr); + } + return expr; + } + private Node buildExprAst(String expr) { + return buildExprAst(expr, astExprBuilder); + } + + private Node buildExprAst(String expr, AstExpressionBuilder builder) { OpenSearchSQLLexer lexer = new OpenSearchSQLLexer(new CaseInsensitiveCharStream(expr)); OpenSearchSQLParser parser = new OpenSearchSQLParser(new CommonTokenStream(lexer)); parser.addErrorListener(new SyntaxAnalysisErrorListener()); - return parser.expression().accept(astExprBuilder); + return parser.expression().accept(builder); } } From 307a51e7e25bfd87fabe9a097febbb39756f460c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 2 Jul 2026 13:20:43 -0700 Subject: [PATCH 26/41] feat: add json_tree explain format (#5576) * Add json_tree explain format Signed-off-by: Simeon Widdis * code review updates Signed-off-by: Simeon Widdis * revert all those explain tests Signed-off-by: Simeon Widdis * revert explain behavior to old behavior outside of json_tree path Signed-off-by: Simeon Widdis * revert doctest updates Signed-off-by: Simeon Widdis * when pushdown is disabled, there's no sourcebuilder Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../opensearch/sql/ast/statement/Explain.java | 9 +- .../calcite/plan/rel/LogicalSystemLimit.java | 4 +- .../sql/executor/ExecutionEngine.java | 14 ++ .../opensearch/sql/executor/QueryService.java | 30 +++- .../sql/executor/execution/AbstractPlan.java | 13 ++ .../sql/executor/execution/ExplainPlan.java | 22 ++- .../sql/executor/execution/QueryPlan.java | 9 +- .../executor/execution/QueryPlanFactory.java | 1 + .../sql/protocol/response/format/Format.java | 3 + .../executor/execution/ExplainPlanTest.java | 4 +- .../sql/executor/execution/QueryPlanTest.java | 4 +- .../sql/calcite/remote/CalciteExplainIT.java | 27 ++++ .../calcite/remote/CalcitePPLExplainIT.java | 49 ++++++ .../opensearch/sql/ppl/PPLIntegTestCase.java | 2 +- .../executor/OpenSearchExecutionEngine.java | 152 +++++++++++++++--- .../scan/AbstractCalciteIndexScan.java | 27 +++- .../transport/TransportPPLQueryAction.java | 14 ++ .../org/opensearch/sql/ppl/PPLService.java | 7 +- .../sql/ppl/parser/AstStatementBuilder.java | 12 +- .../opensearch/sql/ppl/PPLServiceTest.java | 10 -- 20 files changed, 361 insertions(+), 52 deletions(-) rename {protocol => core}/src/main/java/org/opensearch/sql/protocol/response/format/Format.java (93%) diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index b2fcb2b8179..01ce206ad23 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -9,6 +9,7 @@ import lombok.Getter; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Explain Statement. */ @Getter @@ -18,15 +19,21 @@ public class Explain extends Statement { private final Statement statement; private final QueryType queryType; private final ExplainMode mode; + private final Format format; public Explain(Statement statement, QueryType queryType) { - this(statement, queryType, null); + this(statement, queryType, null, null); } public Explain(Statement statement, QueryType queryType, String mode) { + this(statement, queryType, mode, null); + } + + public Explain(Statement statement, QueryType queryType, String mode, Format format) { this.statement = statement; this.queryType = queryType; this.mode = ExplainMode.of(mode); + this.format = format; } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java index 4f999cf0792..2ba6e8812dc 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rel/LogicalSystemLimit.java @@ -83,8 +83,8 @@ public Sort copy( @Override public RelWriter explainTerms(RelWriter pw) { super.explainTerms(pw); - // Show type in the explain - pw.item("type", type); + // Show type in the explain - convert to string for JSON serialization compatibility + pw.item("type", type.name()); return pw; } } diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index 2a5d392a149..9b51876c004 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -21,6 +21,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.pagination.Cursor; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; /** Execution engine that encapsulates execution details. */ public interface ExecutionEngine { @@ -75,6 +76,16 @@ default void explain( getClass().getSimpleName() + " does not support RelNode explain")); } + default void explain( + RelNode plan, + ExplainMode mode, + Format format, + CalcitePlanContext context, + ResponseListener listener) { + // Default: ignore format parameter, delegate to old signature for BWC + explain(plan, mode, context, listener); + } + /** Data class that encapsulates ExprValue. */ @Data class QueryResponse { @@ -163,5 +174,8 @@ class ExplainResponseNodeV2 { private final String logical; private final String physical; private final String extended; + // For json_tree format: parsed JSON objects instead of strings + private Object logicalTree; + private Object physicalTree; } } diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index fe9d3e55dc1..ddb4338bc8f 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -54,6 +54,7 @@ import org.opensearch.sql.planner.logical.LogicalPaginate; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; /** The low level interface of core engine. */ @RequiredArgsConstructor @@ -124,8 +125,19 @@ public void explain( HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explain(plan, queryType, highlightConfig, listener, mode, null); + } + + /** Explain with optional highlight config and format. */ + public void explain( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + ResponseListener listener, + ExplainMode mode, + Format format) { if (shouldUseCalcite(queryType)) { - explainWithCalcite(plan, queryType, highlightConfig, listener, mode); + explainWithCalcite(plan, queryType, highlightConfig, listener, mode, format); } else { explainWithLegacy(plan, queryType, listener, mode, Optional.empty()); } @@ -192,6 +204,16 @@ public void explainWithCalcite( HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explainWithCalcite(plan, queryType, highlightConfig, listener, mode, null); + } + + public void explainWithCalcite( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + ResponseListener listener, + ExplainMode mode, + Format format) { CalcitePlanContext.run( () -> { try { @@ -206,7 +228,11 @@ public void explainWithCalcite( () -> { RelNode relNode = analyze(plan, context); RelNode calcitePlan = convertToCalcitePlan(relNode, context); - executionEngine.explain(calcitePlan, mode, context, listener); + if (format != null) { + executionEngine.explain(calcitePlan, mode, format, context, listener); + } else { + executionEngine.explain(calcitePlan, mode, context, listener); + } }, settings); }, diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java index e470d12507e..fbdabe2fa44 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -12,6 +12,7 @@ import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** AbstractPlan represent the execution entity of the Statement. */ @RequiredArgsConstructor @@ -32,4 +33,16 @@ public abstract class AbstractPlan { */ public abstract void explain( ResponseListener listener, ExplainMode mode); + + /** + * Explain query execution with format. + * + * @param listener query explain response listener. + * @param mode explain mode + * @param format output format + */ + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { + explain(listener, mode); + } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java index 27f7a47e504..0a196c6f484 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -10,12 +10,14 @@ import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Explain plan. */ public class ExplainPlan extends AbstractPlan { private final AbstractPlan plan; private final ExplainMode mode; + private final Format format; private final ResponseListener explainListener; @@ -26,15 +28,27 @@ public ExplainPlan( AbstractPlan plan, ExplainMode mode, ResponseListener explainListener) { + this(queryId, queryType, plan, mode, null, explainListener); + } + + /** Constructor with format. */ + public ExplainPlan( + QueryId queryId, + QueryType queryType, + AbstractPlan plan, + ExplainMode mode, + Format format, + ResponseListener explainListener) { super(queryId, queryType); this.plan = plan; this.mode = mode; + this.format = format; this.explainListener = explainListener; } @Override public void execute() { - plan.explain(explainListener, mode); + plan.explain(explainListener, mode, format); } @Override @@ -42,4 +56,10 @@ public void explain( ResponseListener listener, ExplainMode mode) { throw new UnsupportedOperationException("explain query can not been explained."); } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { + throw new UnsupportedOperationException("explain query can not been explained."); + } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index 3f6407e8873..762997439f4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -16,6 +16,7 @@ import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.QueryType; +import org.opensearch.sql.protocol.response.format.Format; /** Query plan which includes a select query. */ public class QueryPlan extends AbstractPlan { @@ -86,12 +87,18 @@ public void execute() { @Override public void explain( ResponseListener listener, ExplainMode mode) { + explain(listener, mode, null); + } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode, Format format) { if (pageSize.isPresent()) { listener.onFailure( new NotImplementedException( "`explain` feature for paginated requests is not implemented yet.")); } else { - queryService.explain(plan, getQueryType(), highlightConfig, listener, mode); + queryService.explain(plan, getQueryType(), highlightConfig, listener, mode, format); } } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 48e2b3ce5e0..93c73a2315b 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -144,6 +144,7 @@ public AbstractPlan visitExplain( node.getQueryType(), create(node.getStatement(), NO_CONSUMER_RESPONSE_LISTENER, context.getRight()), node.getMode(), + node.getFormat(), context.getRight()); } } diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java similarity index 93% rename from protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java rename to core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java index 6db28e4d4ab..b8daf3bbd85 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java +++ b/core/src/main/java/org/opensearch/sql/protocol/response/format/Format.java @@ -25,6 +25,8 @@ public enum Format { JSON("json"), /** Returns explain output in yaml format */ YAML("yaml"), + /** Returns explain output as structured JSON tree using RelJsonWriter */ + JSON_TREE("json_tree"), /*---- backward compatible format of explain response -----*/ SIMPLE("simple"), @@ -52,6 +54,7 @@ public enum Format { builder = new ImmutableMap.Builder<>(); builder.put(JSON.formatName, JSON); builder.put(YAML.formatName, YAML); + builder.put(JSON_TREE.formatName, JSON_TREE); builder.put(SIMPLE.formatName, SIMPLE); builder.put(STANDARD.formatName, STANDARD); builder.put(EXTENDED.formatName, EXTENDED); diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java index 4cb5c755d14..977c1cb4729 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -36,12 +36,12 @@ public class ExplainPlanTest { @Test public void execute() { - doNothing().when(queryPlan).explain(any(), any()); + doNothing().when(queryPlan).explain(any(), any(), any()); ExplainPlan explainPlan = new ExplainPlan(queryId, queryType, queryPlan, mode, explainListener); explainPlan.execute(); - verify(queryPlan, times(1)).explain(explainListener, mode); + verify(queryPlan, times(1)).explain(explainListener, mode, null); } @Test diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 128df14ff8e..6e05c8c1258 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -58,9 +58,9 @@ public void execute_no_page_size() { @Test public void explain_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); - query.explain(explainListener, mode); + query.explain(explainListener, mode, null); - verify(queryService, times(1)).explain(plan, queryType, null, explainListener, mode); + verify(queryService, times(1)).explain(plan, queryType, null, explainListener, mode, null); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 95c47b9b0b7..d475f11427d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.util.Locale; import org.apache.commons.text.StringEscapeUtils; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -3010,4 +3012,29 @@ public void testExplainUnion() throws IOException { String expected = loadExpectedPlan("explain_union.yaml"); assertYamlEqualsIgnoreId(expected, actual); } + + @Test + public void testExplainJsonTreeFormat() throws IOException { + String query = "source=opensearch-sql_test_index_account | where age > 30 | fields age"; + String result = explainQuery(query, Format.JSON_TREE, ExplainMode.STANDARD); + + // Parse JSON response + JSONObject json = new JSONObject(result); + JSONObject calcite = json.getJSONObject("calcite"); + + // Verify logical plan is a structured JSON object (not a plain string) + JSONObject logical = calcite.getJSONObject("logical"); + Assert.assertTrue("Logical plan should contain 'rels' array", logical.has("rels")); + JSONArray rels = logical.getJSONArray("rels"); + Assert.assertTrue("Rels array should not be empty", rels.length() > 0); + + // Verify first rel is a proper RelNode structure + JSONObject firstRel = rels.getJSONObject(0); + Assert.assertTrue("RelNode should have 'relOp' field", firstRel.has("relOp")); + Assert.assertTrue("RelNode should have 'id' field", firstRel.has("id")); + + // Verify physical plan also has structured format + JSONObject physical = calcite.getJSONObject("physical"); + Assert.assertTrue("Physical plan should contain 'rels' array", physical.has("rels")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java index 674a7d96f8d..78ff6fc0401 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java @@ -7,10 +7,14 @@ import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; +import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.ppl.PPLIntegTestCase; +import org.opensearch.sql.ppl.PPLIntegTestCase.GlobalPushdownConfig; +import org.opensearch.sql.protocol.response.format.Format; public class CalcitePPLExplainIT extends PPLIntegTestCase { @@ -74,6 +78,51 @@ public void testExplainCommandSimple() throws IOException { assertJsonEquals(expected, result); } + @Test + public void testJsonTreeFormat() throws IOException { + var resultStr = + explainQuery( + "source=test | where age > 20 | fields name", Format.JSON_TREE, ExplainMode.STANDARD); + + // Parse JSON + var mapper = new ObjectMapper(); + var result = mapper.readTree(resultStr); + + // Verify tree structure exists + assertTrue(result.has("calcite")); + assertTrue(result.get("calcite").has("logical")); + assertTrue(result.get("calcite").has("physical")); + + // Verify logical and physical are parsed JSON objects, not strings + assertTrue(result.get("calcite").get("logical").isObject()); + assertTrue(result.get("calcite").get("physical").isObject()); + + // Verify sourceBuilder exists in physical plan rels + var physical = result.get("calcite").get("physical"); + assertTrue(physical.has("rels")); + var rels = physical.get("rels"); + assertTrue(rels.isArray()); + + // Find a rel with sourceBuilder (only present when pushdown is enabled) + boolean foundSourceBuilder = false; + for (int i = 0; i < rels.size(); i++) { + var rel = rels.get(i); + if (rel.has("sourceBuilder")) { + foundSourceBuilder = true; + // Verify sourceBuilder is a parsed JSON object, not a string + assertTrue(rel.get("sourceBuilder").isObject()); + // Verify it has expected OpenSearch DSL fields + assertTrue(rel.get("sourceBuilder").has("from")); + assertTrue(rel.get("sourceBuilder").has("size")); + break; + } + } + // Only assert sourceBuilder exists when pushdown is enabled + if (GlobalPushdownConfig.enabled) { + assertTrue("sourceBuilder not found in physical plan rels", foundSourceBuilder); + } + } + /** * Executes the PPL query and returns the result as a string with windows-style line breaks * replaced with Unix-style ones. diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 3db2142cffe..f22cea74734 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -108,7 +108,7 @@ protected String explainQueryToString(String query, ExplainMode mode) throws IOE return explainQuery(query, Format.JSON, mode).replace("\\r\\n", "\\n"); } - private String explainQuery(String query, Format format, ExplainMode mode) throws IOException { + protected String explainQuery(String query, Format format, ExplainMode mode) throws IOException { Response response = client() .performRequest(buildRequest(query, String.format(EXPLAIN_API_ENDPOINT, format, mode))); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 32b7891d344..3a1fa9fe78d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.executor; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Suppliers; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -24,6 +25,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.externalize.RelJsonWriter; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.runtime.Hook; @@ -65,12 +67,14 @@ import org.opensearch.sql.opensearch.functions.DistinctCountApproxAggFunction; import org.opensearch.sql.opensearch.functions.GeoIpFunction; import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.transport.client.node.NodeClient; /** OpenSearch execution engine implementation. */ public class OpenSearchExecutionEngine implements ExecutionEngine { private static final Logger logger = LogManager.getLogger(OpenSearchExecutionEngine.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); private final OpenSearchClient client; @@ -167,38 +171,146 @@ private Hook.Closeable getCodegenInHook(AtomicReference codegen) { }); } + /** + * Parse sourceBuilder JSON strings within the physical plan tree to objects. This finds any + * sourceBuilder fields (which are serialized as JSON strings by RelJsonWriter) and parses them to + * JSON objects for easier client consumption. + */ + @SuppressWarnings("unchecked") + private void parseSourceBuilderInPhysicalTree(Object physicalTree) { + try { + if (!(physicalTree instanceof Map)) { + return; + } + Map tree = (Map) physicalTree; + Object relsObj = tree.get("rels"); + if (!(relsObj instanceof List)) { + return; + } + + List rels = (List) relsObj; + for (Object relObj : rels) { + if (!(relObj instanceof Map)) { + continue; + } + Map rel = (Map) relObj; + + // Parse sourceBuilder if it exists as a JSON string + Object sourceBuilderObj = rel.get("sourceBuilder"); + if (sourceBuilderObj instanceof String) { + try { + String sourceBuilderJson = (String) sourceBuilderObj; + Object parsed = objectMapper.readValue(sourceBuilderJson, Object.class); + rel.put("sourceBuilder", parsed); + } catch (Exception e) { + logger.debug("Failed to parse sourceBuilder JSON: {}", e.getMessage()); + } + } + } + } catch (Exception e) { + logger.warn("Failed to parse sourceBuilder in physical tree: " + e.getMessage()); + } + } + @Override public void explain( RelNode rel, ExplainMode mode, CalcitePlanContext context, ResponseListener listener) { + explain(rel, mode, null, context, listener); + } + + @Override + public void explain( + RelNode rel, + ExplainMode mode, + Format format, + CalcitePlanContext context, + ResponseListener listener) { client.schedule( () -> { try { - if (mode == ExplainMode.SIMPLE) { - String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); - listener.onResponse( - new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); + if (format == Format.JSON_TREE) { + // Use RelJsonWriter for structured JSON tree output + try { + RelJsonWriter logicalWriter = new RelJsonWriter(); + rel.explain(logicalWriter); + String logicalJson = logicalWriter.asString(); + + AtomicReference physicalJson = new AtomicReference<>(); + AtomicReference physicalError = new AtomicReference<>(); + SqlExplainLevel level = + mode == ExplainMode.COST + ? SqlExplainLevel.ALL_ATTRIBUTES + : SqlExplainLevel.EXPPLAN_ATTRIBUTES; + + try (Hook.Closeable closeable = + Hook.PLAN_BEFORE_IMPLEMENTATION.addThread( + obj -> { + try { + RelRoot relRoot = (RelRoot) obj; + RelJsonWriter physicalWriter = new RelJsonWriter(); + relRoot.rel.explain(physicalWriter); + physicalJson.set(physicalWriter.asString()); + } catch (Exception e) { + physicalError.set(e); + } + })) { + // triggers the hook + OpenSearchRelRunners.run(context, rel); + } + + if (physicalError.get() != null) { + throw physicalError.get(); + } + + // Parse JSON strings to objects for structured output + Object logicalTree = objectMapper.readValue(logicalJson, Object.class); + Object physicalTree = objectMapper.readValue(physicalJson.get(), Object.class); + + // Parse sourceBuilder JSON if present in physical plan + parseSourceBuilderInPhysicalTree(physicalTree); + + ExplainResponseNodeV2 response = + new ExplainResponseNodeV2(logicalJson, physicalJson.get(), null); + response.setLogicalTree(logicalTree); + response.setPhysicalTree(physicalTree); + + listener.onResponse(new ExplainResponse(response)); + } catch (Exception e) { + // RelJsonWriter can't handle some custom types (e.g., SystemLimitType enum) + listener.onFailure( + new UnsupportedOperationException( + "Cannot serialize plan to json_tree format: " + e.getMessage(), e)); + return; + } } else { - SqlExplainLevel level = - mode == ExplainMode.COST - ? SqlExplainLevel.ALL_ATTRIBUTES - : SqlExplainLevel.EXPPLAN_ATTRIBUTES; - String logical = RelOptUtil.toString(rel, level); - AtomicReference physical = new AtomicReference<>(); - AtomicReference javaCode = new AtomicReference<>(); - try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { - if (mode == ExplainMode.EXTENDED) { - getCodegenInHook(javaCode); - CalcitePlanContext.skipEncoding.set(true); + // Original string format for json/yaml + if (mode == ExplainMode.SIMPLE) { + String logical = RelOptUtil.toString(rel, SqlExplainLevel.NO_ATTRIBUTES); + listener.onResponse( + new ExplainResponse(new ExplainResponseNodeV2(logical, null, null))); + } else { + SqlExplainLevel level = + mode == ExplainMode.COST + ? SqlExplainLevel.ALL_ATTRIBUTES + : SqlExplainLevel.EXPPLAN_ATTRIBUTES; + String logical = RelOptUtil.toString(rel, level); + AtomicReference physical = new AtomicReference<>(); + AtomicReference javaCode = new AtomicReference<>(); + try (Hook.Closeable closeable = getPhysicalPlanInHook(physical, level)) { + if (mode == ExplainMode.EXTENDED) { + getCodegenInHook(javaCode); + CalcitePlanContext.skipEncoding.set(true); + } + // triggers the hook + OpenSearchRelRunners.run(context, rel); } - // triggers the hook - OpenSearchRelRunners.run(context, rel); + listener.onResponse( + new ExplainResponse( + new ExplainResponseNodeV2(logical, physical.get(), javaCode.get()))); } - listener.onResponse( - new ExplainResponse( - new ExplainResponseNodeV2(logical, physical.get(), javaCode.get()))); } } catch (Exception e) { listener.onFailure(e); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 52d64fb5a73..f3773cfe246 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -33,6 +33,7 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.externalize.RelJsonWriter; import org.apache.calcite.rel.externalize.RelWriterImpl; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -105,13 +106,29 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { + // Build explain string with context and request builder info String explainString = String.valueOf(pushDownContext); - if (pw instanceof RelWriterImpl) { - // Only add request builder to the explain plan - explainString += ", " + pushDownContext.createRequestBuilder(); + if (pw instanceof RelJsonWriter) { + // For JSON output, add structured items + super.explainTerms(pw); + if (!pushDownContext.isEmpty()) { + pw.item("PushDownContext", explainString); + try { + OpenSearchRequestBuilder requestBuilder = pushDownContext.createRequestBuilder(); + pw.item("sourceBuilder", requestBuilder.getSourceBuilder().toString()); + } catch (Exception e) { + // Ignore if request builder cannot be created + } + } + return pw; + } else { + // For text output, use original chained format + if (pw instanceof RelWriterImpl && !pushDownContext.isEmpty()) { + explainString += ", " + pushDownContext.createRequestBuilder(); + } + return super.explainTerms(pw) + .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } - return super.explainTerms(pw) - .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } protected Integer getQuerySizeLimit() { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 973f00c54cb..678ed58f37f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -10,7 +10,9 @@ import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; +import java.util.LinkedHashMap; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; import org.apache.calcite.rel.RelNode; @@ -243,6 +245,18 @@ protected Object buildYamlObject(ExecutionEngine.ExplainResponse response) { new JsonResponseFormatter<>(PRETTY) { @Override protected Object buildJsonObject(ExecutionEngine.ExplainResponse response) { + // For json_tree format, use parsed tree objects instead of strings + if (response.getCalcite() != null + && response.getCalcite().getLogicalTree() != null) { + Map result = new LinkedHashMap<>(); + Map calcite = new LinkedHashMap<>(); + calcite.put("logical", response.getCalcite().getLogicalTree()); + if (response.getCalcite().getPhysicalTree() != null) { + calcite.put("physical", response.getCalcite().getPhysicalTree()); + } + result.put("calcite", calcite); + return result; + } return response; } }; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ea353066cb0..6ad9032432c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -99,7 +99,12 @@ private AbstractPlan plan( .isExplain(request.isExplainRequest()) .fetchSize(request.getFetchSize()) .highlightConfig(request.getHighlightConfig()) - .format(request.getFormat()) + .format( + request.getFormat() != null && !request.getFormat().isEmpty() + ? org.opensearch.sql.protocol.response.format.Format.ofExplain( + request.getFormat()) + .orElse(null) + : null) .explainMode(request.getExplainMode()) .build())); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index d2c1f610238..62503923eee 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -21,6 +21,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; +import org.opensearch.sql.protocol.response.format.Format; /** Build {@link Statement} from PPL Query. */ @RequiredArgsConstructor @@ -45,12 +46,15 @@ public Statement visitPplStatement(OpenSearchPPLParser.PplStatementContext ctx) } if (ctx.explainStatement() != null) { if (ctx.explainStatement().explainMode() == null) { - return new Explain(query, PPL); + return new Explain(query, PPL, null, context.format); } else { - return new Explain(query, PPL, ctx.explainStatement().explainMode().getText()); + return new Explain( + query, PPL, ctx.explainStatement().explainMode().getText(), context.format); } } else { - return context.isExplain ? new Explain(query, PPL, context.explainMode) : query; + return context.isExplain + ? new Explain(query, PPL, context.explainMode, context.format) + : query; } } @@ -74,7 +78,7 @@ public static class StatementBuilderContext { /** Highlight config from the API request. */ private final HighlightConfig highlightConfig; - private final String format; + private final Format format; private final String explainMode; } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 0825f6d1def..37398bbf7e8 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -22,7 +22,6 @@ import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; -import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; @@ -134,15 +133,6 @@ public void testExecuteCsvFormatShouldPass() { @Test public void testExplainShouldPass() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); - return null; - }) - .when(queryService) - .explain(any(), any(), any(), any(), any()); - pplService.explain( new PPLQueryRequest("search source=t a=1", null, EXPLAIN), new ResponseListener() { From c301013db1fbece78e17d6b7288592514ced4533 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:33:43 -0700 Subject: [PATCH 27/41] Support PPL timewrap command (#5241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add PPL timewrap command for time-period comparison Implement the timewrap command that reshapes timechart output by wrapping each time period into a separate data series, enabling day-over-day, week-over-week, and other recurring interval comparisons. Signed-off-by: Jialiang Li Signed-off-by: Kai Huang * Update metadata.rst doctest for timewrap_test index Add timewrap_test to SHOW TABLES expected output (25 tables). Signed-off-by: Jialiang Li Signed-off-by: Kai Huang * Refactor timewrap: extract TimewrapUtils, add precise calendar arithmetic - Extract all timewrap helper methods to TimewrapUtils.java in calcite/utils/ - Add precise EXTRACT-based period computation for month/quarter/year - Add cumDaysBeforeMonth with leap year CASE expression for precise quarter offset - Month/quarter/year period assignment now uses calendar arithmetic instead of approximate fixed-length conversions Signed-off-by: Jialiang Li Signed-off-by: Kai Huang * Replace Calcite PIVOT with post-processing pivot, add series parameter - Remove Calcite PIVOT from timewrap: no more MAX_PERIODS limit or crash risk - Add post-processing pivot in execution engine: dynamically builds columns from actual data using HashMap grouping - Add series parameter: series=relative (default), series=short (s0, s1) - Add series=exact grammar support (falls back to short at runtime) - Add time_format grammar support for series=exact - Benchmark: 1000 period columns in ~50ms (Calcite PIVOT crashed at 1000) - 32 IT tests, all using verifySchema + verifyDataRows Signed-off-by: Jialiang Li Signed-off-by: Kai Huang * Support timewrap on the analytics-engine route The timewrap pivot (turning the unpivoted [display_ts, value, __base_offset__, __period__] rows into Splunk-style period columns) was done as post-processing in OpenSearchExecutionEngine.buildResultSet, gated on CalcitePlanContext thread-locals. The analytics-engine route executes the RelNode via AnalyticsExecutionEngine and never reaches that code, so timewrap queries came back unpivoted (CalciteTimewrapCommandIT 0/32 on the analytics route). Extract the pivot into a shared core helper (TimewrapPivot) and call it from both execution engines so they produce identical output. AnalyticsExecutionEngine captures the timewrap signals at execute() entry via TimewrapSignals because the result callback runs on a different worker thread than the planning thread that set the thread-locals. CalciteTimewrapCommandIT: 32/32 on both the Calcite/v2 and analytics-engine routes. Signed-off-by: Kai Huang * Address timewrap review: thread-local leak, anonymizer, floor-divide, docs, unit tests Fixes from PR review: - Thread-local leak: visitTimewrap sets timewrap signals on CalcitePlanContext thread-locals during planning, but they were only cleared on the execute success path. explain and exception-after-planning paths leaked the signals onto the pooled worker thread, so the next query was wrongly treated as timewrap. Centralize clearing in CalcitePlanContext.clearTimewrapSignals(), called from CalcitePlanContext.run()'s finally (v2 path) and from RestUnifiedQueryAction's finally (analytics path, which doesn't use run()). Both engines' existing clears now route through the one helper. - Anonymizer: add visitTimewrap to PPLQueryDataAnonymizer so anonymized query logs include the timewrap command instead of silently dropping the pipe segment. span magnitude is masked; align/series are constrained keywords. - align=now off-by-one: baseOffset used integer DIVIDE (truncates toward zero), giving wrong period labels when the reference is below maxEpoch (future-dated data under align=now). Use FLOOR(double-divide) for true floor division. - Docs/dead code: remove the unimplemented "max 20 period columns" claim from timewrap.md and the unused MAX_PERIODS constant (the pivot is intentionally unbounded). - Tests: add CalcitePPLTimewrapTest (verifyLogical + verifyPPLToSparkSQL) per the PPL-command checklist, and a timewrap case in PPLQueryDataAnonymizerTest. CalciteTimewrapCommandIT: 32/32 on both the Calcite/v2 and analytics routes. Signed-off-by: Kai Huang * Address timewrap review: remove dead time_format plumbing, optimize pivot Follow-up cleanups from PR review: - series=exact: keep the documented fallback to short "s" naming, but remove the write-only CalcitePlanContext.timewrapTimeFormat thread-local (set in visitTimewrap and cleared in clearTimewrapSignals, but never read). The AST Timewrap.timeFormat field is retained for when exact formatting is implemented. Collapse the identical short/exact switch arms with a comment. - TimewrapPivot: precompute each period's display name once into a Map instead of re-running split/parseInt/switch inside the per-row loop (was O(rows x valueCols)). Collapse the unreachable two-pass column-index scan into one — visitTimewrap always emits the bookkeeping columns last, so the fallback scan never ran. CalciteTimewrapCommandIT: 32/32 on both the Calcite/v2 and analytics routes. Signed-off-by: Kai Huang * Add CalciteExplainIT explain-plan tests for timewrap Two explain-plan tests following the existing CalciteExplainIT pattern (explainQueryYaml + loadExpectedPlan + assertYamlEqualsIgnoreId): - testExplainTimewrap: fixed-length unit (1day), epoch-based arithmetic path. - testExplainTimewrapMonth: variable-length unit (1month), EXTRACT-based calendar arithmetic path. Both pin the align=end reference with a WHERE @timestamp <= upper bound so the base_offset literal is deterministic across runs. Signed-off-by: Kai Huang * Add no-pushdown expected plans for timewrap explain tests CalciteNoPushdownIT re-runs CalciteExplainIT with pushdown disabled, which loads expected plans from expectedOutput/calcite_no_pushdown/ instead of expectedOutput/calcite/. The two timewrap explain tests were missing their no-pushdown variants, causing "resource ... not found" failures in CI. Logical plans match the pushdown variants; the physical plans differ (EnumerableLimit/EnumerableSort + full index scan, no pushed-down aggregation). Signed-off-by: Kai Huang * Reject variable-length units in spanToSeconds spanToSeconds is only used on the fixed-length timewrap path (s/m/h/d/w), where the second count is exact. The M/q/y arms returned approximate 30/91/365-day values that were never used for wrapping (those units go through the calendar arithmetic path with exact leap-year handling). Throw instead so the approximation can never be silently consumed. Signed-off-by: Kai Huang * Narrow timestamp-parse catches to DateTimeParseException Both LocalDateTime.parse and Instant.parse throw only DateTimeParseException on bad input; catch that specific type instead of Exception so genuine programming errors surface. Signed-off-by: Kai Huang * Add leak-guard test for timewrap pivot signals Asserts that after a timewrap query runs through CalcitePlanContext.run, the pivot thread-locals are cleared so the next non-timewrap query on the same pooled thread is not wrongly pivoted (no __base_offset__/__period__ artifacts). Verified the test fails if run()'s clearTimewrapSignals guard is removed. Signed-off-by: Kai Huang --------- Signed-off-by: Jialiang Li Signed-off-by: Kai Huang --- .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/tree/Timewrap.java | 48 ++ .../sql/calcite/CalcitePlanContext.java | 24 + .../sql/calcite/CalciteRelNodeVisitor.java | 146 ++++ .../sql/calcite/utils/TimewrapPivot.java | 208 +++++ .../sql/calcite/utils/TimewrapUtils.java | 356 ++++++++ .../analytics/AnalyticsExecutionEngine.java | 19 +- .../executor/analytics/TimewrapSignals.java | 61 ++ .../utils/TimewrapSignalsLeakTest.java | 82 ++ docs/category.json | 1 + docs/user/dql/metadata.rst | 5 +- docs/user/ppl/cmd/timewrap.md | 175 ++++ doctest/test_data/timewrap_test.json | 24 + doctest/test_docs.py | 1 + doctest/test_mapping/timewrap_test.json | 19 + .../sql/calcite/remote/CalciteExplainIT.java | 24 + .../remote/CalciteTimewrapCommandIT.java | 762 ++++++++++++++++++ .../sql/legacy/SQLIntegTestCase.java | 7 +- .../calcite/explain_timewrap.yaml | 18 + .../calcite/explain_timewrap_month.yaml | 18 + .../calcite_no_pushdown/explain_timewrap.yaml | 22 + .../explain_timewrap_month.yaml | 22 + .../src/test/resources/timewrap_test.json | 66 ++ .../executor/OpenSearchExecutionEngine.java | 18 + .../plugin/rest/RestUnifiedQueryAction.java | 11 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 6 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 23 + .../opensearch/sql/ppl/parser/AstBuilder.java | 35 + .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 20 + .../ppl/calcite/CalcitePPLTimewrapTest.java | 174 ++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 11 + 31 files changed, 2404 insertions(+), 7 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java create mode 100644 core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java create mode 100644 core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java create mode 100644 core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java create mode 100644 docs/user/ppl/cmd/timewrap.md create mode 100644 doctest/test_data/timewrap_test.json create mode 100644 doctest/test_mapping/timewrap_test.json create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml create mode 100644 integ-test/src/test/resources/timewrap_test.json create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index be02547a2da..6d8415fd7ea 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -91,6 +91,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; @@ -301,6 +302,10 @@ public T visitChart(Chart node, C context) { return visitChildren(node, context); } + public T visitTimewrap(Timewrap node, C context) { + return visitChildren(node, context); + } + public T visitRegex(Regex node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java b/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java new file mode 100644 index 00000000000..88c151db331 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Timewrap.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.SpanUnit; + +/** AST node representing the timewrap command. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Timewrap extends UnresolvedPlan { + private final SpanUnit unit; + private final int value; + private final String align; // "end" or "now" + private final String series; // "relative", "short", or "exact" + private final String timeFormat; // format string for series=exact, nullable + private final Literal spanLiteral; // original span literal for display + + private UnresolvedPlan child; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitTimewrap(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 7ca7ab09304..3f81cdbae4a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -44,6 +44,18 @@ public class CalcitePlanContext { /** This thread local variable is only used to skip script encoding in script pushdown. */ public static final ThreadLocal skipEncoding = ThreadLocal.withInitial(() -> false); + /** When true, the execution engine strips all-null columns from the result (used by timewrap). */ + public static final ThreadLocal stripNullColumns = ThreadLocal.withInitial(() -> false); + + /** + * Timewrap span unit name for column renaming in the execution engine. When set, the execution + * engine uses __base_offset__ to compute absolute period names (e.g., "501days_before"). + */ + public static final ThreadLocal timewrapUnitName = new ThreadLocal<>(); + + /** Timewrap series mode: "relative", "short", or "exact". */ + public static final ThreadLocal timewrapSeries = new ThreadLocal<>(); + /** Thread-local switch that tells whether the current query prefers legacy behavior. */ private static final ThreadLocal legacyPreferredFlag = ThreadLocal.withInitial(() -> true); @@ -169,6 +181,7 @@ public static void run(Runnable action, Settings settings) { action.run(); } finally { legacyPreferredFlag.remove(); + clearTimewrapSignals(); } } @@ -179,6 +192,17 @@ public static boolean isLegacyPreferred() { return legacyPreferredFlag.get(); } + /** + * Resets the timewrap thread-locals set by {@code CalciteRelNodeVisitor.visitTimewrap}. Called + * from the query lifecycle's {@code finally} on every path (execute, explain, and exceptions) so + * the signals never leak onto the next query that reuses this pooled worker thread. + */ + public static void clearTimewrapSignals() { + stripNullColumns.set(false); + timewrapUnitName.set(null); + timewrapSeries.set(null); + } + public void putRexLambdaRefMap(Map candidateMap) { this.rexLambdaRefMap.putAll(candidateMap); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index c4bb8bcd910..13ccdf15197 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Streams; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -160,6 +161,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Trendline.TrendlineType; import org.opensearch.sql.ast.tree.Union; @@ -176,6 +178,7 @@ import org.opensearch.sql.calcite.utils.JoinAndLookupUtils; import org.opensearch.sql.calcite.utils.PPLHintUtils; import org.opensearch.sql.calcite.utils.PlanUtils; +import org.opensearch.sql.calcite.utils.TimewrapUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.calcite.utils.WildcardUtils; import org.opensearch.sql.common.error.ErrorCode; @@ -187,6 +190,7 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.HighlightExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.expression.parse.RegexCommonUtils; import org.opensearch.sql.utils.ParseUtils; @@ -3823,6 +3827,148 @@ public RelNode visitChart(Chart node, CalcitePlanContext context) { return relBuilder.peek(); } + @Override + public RelNode visitTimewrap(Timewrap node, CalcitePlanContext context) { + visitChildren(node, context); + + // Signal the execution engine to strip all-null columns and rename with absolute offsets + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set( + TimewrapUtils.unitBaseName(node.getUnit(), node.getValue()) + "|_before"); + CalcitePlanContext.timewrapSeries.set(node.getSeries()); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + List fieldNames = + b.peek().getRowType().getFieldNames().stream().filter(f -> !isMetadataField(f)).toList(); + String tsFieldName = fieldNames.get(0); + List valueFieldNames = fieldNames.subList(1, fieldNames.size()); + + boolean variableLength = TimewrapUtils.isVariableLengthUnit(node.getUnit()); + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + + RexNode periodNum; + RexNode displayTimestamp; + RexNode baseOffset; + + if (variableLength) { + // --- Variable-length units (month, quarter, year): EXTRACT-based calendar arithmetic --- + RexNode tsField = b.field(tsFieldName); + RexNode tsUnitNum = + TimewrapUtils.calendarUnitNumber(rx, tsField, node.getUnit(), node.getValue()); + + b.projectPlus(b.aggregateCall(SqlStdOperatorTable.MAX, tsField).over().as("__max_ts__")); + RexNode maxTs = b.field("__max_ts__"); + RexNode maxUnitNum = + TimewrapUtils.calendarUnitNumber(rx, maxTs, node.getUnit(), node.getValue()); + + periodNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall(SqlStdOperatorTable.MINUS, maxUnitNum, tsUnitNum), + rx.makeExactLiteral(BigDecimal.ONE, bigintType)); + + RexNode tsEpoch = + rx.makeCast(bigintType, rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, tsField), true); + RexNode unitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, tsField, node.getUnit()); + RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MINUS, tsEpoch, unitStartEpoch); + RexNode maxUnitStartEpoch = TimewrapUtils.calendarUnitStartEpoch(rx, maxTs, node.getUnit()); + RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, maxUnitStartEpoch, offsetSec); + displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch); + + long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000; + Long referenceEpoch = null; + if ("end".equals(node.getAlign())) { + referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node); + } + if (referenceEpoch == null) { + referenceEpoch = nowEpochSec; + } + long refUnitNum = + TimewrapUtils.calendarUnitNumberFromEpoch( + referenceEpoch, node.getUnit(), node.getValue()); + RexNode refUnitNumLit = rx.makeBigintLiteral(BigDecimal.valueOf(refUnitNum)); + baseOffset = rx.makeCall(SqlStdOperatorTable.MINUS, refUnitNumLit, maxUnitNum); + + } else { + // --- Fixed-length units (sec, min, hr, day, week): epoch-based arithmetic --- + long spanSec = TimewrapUtils.spanToSeconds(node.getUnit(), node.getValue()); + + RexNode tsEpochExpr = + rx.makeCast( + bigintType, + rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, b.field(tsFieldName)), + true); + b.projectPlus( + b.alias(tsEpochExpr, "__ts_epoch__"), + b.aggregateCall(SqlStdOperatorTable.MAX, tsEpochExpr).over().as("__max_epoch__")); + + RexNode tsEpoch = b.field("__ts_epoch__"); + RexNode maxEpoch = b.field("__max_epoch__"); + RexNode spanLit = rx.makeBigintLiteral(BigDecimal.valueOf(spanSec)); + + RexNode diff = rx.makeCall(SqlStdOperatorTable.MINUS, maxEpoch, tsEpoch); + periodNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall(SqlStdOperatorTable.DIVIDE, diff, spanLit), + rx.makeExactLiteral(BigDecimal.ONE, bigintType)); + + RexNode offsetSec = rx.makeCall(SqlStdOperatorTable.MOD, tsEpoch, spanLit); + RexNode latestPeriodStart = + rx.makeCall( + SqlStdOperatorTable.MINUS, + maxEpoch, + rx.makeCall(SqlStdOperatorTable.MOD, maxEpoch, spanLit)); + RexNode displayEpoch = rx.makeCall(SqlStdOperatorTable.PLUS, latestPeriodStart, offsetSec); + displayTimestamp = rx.makeCall(PPLBuiltinOperators.FROM_UNIXTIME, displayEpoch); + + long nowEpochSec = context.functionProperties.getQueryStartClock().millis() / 1000; + Long referenceEpoch = null; + if ("end".equals(node.getAlign())) { + referenceEpoch = TimewrapUtils.extractTimestampUpperBound(node); + } + if (referenceEpoch == null) { + referenceEpoch = nowEpochSec; + } + RexNode refLit = rx.makeBigintLiteral(BigDecimal.valueOf(referenceEpoch)); + // Floor-divide (ref - maxEpoch) by span: integer DIVIDE truncates toward zero, which is wrong + // when the reference is below maxEpoch (e.g. align=now over future-dated data) — it would + // shift period labels by one across the latest/before/after boundary. Cast to DOUBLE and + // FLOOR + // to get true floor division, then back to BIGINT. + RelDataType doubleType = rx.getTypeFactory().createSqlType(SqlTypeName.DOUBLE); + RexNode refDiff = rx.makeCall(SqlStdOperatorTable.MINUS, refLit, maxEpoch); + RexNode refDiffDouble = rx.makeCast(doubleType, refDiff, true); + baseOffset = + rx.makeCast( + bigintType, + rx.makeCall( + SqlStdOperatorTable.FLOOR, + rx.makeCall(SqlStdOperatorTable.DIVIDE, refDiffDouble, spanLit)), + true); + } + + // Step 3: Project [display_timestamp, value_columns..., base_offset, period] + // base_offset is included in the group key so it survives the PIVOT + List projections = new ArrayList<>(); + projections.add(b.alias(displayTimestamp, tsFieldName)); + for (String vf : valueFieldNames) { + projections.add(b.field(vf)); + } + projections.add(b.alias(baseOffset, "__base_offset__")); + projections.add(b.alias(periodNum, "__period__")); + b.project(projections); + + // Step 4: Sort by offset, then period (execution engine will pivot) + // No Calcite PIVOT -- the execution engine pivots dynamically after reading all rows. + // Output schema: [display_timestamp, value_columns..., __base_offset__, __period__] + b.sort(b.field(tsFieldName), b.field("__period__")); + + return b.peek(); + } + /** * Aggregate by column split then rank by grand total (summed value of each category). The output * is [col-split, grand-total, row-number] diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java new file mode 100644 index 00000000000..c4712ff957e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapPivot.java @@ -0,0 +1,208 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.executor.ExecutionEngine.Schema.Column; + +/** + * Pivots the unpivoted rows produced by {@code CalciteRelNodeVisitor.visitTimewrap} into the + * Splunk-style period columns timewrap returns. + * + *

The timewrap RelNode intentionally does NOT pivot — the set of period columns is only known + * once all rows are read. Instead it emits rows of shape {@code [display_ts, value_col(s)..., + * __base_offset__, __period__]} and signals this post-processing step via the thread-locals on + * {@link CalcitePlanContext}. This helper performs the dynamic pivot so both execution engines (the + * v2 {@code OpenSearchExecutionEngine} and the analytics-route {@code AnalyticsExecutionEngine}) + * produce identical output. + */ +public final class TimewrapPivot { + + private TimewrapPivot() {} + + /** Result of a pivot: the rebuilt columns and rows. */ + public record Result(List columns, List values) {} + + /** + * Returns true when the current query is a timewrap query whose results need pivoting. Reads the + * {@link CalcitePlanContext} thread-locals set by {@code visitTimewrap}. + */ + public static boolean isTimewrap() { + return Boolean.TRUE.equals(CalcitePlanContext.stripNullColumns.get()) + && CalcitePlanContext.timewrapUnitName.get() != null; + } + + /** + * Pivots the unpivoted timewrap rows into period columns. If the input is empty or the + * bookkeeping columns are absent, the input is returned unchanged. + * + * @param columns the unpivoted columns {@code [display_ts, value_col(s)..., __base_offset__, + * __period__]} + * @param values the unpivoted rows + * @param unitInfo the {@code visitTimewrap} unit descriptor + * ("spanValue|singular|plural|_before"); null means this is not a timewrap query and the + * input is returned unchanged + * @param seriesMode the timewrap {@code series} mode (relative / short / exact); may be null + */ + public static Result pivot( + List columns, List values, String unitInfo, String seriesMode) { + if (unitInfo == null || values.isEmpty()) { + return new Result(columns, values); + } + + // Locate the bookkeeping and value columns. visitTimewrap always emits + // [display_ts, value_col(s)..., __base_offset__, __period__], so column 0 is the timestamp, + // __period__/__base_offset__ are bookkeeping, and every other column is a value column. + int tsIdx = 0; + int periodIdx = -1; + int baseOffsetIdx = -1; + List valueIdxs = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + String name = columns.get(i).getName(); + if ("__period__".equals(name)) periodIdx = i; + else if ("__base_offset__".equals(name)) baseOffsetIdx = i; + else if (i > 0) valueIdxs.add(i); + } + if (periodIdx < 0 || baseOffsetIdx < 0) { + return new Result(columns, values); + } + + // Read __base_offset__ (constant across all rows). + long baseOffset = 0; + ExprValue boVal = values.getFirst().tupleValue().get("__base_offset__"); + if (boVal != null && !boVal.isNull()) { + baseOffset = boVal.longValue(); + } + + // Collect distinct periods (sorted descending = oldest first in output) and precompute each + // period's name once. The name depends only on (period, baseOffset, unitInfo, seriesMode), so + // computing it inside the per-row loop would repeat the same split/parse/switch + // O(rows x valueCols) times. + Set periodSet = new TreeSet<>(Collections.reverseOrder()); + for (ExprValue row : values) { + ExprValue pv = row.tupleValue().get("__period__"); + if (pv != null && !pv.isNull()) { + periodSet.add(pv.longValue()); + } + } + List periods = new ArrayList<>(periodSet); + Map periodNames = new HashMap<>(); + for (long period : periods) { + periodNames.put(period, renameTimewrapPeriod(period, baseOffset, unitInfo, seriesMode)); + } + + // Value column names. + List valueColNames = new ArrayList<>(); + for (int vi : valueIdxs) { + valueColNames.add(columns.get(vi).getName()); + } + + // Build output column names: [ts, val1_period1, val1_period2, ..., val2_period1, ...]. + // Splunk order: for each period, all value columns (oldest period first). + List outColNames = new ArrayList<>(); + outColNames.add(columns.get(tsIdx).getName()); + List outColTypes = new ArrayList<>(); + outColTypes.add(columns.get(tsIdx).getExprType()); + + for (long period : periods) { + for (int vi = 0; vi < valueColNames.size(); vi++) { + outColNames.add(valueColNames.get(vi) + "_" + periodNames.get(period)); + outColTypes.add(columns.get(valueIdxs.get(vi)).getExprType()); + } + } + + // Group rows by display_ts, pivot periods into columns. LinkedHashMap preserves the + // ts-sorted insertion order Calcite produced. + Map> pivoted = new LinkedHashMap<>(); + String tsColName = columns.get(tsIdx).getName(); + for (ExprValue row : values) { + Map tuple = row.tupleValue(); + String tsKey = tuple.get(tsColName).toString(); + long period = tuple.get("__period__").longValue(); + + Map outRow = + pivoted.computeIfAbsent( + tsKey, + k -> { + Map r = new LinkedHashMap<>(); + r.put(outColNames.get(0), tuple.get(tsColName)); + // Initialize all period columns to null. + for (int i = 1; i < outColNames.size(); i++) { + r.put(outColNames.get(i), ExprNullValue.of()); + } + return r; + }); + + // Fill in the value for this period. + String periodName = periodNames.get(period); + for (int vi = 0; vi < valueColNames.size(); vi++) { + String colName = valueColNames.get(vi) + "_" + periodName; + ExprValue val = tuple.get(valueColNames.get(vi)); + if (val != null) { + outRow.put(colName, val); + } + } + } + + // Build output. + List outColumns = new ArrayList<>(); + for (int i = 0; i < outColNames.size(); i++) { + outColumns.add(new Column(outColNames.get(i), null, outColTypes.get(i))); + } + List outValues = new ArrayList<>(); + for (Map outRow : pivoted.values()) { + outValues.add(ExprTupleValue.fromExprValueMap(outRow)); + } + return new Result(outColumns, outValues); + } + + /** + * Generates a period name from a relative period number and base offset. Returns the suffix only + * (no value prefix). E.g., "2days_before", "latest_day", "s2". unitInfo format: + * "spanValue|singular|plural|_before". + */ + private static String renameTimewrapPeriod( + long relativePeriod, long baseOffset, String unitInfo, String seriesMode) { + String[] parts = unitInfo.split("\\|", -1); + if (parts.length < 4) return String.valueOf(relativePeriod); + int spanValue = Integer.parseInt(parts[0]); + String singular = parts[1]; + String plural = parts[2]; + + long absolutePeriod = (baseOffset + relativePeriod - 1) * spanValue; + + String mode = seriesMode == null ? "relative" : seriesMode; + + return switch (mode) { + // series=exact (+ time_format) is not yet implemented; it intentionally falls back to the + // short "s" naming. TODO: format the period start date with time_format. + case "short", "exact" -> "s" + absolutePeriod; + default -> { + if (absolutePeriod == 0) { + yield "latest_" + singular; + } else if (absolutePeriod > 0) { + String unit = absolutePeriod == 1 ? singular : plural; + yield absolutePeriod + unit + "_before"; + } else { + long absPeriod = Math.abs(absolutePeriod); + String unit = absPeriod == 1 ? singular : plural; + yield absPeriod + unit + "_after"; + } + } + }; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java new file mode 100644 index 00000000000..8c172027314 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/TimewrapUtils.java @@ -0,0 +1,356 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.tree.Filter; +import org.opensearch.sql.ast.tree.Timewrap; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +/** Utility methods for the timewrap command's Calcite plan construction. */ +public class TimewrapUtils { + + /** Check if the span unit is variable-length (month, quarter, year). */ + public static boolean isVariableLengthUnit(SpanUnit unit) { + return "M".equals(unit.getName()) || "q".equals(unit.getName()) || "y".equals(unit.getName()); + } + + /** + * Convert a fixed-length span unit and value to seconds. Only fixed-length units (second, minute, + * hour, day, week) are supported; variable-length units (month, quarter, year) have no exact + * second count and must use the calendar arithmetic path ({@link #calendarUnitNumber} / {@link + * #calendarUnitStartEpoch}) instead. + */ + public static long spanToSeconds(SpanUnit unit, int value) { + return switch (unit.getName()) { + case "s" -> value; + case "m" -> value * 60L; + case "h" -> value * 3_600L; + case "d" -> value * 86_400L; + case "w" -> value * 7L * 86_400L; + case "M", "q", "y" -> + throw new IllegalArgumentException( + "Variable-length unit '" + + unit.getName() + + "' cannot be converted to a fixed number of seconds; use the calendar" + + " arithmetic path instead"); + default -> + throw new SemanticCheckException("Unsupported time unit in timewrap: " + unit.getName()); + }; + } + + /** + * Get the timescale base name for column naming. Returns "spanValue|singular|plural" e.g., + * "1|day|days". + */ + public static String unitBaseName(SpanUnit unit, int value) { + String singular = + switch (unit.getName()) { + case "s" -> "second"; + case "m" -> "minute"; + case "h" -> "hour"; + case "d" -> "day"; + case "w" -> "week"; + case "M" -> "month"; + case "q" -> "quarter"; + case "y" -> "year"; + default -> "period"; + }; + String plural = singular + "s"; + return value + "|" + singular + "|" + plural; + } + + /** + * Compute a calendar unit number for a timestamp as a Calcite RexNode. For months: year*12 + + * month. For quarters: year*4 + quarter. For years: year. Divided by spanValue. + */ + public static RexNode calendarUnitNumber( + RexBuilder rx, RexNode tsField, SpanUnit unit, int spanValue) { + RexNode year = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("YEAR"), tsField); + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + + RexNode unitNum; + switch (unit.getName()) { + case "M" -> { + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + unitNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + year, + rx.makeExactLiteral(BigDecimal.valueOf(12), bigintType)), + month); + } + case "q" -> { + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + RexNode quarter = + rx.makeCall( + SqlStdOperatorTable.DIVIDE, + rx.makeCall( + SqlStdOperatorTable.MINUS, + month, + rx.makeExactLiteral(BigDecimal.ONE, bigintType)), + rx.makeExactLiteral(BigDecimal.valueOf(3), bigintType)); + unitNum = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + year, + rx.makeExactLiteral(BigDecimal.valueOf(4), bigintType)), + quarter); + } + case "y" -> unitNum = year; + default -> throw new SemanticCheckException("Not a variable-length unit: " + unit.getName()); + } + + if (spanValue > 1) { + unitNum = + rx.makeCall( + SqlStdOperatorTable.DIVIDE, + unitNum, + rx.makeExactLiteral(BigDecimal.valueOf(spanValue), bigintType)); + } + return unitNum; + } + + /** + * Compute the epoch seconds of the start of the calendar unit containing a timestamp. Month: + * first day of the month. Quarter: first day of the quarter (precise with leap year). Year: Jan + * 1. + */ + public static RexNode calendarUnitStartEpoch(RexBuilder rx, RexNode tsField, SpanUnit unit) { + RelDataType bigintType = rx.getTypeFactory().createSqlType(SqlTypeName.BIGINT); + RexNode tsEpoch = + rx.makeCast(bigintType, rx.makeCall(PPLBuiltinOperators.UNIX_TIMESTAMP, tsField), true); + RexNode hour = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("HOUR"), tsField); + RexNode minute = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MINUTE"), tsField); + RexNode second = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("SECOND"), tsField); + RexNode one = rx.makeExactLiteral(BigDecimal.ONE, bigintType); + RexNode sec86400 = rx.makeExactLiteral(BigDecimal.valueOf(86400), bigintType); + + RexNode timeWithinDay = + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.PLUS, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + hour, + rx.makeExactLiteral(BigDecimal.valueOf(3600), bigintType)), + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + minute, + rx.makeExactLiteral(BigDecimal.valueOf(60), bigintType))), + second); + + if ("M".equals(unit.getName())) { + RexNode dayOfMonth = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DAY"), tsField); + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + rx.makeCall(SqlStdOperatorTable.MINUS, dayOfMonth, one), + sec86400)), + timeWithinDay); + } else if ("y".equals(unit.getName())) { + RexNode doy = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DOY"), tsField); + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall( + SqlStdOperatorTable.MULTIPLY, + rx.makeCall(SqlStdOperatorTable.MINUS, doy, one), + sec86400)), + timeWithinDay); + } else { + // Quarter: precise day-within-quarter via cumulative day lookup + leap year + RexNode doy = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("DOY"), tsField); + RexNode month = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("MONTH"), tsField); + RexNode year = rx.makeCall(PPLBuiltinOperators.EXTRACT, rx.makeLiteral("YEAR"), tsField); + + RexNode monthsIntoQuarter = + rx.makeCall( + SqlStdOperatorTable.MOD, + rx.makeCall(SqlStdOperatorTable.MINUS, month, one), + rx.makeExactLiteral(BigDecimal.valueOf(3), bigintType)); + RexNode quarterStartMonth = rx.makeCall(SqlStdOperatorTable.MINUS, month, monthsIntoQuarter); + + RexNode cumDaysBeforeQS = cumDaysBeforeMonth(rx, quarterStartMonth, year, bigintType); + RexNode quarterStartDOY = rx.makeCall(SqlStdOperatorTable.PLUS, cumDaysBeforeQS, one); + RexNode dayWithinQuarter = rx.makeCall(SqlStdOperatorTable.MINUS, doy, quarterStartDOY); + + return rx.makeCall( + SqlStdOperatorTable.MINUS, + rx.makeCall( + SqlStdOperatorTable.MINUS, + tsEpoch, + rx.makeCall(SqlStdOperatorTable.MULTIPLY, dayWithinQuarter, sec86400)), + timeWithinDay); + } + } + + /** + * Build a CASE expression for cumulative days before a given month, with leap year handling. + * Month 1→0, Month 2→31, Month 3→59+leap, ..., Month 12→334+leap. + */ + public static RexNode cumDaysBeforeMonth( + RexBuilder rx, RexNode month, RexNode year, RelDataType bigintType) { + RexNode mod4 = + rx.makeCall( + SqlStdOperatorTable.MOD, year, rx.makeExactLiteral(BigDecimal.valueOf(4), bigintType)); + RexNode mod100 = + rx.makeCall( + SqlStdOperatorTable.MOD, + year, + rx.makeExactLiteral(BigDecimal.valueOf(100), bigintType)); + RexNode mod400 = + rx.makeCall( + SqlStdOperatorTable.MOD, + year, + rx.makeExactLiteral(BigDecimal.valueOf(400), bigintType)); + RexNode zero = rx.makeExactLiteral(BigDecimal.ZERO, bigintType); + RexNode isLeap = + rx.makeCall( + SqlStdOperatorTable.CASE, + rx.makeCall( + SqlStdOperatorTable.AND, + rx.makeCall(SqlStdOperatorTable.EQUALS, mod4, zero), + rx.makeCall( + SqlStdOperatorTable.OR, + rx.makeCall(SqlStdOperatorTable.NOT_EQUALS, mod100, zero), + rx.makeCall(SqlStdOperatorTable.EQUALS, mod400, zero))), + rx.makeExactLiteral(BigDecimal.ONE, bigintType), + zero); + + int[] cumDays = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}; + + List caseArgs = new ArrayList<>(); + for (int m = 1; m <= 12; m++) { + caseArgs.add( + rx.makeCall( + SqlStdOperatorTable.EQUALS, + month, + rx.makeExactLiteral(BigDecimal.valueOf(m), bigintType))); + RexNode days = rx.makeExactLiteral(BigDecimal.valueOf(cumDays[m - 1]), bigintType); + if (m >= 3) { + days = rx.makeCall(SqlStdOperatorTable.PLUS, days, isLeap); + } + caseArgs.add(days); + } + caseArgs.add(zero); + + return rx.makeCall(SqlStdOperatorTable.CASE, caseArgs.toArray(new RexNode[0])); + } + + /** Compute calendar unit number from an epoch at plan time (Java). */ + public static long calendarUnitNumberFromEpoch(long epochSec, SpanUnit unit, int spanValue) { + java.time.Instant instant = java.time.Instant.ofEpochSecond(epochSec); + java.time.ZonedDateTime zdt = instant.atZone(java.time.ZoneOffset.UTC); + long unitNum; + switch (unit.getName()) { + case "M" -> unitNum = zdt.getYear() * 12L + zdt.getMonthValue(); + case "q" -> unitNum = zdt.getYear() * 4L + (zdt.getMonthValue() - 1) / 3; + case "y" -> unitNum = zdt.getYear(); + default -> throw new SemanticCheckException("Not a variable-length unit: " + unit.getName()); + } + return unitNum / spanValue; + } + + /** + * Walk the AST from a Timewrap node to the deepest Filter node and extract the timestamp upper + * bound. The frontend time picker always appends the timestamp filter as the first pipe (closest + * to source), making it the deepest Filter in the AST chain: + * + * 

+   *   Timewrap -> Chart -> [user filters] -> Filter(@timestamp >= X AND @timestamp <= Y) -> Source
+   *
+ * + * We walk all Filter nodes and return the last (deepest) timestamp upper bound found. This + * ensures user filters like `where age > 30` between timechart and the time picker filter don't + * interfere. + */ + public static Long extractTimestampUpperBound(Timewrap node) { + Node current = node; + Long lastBound = null; + while (current != null && !current.getChild().isEmpty()) { + current = current.getChild().get(0); + if (current instanceof Filter filter) { + Long bound = findUpperBound(filter.getCondition()); + if (bound != null) { + lastBound = bound; + } + } + } + return lastBound; + } + + private static Long findUpperBound(UnresolvedExpression expr) { + if (expr instanceof And and) { + Long left = findUpperBound(and.getLeft()); + Long right = findUpperBound(and.getRight()); + if (left != null && right != null) return Math.min(left, right); + return left != null ? left : right; + } + if (expr instanceof Compare cmp) { + String op = cmp.getOperator(); + if (("<=".equals(op) || "<".equals(op)) && isTimestampField(cmp.getLeft())) { + return parseTimestampLiteral(cmp.getRight()); + } + if ((">=".equals(op) || ">".equals(op)) && isTimestampField(cmp.getRight())) { + return parseTimestampLiteral(cmp.getLeft()); + } + } + return null; + } + + private static boolean isTimestampField(UnresolvedExpression expr) { + if (expr instanceof Field field) { + String name = field.getField().toString(); + return "@timestamp".equals(name) || "timestamp".equals(name); + } + return false; + } + + private static Long parseTimestampLiteral(UnresolvedExpression expr) { + if (expr instanceof Literal lit && lit.getValue() instanceof String s) { + try { + java.time.LocalDateTime ldt = + java.time.LocalDateTime.parse( + s, java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + return ldt.toEpochSecond(java.time.ZoneOffset.UTC); + } catch (java.time.format.DateTimeParseException e) { + try { + return java.time.Instant.parse(s).getEpochSecond(); + } catch (java.time.format.DateTimeParseException ignored) { + return null; + } + } + } + return null; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index 18d87fc18d0..b98204fb31a 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -112,6 +112,11 @@ public void execute( ProfileContext profileCtx = QueryProfiling.current(); long execStart = System.nanoTime(); + // Capture the timewrap pivot signals now: they live in CalcitePlanContext thread-locals set by + // visitTimewrap on this (planning) thread, but the result callback below runs on a different + // analytics worker thread. Clearing them here keeps this thread's thread-locals clean. + TimewrapSignals timewrap = TimewrapSignals.captureAndClear(); + planExecutor.execute( plan, queryCtx, @@ -125,10 +130,12 @@ public void onResponse(Iterable rows) { List fields = plan.getRowType().getFieldList(); List results = convertRows(rows, fields); Schema schema = buildSchema(fields, results); + QueryResponse response = + timewrap.pivot(new QueryResponse(schema, results, Cursor.None)); profileCtx .getOrCreateMetric(MetricName.EXECUTE) .set(System.nanoTime() - execStart); - listener.onResponse(new QueryResponse(schema, results, Cursor.None)); + listener.onResponse(response); } catch (Exception e) { listener.onFailure(e); } @@ -172,6 +179,9 @@ public void executeWithProfile( ProfileContext profileCtx = QueryProfiling.current(); long execStart = System.nanoTime(); + // See execute(): capture the timewrap pivot signals on this planning thread. + TimewrapSignals timewrap = TimewrapSignals.captureAndClear(); + planExecutor.executeWithProfile( plan, queryCtx, @@ -182,7 +192,7 @@ public void onResponse(ProfiledResult result) { // ProfiledResult delivers the profile on BOTH success and failure paths // so users get stage timing visibility even when a query partially fails. profileCtx.getOrCreateMetric(MetricName.EXECUTE).set(System.nanoTime() - execStart); - QueryResponse response = buildProfiledResponse(plan, result); + QueryResponse response = buildProfiledResponse(plan, result, timewrap); listener.onResponse(response); } catch (Exception e) { listener.onFailure(e); @@ -196,12 +206,13 @@ public void onFailure(Exception e) { }); } - private QueryResponse buildProfiledResponse(RelNode plan, ProfiledResult result) { + private QueryResponse buildProfiledResponse( + RelNode plan, ProfiledResult result, TimewrapSignals timewrap) { List fields = plan.getRowType().getFieldList(); List results = result.rows() != null ? convertRows(result.rows(), fields) : List.of(); Schema schema = buildSchema(fields, results); - QueryResponse response = new QueryResponse(schema, results, Cursor.None); + QueryResponse response = timewrap.pivot(new QueryResponse(schema, results, Cursor.None)); response.setProfile(result.profile()); if (!result.isSuccess()) { response.setError(result.failure()); diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java b/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java new file mode 100644 index 00000000000..d7907e93d87 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/TimewrapSignals.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.analytics; + +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.utils.TimewrapPivot; +import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; +import org.opensearch.sql.executor.ExecutionEngine.Schema; + +/** + * A snapshot of the timewrap pivot signals that {@code CalciteRelNodeVisitor.visitTimewrap} stores + * in {@link CalcitePlanContext} thread-locals. + * + *

On the analytics route, planning and result-conversion run on different threads — the result + * callback fires on an analytics worker pool, not the SQL worker thread that planned the query. + * Capturing the signals at execute() entry (on the planning thread) and carrying them into the + * callback keeps the pivot correct across that thread hop. Capturing also clears the thread-locals + * so they don't leak onto the planning thread's next query. + */ +public final class TimewrapSignals { + + private final boolean active; + private final String unitName; + private final String series; + + private TimewrapSignals(boolean active, String unitName, String series) { + this.active = active; + this.unitName = unitName; + this.series = series; + } + + /** Captures the current thread's timewrap signals and clears the thread-locals. */ + public static TimewrapSignals captureAndClear() { + boolean active = TimewrapPivot.isTimewrap(); + String unitName = CalcitePlanContext.timewrapUnitName.get(); + String series = CalcitePlanContext.timewrapSeries.get(); + CalcitePlanContext.clearTimewrapSignals(); + return new TimewrapSignals(active, unitName, series); + } + + /** + * Applies the timewrap pivot to {@code response} if this snapshot is from a timewrap query; + * otherwise returns it unchanged. The returned response carries over the input's profile/error. + */ + public QueryResponse pivot(QueryResponse response) { + if (!active) { + return response; + } + TimewrapPivot.Result pivoted = + TimewrapPivot.pivot( + response.getSchema().getColumns(), response.getResults(), unitName, series); + QueryResponse out = + new QueryResponse(new Schema(pivoted.columns()), pivoted.values(), response.getCursor()); + out.setProfile(response.getProfile()); + out.setError(response.getError()); + return out; + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java b/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java new file mode 100644 index 00000000000..eef98cdc4bf --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/utils/TimewrapSignalsLeakTest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.executor.ExecutionEngine.Schema.Column; + +/** + * Leak guard for the timewrap pivot signals. {@code visitTimewrap} stashes its pivot state in the + * static {@link CalcitePlanContext} thread-locals read by {@link TimewrapPivot#isTimewrap()}. These + * are set on a pooled worker thread, so if they are not cleared at the end of the query lifecycle + * they would leak onto the next, non-timewrap query that reuses the same thread and wrongly pivot + * its result. This test asserts the lifecycle guard ({@code CalcitePlanContext.run}'s finally) + * clears them. + */ +public class TimewrapSignalsLeakTest { + + @AfterEach + public void cleanUp() { + CalcitePlanContext.clearTimewrapSignals(); + } + + @Test + public void timewrapSignalsDoNotLeakOntoNextQueryOnSameThread() { + Settings settings = mock(Settings.class); + lenient().when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); + + // First "query" behaves like a timewrap query: it sets the pivot signals mid-flight, exactly as + // visitTimewrap does. run()'s finally must clear them before returning. + CalcitePlanContext.run( + () -> { + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set("1|day|days|_before"); + CalcitePlanContext.timewrapSeries.set("relative"); + }, + settings); + + // Same thread, next query: the pivot gate must be closed. + assertFalse( + TimewrapPivot.isTimewrap(), + "timewrap signals leaked onto the next query on the same pooled thread"); + + // And a plain (non-timewrap) result must pass through the pivot untouched — no __base_offset__ + // or __period__ artifact columns, same row. + List columns = + List.of( + new Column("host", null, ExprCoreType.STRING), + new Column("count", null, ExprCoreType.LONG)); + List rows = + List.of(ExprValueUtils.tupleValue(ImmutableMap.of("host", "h1", "count", 42L))); + + TimewrapPivot.Result result = + TimewrapPivot.pivot( + columns, + rows, + CalcitePlanContext.timewrapUnitName.get(), + CalcitePlanContext.timewrapSeries.get()); + + assertEquals( + List.of("host", "count"), + result.columns().stream().map(Column::getName).toList(), + "non-timewrap result gained artifact columns after a prior timewrap query"); + assertEquals(rows, result.values()); + } +} diff --git a/docs/category.json b/docs/category.json index ac1dda966d2..1e45f3e65bb 100644 --- a/docs/category.json +++ b/docs/category.json @@ -45,6 +45,7 @@ "user/ppl/cmd/syntax.md", "user/ppl/cmd/chart.md", "user/ppl/cmd/timechart.md", + "user/ppl/cmd/timewrap.md", "user/ppl/cmd/top.md", "user/ppl/cmd/trendline.md", "user/ppl/cmd/transpose.md", diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index e4f55ef1b3e..92aaa0c7db0 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -35,7 +35,7 @@ Example 1: Show All Indices Information SQL query:: os> SHOW TABLES LIKE '%' - fetched rows / total rows = 24/24 + fetched rows / total rows = 25/25 +----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------+ | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | |----------------+-------------+-------------------+------------+---------+----------+------------+-----------+---------------------------+----------------| @@ -48,7 +48,7 @@ SQL query:: | docTestCluster | null | events_many_hosts | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | events_null | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | json_test | BASE TABLE | null | null | null | null | null | null | - | docTestCluster | null | mvcombine_data | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | mvcombine_data | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nested | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | occupation | BASE TABLE | null | null | null | null | null | null | @@ -59,6 +59,7 @@ SQL query:: | docTestCluster | null | time_data | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | time_data2 | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | time_test | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | timewrap_test | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | weblogs | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | | docTestCluster | null | work_information | BASE TABLE | null | null | null | null | null | null | diff --git a/docs/user/ppl/cmd/timewrap.md b/docs/user/ppl/cmd/timewrap.md new file mode 100644 index 00000000000..6e4b9414a78 --- /dev/null +++ b/docs/user/ppl/cmd/timewrap.md @@ -0,0 +1,175 @@ + +# timewrap + +The `timewrap` command reshapes `timechart` output by wrapping each time period into a separate data series. This enables side-by-side comparisons of the same metric across recurring time intervals, such as day-over-day or week-over-week analysis. + +## Syntax + +The `timewrap` command has the following syntax: + +```syntax +... | timechart ... | timewrap [align=end|now] +``` + +## Parameters + +The `timewrap` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | The wrapping interval, in the form `[int]`. If the integer is omitted, `1` is assumed. For example, `1day`, `2week`, or just `day`. For a complete list of supported time units, see [Time units](#time-units). | +| `align` | Optional | Controls the reference point for period alignment and column naming. Default is `end`. `end` aligns to the search end time (the upper bound of the `where` clause on `@timestamp`, or current time if no time filter). `now` always aligns to the current query execution time. | + +## Notes + +The following considerations apply when using the `timewrap` command: + +* The `timewrap` command must follow a `timechart` command. It is a post-processing command that reshapes timechart output. +* Column names follow the pattern `__before` for periods before the reference point, `_latest_` for the period containing the reference point, and `__after` for periods after the reference point. +* Column order is oldest first (leftmost) to newest (rightmost). +* Only columns with data are included in the output. Unused period columns are automatically removed. +* Incomplete periods (where data does not span the full wrap interval) show `null` for missing time offsets. +* Only `timechart` without the `BY` clause is currently supported. The `BY` clause is a future enhancement. + +### Time units + +The following time units are available for the `` parameter: + +* Seconds (`s`, `sec`, `second`, `secs`, `seconds`) +* Minutes (`m`, `min`, `minute`, `mins`, `minutes`) --- note: `m` means minutes, not months +* Hours (`h`, `hr`, `hour`, `hrs`, `hours`) +* Days (`d`, `day`, `days`) +* Weeks (`w`, `week`, `weeks`) + +Variable-length time units (`month`, `quarter`, `year`) are not yet supported. + +### Column naming + +Column names are constructed from the aggregation function name and an absolute time offset from the reference point: + +| Position relative to reference | Column name format | Example | +| --- | --- | --- | +| Before the reference point | `__before` | `sum(requests)_2days_before` | +| At the reference point | `_latest_` | `sum(requests)_latest_day` | +| After the reference point | `__after` | `sum(requests)_1day_after` | + +## Example 1: Day-over-day comparison + +The following query compares the sum of requests per 6-hour interval across 3 days: + +```ppl +source=timewrap_test +| where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-03 18:00:00' +| timechart span=6h sum(requests) +| timewrap 1day +``` + +```text +fetched rows / total rows = 4/4 ++---------------------+----------------------------+---------------------------+--------------------------+ +| @timestamp | sum(requests)_2days_before | sum(requests)_1day_before | sum(requests)_latest_day | +|---------------------+----------------------------+---------------------------+--------------------------| +| 2024-07-03 00:00:00 | 180 | 205 | 165 | +| 2024-07-03 06:00:00 | 240 | 260 | 225 | +| 2024-07-03 12:00:00 | 310 | 330 | 285 | +| 2024-07-03 18:00:00 | 190 | 215 | 165 | ++---------------------+----------------------------+---------------------------+--------------------------+ +``` + +Each column represents one day of data. The `latest_day` column contains the most recent period (July 3). The `2days_before` column contains the oldest period (July 1). + +## Example 2: Comparing averages across 2 days + +The following query compares the average requests per 6-hour interval: + +```ppl +source=timewrap_test +| where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-02 18:00:00' +| timechart span=6h avg(requests) +| timewrap 1day +``` + +```text +fetched rows / total rows = 4/4 ++---------------------+---------------------------+--------------------------+ +| @timestamp | avg(requests)_1day_before | avg(requests)_latest_day | +|---------------------+---------------------------+--------------------------| +| 2024-07-02 00:00:00 | 90.0 | 102.5 | +| 2024-07-02 06:00:00 | 120.0 | 130.0 | +| 2024-07-02 12:00:00 | 155.0 | 165.0 | +| 2024-07-02 18:00:00 | 95.0 | 107.5 | ++---------------------+---------------------------+--------------------------+ +``` + +## Example 3: Single day produces one period + +When all data fits within a single wrap interval, only one period column is produced: + +```ppl +source=timewrap_test +| where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-01 18:00:00' +| timechart span=6h sum(requests) +| timewrap 1day +``` + +```text +fetched rows / total rows = 4/4 ++---------------------+--------------------------+ +| @timestamp | sum(requests)_latest_day | +|---------------------+--------------------------| +| 2024-07-01 00:00:00 | 180 | +| 2024-07-01 06:00:00 | 240 | +| 2024-07-01 12:00:00 | 310 | +| 2024-07-01 18:00:00 | 190 | ++---------------------+--------------------------+ +``` + +## Example 4: Count events day-over-day + +```ppl +source=timewrap_test +| where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-03 18:00:00' +| timechart span=6h count() +| timewrap 1day +``` + +```text +fetched rows / total rows = 4/4 ++---------------------+----------------------+---------------------+--------------------+ +| @timestamp | count()_2days_before | count()_1day_before | count()_latest_day | +|---------------------+----------------------+---------------------+--------------------| +| 2024-07-03 00:00:00 | 2 | 2 | 2 | +| 2024-07-03 06:00:00 | 2 | 2 | 2 | +| 2024-07-03 12:00:00 | 2 | 2 | 2 | +| 2024-07-03 18:00:00 | 2 | 2 | 2 | ++---------------------+----------------------+---------------------+--------------------+ +``` + +## Example 5: Comparing errors across 2 days + +```ppl +source=timewrap_test +| where @timestamp >= '2024-07-02 00:00:00' and @timestamp <= '2024-07-03 18:00:00' +| timechart span=6h sum(errors) +| timewrap 1day +``` + +```text +fetched rows / total rows = 4/4 ++---------------------+-------------------------+------------------------+ +| @timestamp | sum(errors)_1day_before | sum(errors)_latest_day | +|---------------------+-------------------------+------------------------| +| 2024-07-03 00:00:00 | 4 | 1 | +| 2024-07-03 06:00:00 | 6 | 3 | +| 2024-07-03 12:00:00 | 9 | 6 | +| 2024-07-03 18:00:00 | 3 | 1 | ++---------------------+-------------------------+------------------------+ +``` + +## Limitations + +The `timewrap` command has the following limitations: + +* The `timewrap` command must follow a `timechart` command. Using it after any other command results in an error. +* Only `timechart` without the `BY` clause is supported. The `BY` clause (column split) is a future enhancement. +* Variable-length time units (`month`, `quarter`, `year`) are not yet supported. Use fixed-length units (`s`, `m`, `h`, `d`, `w`). diff --git a/doctest/test_data/timewrap_test.json b/doctest/test_data/timewrap_test.json new file mode 100644 index 00000000000..d38977f629f --- /dev/null +++ b/doctest/test_data/timewrap_test.json @@ -0,0 +1,24 @@ +{"@timestamp":"2024-07-01T00:00:00Z","host":"web-01","requests":100,"errors":2} +{"@timestamp":"2024-07-01T06:00:00Z","host":"web-01","requests":150,"errors":5} +{"@timestamp":"2024-07-01T12:00:00Z","host":"web-01","requests":200,"errors":3} +{"@timestamp":"2024-07-01T18:00:00Z","host":"web-01","requests":120,"errors":1} +{"@timestamp":"2024-07-01T00:00:00Z","host":"web-02","requests":80,"errors":0} +{"@timestamp":"2024-07-01T06:00:00Z","host":"web-02","requests":90,"errors":1} +{"@timestamp":"2024-07-01T12:00:00Z","host":"web-02","requests":110,"errors":2} +{"@timestamp":"2024-07-01T18:00:00Z","host":"web-02","requests":70,"errors":0} +{"@timestamp":"2024-07-02T00:00:00Z","host":"web-01","requests":110,"errors":3} +{"@timestamp":"2024-07-02T06:00:00Z","host":"web-01","requests":160,"errors":4} +{"@timestamp":"2024-07-02T12:00:00Z","host":"web-01","requests":210,"errors":6} +{"@timestamp":"2024-07-02T18:00:00Z","host":"web-01","requests":130,"errors":2} +{"@timestamp":"2024-07-02T00:00:00Z","host":"web-02","requests":95,"errors":1} +{"@timestamp":"2024-07-02T06:00:00Z","host":"web-02","requests":100,"errors":2} +{"@timestamp":"2024-07-02T12:00:00Z","host":"web-02","requests":120,"errors":3} +{"@timestamp":"2024-07-02T18:00:00Z","host":"web-02","requests":85,"errors":1} +{"@timestamp":"2024-07-03T00:00:00Z","host":"web-01","requests":90,"errors":1} +{"@timestamp":"2024-07-03T06:00:00Z","host":"web-01","requests":140,"errors":2} +{"@timestamp":"2024-07-03T12:00:00Z","host":"web-01","requests":180,"errors":4} +{"@timestamp":"2024-07-03T18:00:00Z","host":"web-01","requests":100,"errors":1} +{"@timestamp":"2024-07-03T00:00:00Z","host":"web-02","requests":75,"errors":0} +{"@timestamp":"2024-07-03T06:00:00Z","host":"web-02","requests":85,"errors":1} +{"@timestamp":"2024-07-03T12:00:00Z","host":"web-02","requests":105,"errors":2} +{"@timestamp":"2024-07-03T18:00:00Z","host":"web-02","requests":65,"errors":0} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 6283252065f..e179c85eb54 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -59,6 +59,7 @@ 'time_data2': 'time_test_data2.json', 'time_test': 'time_test.json', 'mvcombine_data': 'mvcombine.json', + 'timewrap_test': 'timewrap_test.json', } DEBUG_MODE = os.environ.get('DOCTEST_DEBUG', 'false').lower() == 'true' diff --git a/doctest/test_mapping/timewrap_test.json b/doctest/test_mapping/timewrap_test.json new file mode 100644 index 00000000000..8222188feb4 --- /dev/null +++ b/doctest/test_mapping/timewrap_test.json @@ -0,0 +1,19 @@ +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + }, + "host": { + "type": "keyword" + }, + "requests": { + "type": "integer" + }, + "errors": { + "type": "integer" + } + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index d475f11427d..9244981125f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -565,6 +565,30 @@ public void testExplainTimechartPerDay() throws IOException { assertTrue(result.contains("per_day(cpu_usage)=[SUM($0)]")); } + @Test + public void testExplainTimewrap() throws IOException { + // Pin the align=end reference with a WHERE upper bound so the base_offset literal is + // deterministic (otherwise it falls back to the query clock). + var result = + explainQueryYaml( + "source=events | where @timestamp <= '2024-07-03 18:00:00'" + + " | timechart span=6h avg(cpu_usage) | timewrap 1day"); + String expected = loadExpectedPlan("explain_timewrap.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testExplainTimewrapMonth() throws IOException { + // Variable-length unit (month) exercises the EXTRACT-based calendar arithmetic branch, which + // produces a different plan from the fixed-length epoch-based path above. + var result = + explainQueryYaml( + "source=events | where @timestamp <= '2024-07-03 18:00:00'" + + " | timechart span=1d avg(cpu_usage) | timewrap 1month"); + String expected = loadExpectedPlan("explain_timewrap_month.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void noPushDownForAggOnWindow() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java new file mode 100644 index 00000000000..9383e56f490 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimewrapCommandIT.java @@ -0,0 +1,762 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteTimewrapCommandIT extends PPLIntegTestCase { + + // Standard WHERE clause covering all test data — simulates frontend time picker + private static final String WHERE_ALL = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-04 06:00:00'"; + private static final String WHERE_JUL1_TO_JUL3 = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-03 18:00:00'"; + private static final String WHERE_JUL2_TO_JUL3 = + " | where @timestamp >= '2024-07-02 00:00:00' and @timestamp <= '2024-07-03 18:00:00'"; + private static final String WHERE_JUL1_ONLY = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-01 18:00:00'"; + private static final String WHERE_JUL1_TO_JUL2 = + " | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <= '2024-07-02 18:00:00'"; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.TIMEWRAP_TEST); + } + + // --- Day-over-day with different aggregations --- + + @Test + public void testTimewrapDayOverDayWithSum() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + @Test + public void testTimewrapDayOverDayWithAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h avg(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_3days_before", "double"), + schema("avg(requests)_2days_before", "double"), + schema("avg(requests)_1day_before", "double"), + schema("avg(requests)_latest_day", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 90.0, 102.5, 82.5, 40.0), + rows("2024-07-04 06:00:00", 120.0, 130.0, 112.5, 50.0), + rows("2024-07-04 12:00:00", null, 155.0, 165.0, 142.5), + rows("2024-07-04 18:00:00", null, 95.0, 107.5, 82.5)); + } + + @Test + public void testTimewrapDayOverDayWithCount() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + WHERE_ALL + " | timechart span=6h count() | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_3days_before", "bigint"), + schema("count()_2days_before", "bigint"), + schema("count()_1day_before", "bigint"), + schema("count()_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 2, 2, 2, 2), + rows("2024-07-04 06:00:00", 2, 2, 2, 2), + rows("2024-07-04 12:00:00", null, 2, 2, 2), + rows("2024-07-04 18:00:00", null, 2, 2, 2)); + } + + @Test + public void testTimewrapWithDifferentAggField() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(errors) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_3days_before", "bigint"), + schema("sum(errors)_2days_before", "bigint"), + schema("sum(errors)_1day_before", "bigint"), + schema("sum(errors)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 2, 4, 1, 0), + rows("2024-07-04 06:00:00", 6, 6, 3, 1), + rows("2024-07-04 12:00:00", null, 5, 9, 6), + rows("2024-07-04 18:00:00", null, 1, 3, 1)); + } + + // --- Incomplete period / null fill --- + + @Test + public void testTimewrapIncompletePeriodNullFill() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- Different timescales --- + + @Test + public void testTimewrapWeekSpanSinglePeriod() throws IOException { + // 3 days of daily data in 1 week -> single period, 3 rows + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=1day sum(requests) | timewrap 1week"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_week", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 920), + rows("2024-07-02 00:00:00", 1010), + rows("2024-07-03 00:00:00", 840)); + } + + @Test + public void testTimewrapTwelveHourSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_72hours_before", "bigint"), + schema("sum(requests)_60hours_before", "bigint"), + schema("sum(requests)_48hours_before", "bigint"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 310, 205, 330, 165, 285, 80), + rows("2024-07-04 06:00:00", 240, 190, 260, 215, 225, 165, 100)); + } + + @Test + public void testTimewrapWithMinuteSpan() throws IOException { + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:04:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_4minutes_before", "bigint"), + schema("count()_3minutes_before", "bigint"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:04:00", 1, 1, 1, 1, 1)); + } + + // --- WHERE clause with different time ranges --- + + @Test + public void testTimewrapWithWhereThreeDays() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapWithWhereTwoDays() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 205, 165), + rows("2024-07-03 06:00:00", 260, 225), + rows("2024-07-03 12:00:00", 330, 285), + rows("2024-07-03 18:00:00", 215, 165)); + } + + @Test + public void testTimewrapWithWhereSingleDay() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_ONLY + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 180), + rows("2024-07-01 06:00:00", 240), + rows("2024-07-01 12:00:00", 310), + rows("2024-07-01 18:00:00", 190)); + } + + @Test + public void testTimewrapWithWhereAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h avg(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_1day_before", "double"), + schema("avg(requests)_latest_day", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 00:00:00", 90.0, 102.5), + rows("2024-07-02 06:00:00", 120.0, 130.0), + rows("2024-07-02 12:00:00", 155.0, 165.0), + rows("2024-07-02 18:00:00", 95.0, 107.5)); + } + + @Test + public void testTimewrapWithWhere12hSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 12:00:00", 180, 310, 205, 330), + rows("2024-07-02 18:00:00", 240, 190, 260, 215)); + } + + @Test + public void testTimewrapWithWhereCount() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h count() | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_2days_before", "bigint"), + schema("count()_1day_before", "bigint"), + schema("count()_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 2, 2, 2), + rows("2024-07-03 06:00:00", 2, 2, 2), + rows("2024-07-03 12:00:00", 2, 2, 2), + rows("2024-07-03 18:00:00", 2, 2, 2)); + } + + @Test + public void testTimewrapWithWhereErrors() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(errors) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_1day_before", "bigint"), + schema("sum(errors)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 4, 1), + rows("2024-07-03 06:00:00", 6, 3), + rows("2024-07-03 12:00:00", 9, 6), + rows("2024-07-03 18:00:00", 3, 1)); + } + + // --- WHERE upper bound above data: shifts column numbers --- + + @Test + public void testTimewrapWithWhereUpperBoundAboveData() throws IOException { + // WHERE upper bound = July 10 (~5.75 days after max data July 4 06:00) + // baseOffset = floor(Jul10/86400) - floor(Jul4_06/86400) = 5 + // periodFromNow for oldest(rel=4): (5+4-1)*1=8, newest(rel=1): (5+1-1)*1=5 + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-10 00:00:00' | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_8days_before", "bigint"), + schema("sum(requests)_7days_before", "bigint"), + schema("sum(requests)_6days_before", "bigint"), + schema("sum(requests)_5days_before", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- align=end vs align=now --- + + @Test + public void testTimewrapAlignEndIsDefault() throws IOException { + JSONObject resultDefault = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day"); + JSONObject resultEnd = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day align=end"); + + verifySchema( + resultEnd, + schema("@timestamp", "timestamp"), + schema("sum(requests)_3days_before", "bigint"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultEnd, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + verifyDataRowsInOrder( + resultDefault, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + @Test + public void testTimewrapAlignNow() throws IOException { + // align=now uses current time — column names are dynamic + // Extract actual column names from the result for verification + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_ALL + + " | timechart span=6h sum(requests) | timewrap 1day align=now"); + + // Get actual column names from result schema + org.json.JSONArray schemaArr = result.getJSONArray("schema"); + String c1 = schemaArr.getJSONObject(1).getString("name"); + String c2 = schemaArr.getJSONObject(2).getString("name"); + String c3 = schemaArr.getJSONObject(3).getString("name"); + String c4 = schemaArr.getJSONObject(4).getString("name"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema(c1, "bigint"), + schema(c2, "bigint"), + schema(c3, "bigint"), + schema(c4, "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-04 00:00:00", 180, 205, 165, 80), + rows("2024-07-04 06:00:00", 240, 260, 225, 100), + rows("2024-07-04 12:00:00", null, 310, 330, 285), + rows("2024-07-04 18:00:00", null, 190, 215, 165)); + } + + // --- Every timescale --- + + @Test + public void testTimewrapSecondSpan() throws IOException { + // 5 events at minute-level, wrap by 1 minute (60sec) + // timechart span=1m gives 3 buckets (00:00, 01:00, 02:00) + // timewrap 1min: each bucket is in a different 1-minute period → 1 offset row, 3 periods + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:02:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:02:00", 1, 1, 1)); + } + + @Test + public void testTimewrapMinuteSpan() throws IOException { + loadIndex(Index.EVENTS); + JSONObject result = + executeQuery( + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-01 00:04:00' | timechart span=1m count() | timewrap 1min"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_4minutes_before", "bigint"), + schema("count()_3minutes_before", "bigint"), + schema("count()_2minutes_before", "bigint"), + schema("count()_1minute_before", "bigint"), + schema("count()_latest_minute", "bigint")); + verifyDataRows(result, rows("2024-07-01 00:04:00", 1, 1, 1, 1, 1)); + } + + @Test + public void testTimewrapHourSpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h sum(requests) | timewrap 12h"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_36hours_before", "bigint"), + schema("sum(requests)_24hours_before", "bigint"), + schema("sum(requests)_12hours_before", "bigint"), + schema("sum(requests)_latest_hour", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 12:00:00", 180, 310, 205, 330), + rows("2024-07-02 18:00:00", 240, 190, 260, 215)); + } + + @Test + public void testTimewrapDaySpan() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapWeekSpan() throws IOException { + // 2 days of daily data in 1 week -> single period, 2 rows + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=1day sum(requests) | timewrap 1week"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_week", "bigint")); + verifyDataRowsInOrder( + result, rows("2024-07-01 00:00:00", 920), rows("2024-07-02 00:00:00", 1010)); + } + + @Test + public void testTimewrapMonthSpan() throws IOException { + // Jul 1-4 only: all data within same month → single month period + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-04 06:00:00' | timechart span=1day sum(requests) | timewrap 1month"); + + verifySchema( + result, schema("@timestamp", "timestamp"), schema("sum(requests)_latest_month", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-01 00:00:00", 920), + rows("2024-07-02 00:00:00", 1010), + rows("2024-07-03 00:00:00", 840), + rows("2024-07-04 00:00:00", 180)); + } + + @Test + public void testTimewrapQuarterSpan() throws IOException { + // Jan 15 (Q1) and Apr 15 (Q2) → 2 quarter periods + // With precise day-within-quarter offset: Jan 15 = day 15 of Q1, Apr 15 = day 15 of Q2 + // Both are at the same offset (day 15) → they align on the same row + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-01-15 00:00:00' and @timestamp <=" + + " '2024-04-15 12:00:00' | timechart span=1day sum(requests) | timewrap 1quarter"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1quarter_before", "bigint"), + schema("sum(requests)_latest_quarter", "bigint")); + // Day 15 of each quarter aligns -- both values on the same row + verifyDataRows(result, rows("2024-04-15 00:00:00", 300, 350)); + } + + @Test + public void testTimewrapYearSpan() throws IOException { + // Jan 15 2024 (300) and Jan 15 2025 (400) -- 2 data points in 2 different years + // timechart span=1year: 2 yearly buckets + // timewrap 1year: 2 periods, 1 offset row + JSONObject result = + executeQuery( + "source=timewrap_test | where @timestamp >= '2024-01-15 12:00:00' and @timestamp <=" + + " '2025-01-15 12:00:00' | timechart span=1year sum(requests) | timewrap 1year"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_1year_before", "bigint"), + schema("sum(requests)_latest_year", "bigint")); + // 2024 yearly sum = all 2024 data in WHERE range; 2025 = Jan 15 only (400) + verifyDataRows(result, rows("2025-01-01 00:00:00", 4050, 400)); + } + + // --- series parameter --- + + @Test + public void testTimewrapSeriesRelativeIsDefault() throws IOException { + // series=relative is the default — same as no series parameter + JSONObject resultDefault = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day"); + JSONObject resultRelative = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day series=relative"); + + verifySchema( + resultRelative, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultRelative, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + verifySchema( + resultDefault, + schema("@timestamp", "timestamp"), + schema("sum(requests)_2days_before", "bigint"), + schema("sum(requests)_1day_before", "bigint"), + schema("sum(requests)_latest_day", "bigint")); + verifyDataRowsInOrder( + resultDefault, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapSeriesShort() throws IOException { + // series=short: columns named _s + // With align=end and WHERE upper bound = Jul 3 18:00, baseOffset=0 + // Periods: oldest=2, middle=1, newest=0 + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL3 + + " | timechart span=6h sum(requests) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(requests)_s2", "bigint"), + schema("sum(requests)_s1", "bigint"), + schema("sum(requests)_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 180, 205, 165), + rows("2024-07-03 06:00:00", 240, 260, 225), + rows("2024-07-03 12:00:00", 310, 330, 285), + rows("2024-07-03 18:00:00", 190, 215, 165)); + } + + @Test + public void testTimewrapSeriesShortWithCount() throws IOException { + // series=short with count aggregation + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h count() | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("count()_s1", "bigint"), + schema("count()_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 2, 2), + rows("2024-07-03 06:00:00", 2, 2), + rows("2024-07-03 12:00:00", 2, 2), + rows("2024-07-03 18:00:00", 2, 2)); + } + + @Test + public void testTimewrapSeriesShortWeekSpan() throws IOException { + // series=short with week span, single period = s0, daily buckets + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=1day sum(requests) | timewrap 1week series=short"); + + verifySchema(result, schema("@timestamp", "timestamp"), schema("sum(requests)_s0", "bigint")); + verifyDataRowsInOrder( + result, rows("2024-07-01 00:00:00", 920), rows("2024-07-02 00:00:00", 1010)); + } + + @Test + public void testTimewrapSeriesShortWithAvg() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL1_TO_JUL2 + + " | timechart span=6h avg(requests) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("avg(requests)_s1", "double"), + schema("avg(requests)_s0", "double")); + verifyDataRowsInOrder( + result, + rows("2024-07-02 00:00:00", 90.0, 102.5), + rows("2024-07-02 06:00:00", 120.0, 130.0), + rows("2024-07-02 12:00:00", 155.0, 165.0), + rows("2024-07-02 18:00:00", 95.0, 107.5)); + } + + @Test + public void testTimewrapSeriesShortWithErrors() throws IOException { + JSONObject result = + executeQuery( + "source=timewrap_test" + + WHERE_JUL2_TO_JUL3 + + " | timechart span=6h sum(errors) | timewrap 1day series=short"); + + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("sum(errors)_s1", "bigint"), + schema("sum(errors)_s0", "bigint")); + verifyDataRowsInOrder( + result, + rows("2024-07-03 00:00:00", 4, 1), + rows("2024-07-03 06:00:00", 6, 3), + rows("2024-07-03 12:00:00", 9, 6), + rows("2024-07-03 18:00:00", 3, 1)); + } + + // BY clause tests are pending -- blocked by timechart BY output format gap. + // See docs/dev/ppl-timewrap-command.md for design options. +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 05a670de393..fc15c908c63 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -989,7 +989,12 @@ public enum Index { "events_traffic", "events_traffic", getMappingFile("events_traffic_index_mapping.json"), - "src/test/resources/events_traffic.json"); + "src/test/resources/events_traffic.json"), + TIMEWRAP_TEST( + "timewrap_test", + "timewrap_test", + "{\"mappings\":{\"properties\":{\"@timestamp\":{\"type\":\"date\"},\"host\":{\"type\":\"keyword\"},\"requests\":{\"type\":\"integer\"},\"errors\":{\"type\":\"integer\"}}}}", + "src/test/resources/timewrap_test.json"); private final String name; private final String type; diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml new file mode 100644 index 00000000000..62b38aafcfe --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))], avg(cpu_usage)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600, MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL, 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 6, 'h')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], expr#6=[-($t3, $t5)], expr#7=[+($t6, $t2)], expr#8=[FROM_UNIXTIME($t7)], expr#9=[1720029600:BIGINT], expr#10=[-($t9, $t3)], expr#11=[CAST($t10):DOUBLE NOT NULL], expr#12=[/($t11, $t4)], expr#13=[FLOOR($t12)], expr#14=[CAST($t13):BIGINT NOT NULL], expr#15=[-($t3, $t1)], expr#16=[/($t15, $t4)], expr#17=[1:BIGINT], expr#18=[+($t16, $t17)], @timestamp=[$t8], avg(cpu_usage)=[$t0], __base_offset__=[$t14], __period__=[$t18]) + EnumerableWindow(window#0=[window(aggs [MAX($1)])]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[UNIX_TIMESTAMP($t0)], expr#3=[CAST($t2):BIGINT NOT NULL], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], avg(cpu_usage)=[$t1], $1=[$t3], $2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(<=($0, '2024-07-03 18:00:00'), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(cpu_usage)=AVG($0)), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":null,"to":"2024-07-03T18:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"6h"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml new file mode 100644 index 00000000000..dc9b7eb1426 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timewrap_month.yaml @@ -0,0 +1,18 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(-(CAST(UNIX_TIMESTAMP(MAX($0) OVER ())):BIGINT NOT NULL, *(-(EXTRACT('DAY', MAX($0) OVER ()), 1), 86400)), +(+(*(EXTRACT('HOUR', MAX($0) OVER ()), 3600), *(EXTRACT('MINUTE', MAX($0) OVER ()), 60)), EXTRACT('SECOND', MAX($0) OVER ()))), -(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, -(-(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, *(-(EXTRACT('DAY', $0), 1), 86400)), +(+(*(EXTRACT('HOUR', $0), 3600), *(EXTRACT('MINUTE', $0), 60)), EXTRACT('SECOND', $0))))))], avg(cpu_usage)=[$1], __base_offset__=[-(24295, +(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())))], __period__=[+(-(+(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())), +(*(EXTRACT('YEAR', $0), 12), EXTRACT('MONTH', $0))), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'd')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[UNIX_TIMESTAMP($t4)], expr#6=[CAST($t5):BIGINT NOT NULL], expr#7=['DAY'], expr#8=[EXTRACT($t7, $t4)], expr#9=[1:BIGINT], expr#10=[-($t8, $t9)], expr#11=[86400:BIGINT], expr#12=[*($t10, $t11)], expr#13=[-($t6, $t12)], expr#14=['HOUR'], expr#15=[EXTRACT($t14, $t4)], expr#16=[3600:BIGINT], expr#17=[*($t15, $t16)], expr#18=['MINUTE'], expr#19=[EXTRACT($t18, $t4)], expr#20=[60:BIGINT], expr#21=[*($t19, $t20)], expr#22=[+($t17, $t21)], expr#23=['SECOND'], expr#24=[EXTRACT($t23, $t4)], expr#25=[+($t22, $t24)], expr#26=[-($t13, $t25)], expr#27=[+($t26, $t2)], expr#28=[FROM_UNIXTIME($t27)], expr#29=[24295:BIGINT], expr#30=['YEAR'], expr#31=[EXTRACT($t30, $t4)], expr#32=[12:BIGINT], expr#33=[*($t31, $t32)], expr#34=['MONTH'], expr#35=[EXTRACT($t34, $t4)], expr#36=[+($t33, $t35)], expr#37=[-($t29, $t36)], expr#38=[-($t36, $t3)], expr#39=[+($t38, $t9)], @timestamp=[$t28], avg(cpu_usage)=[$t1], __base_offset__=[$t37], __period__=[$t39]) + EnumerableWindow(window#0=[window(aggs [MAX($0)])]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[UNIX_TIMESTAMP($t0)], expr#3=[CAST($t2):BIGINT NOT NULL], expr#4=['DAY'], expr#5=[EXTRACT($t4, $t0)], expr#6=[1:BIGINT], expr#7=[-($t5, $t6)], expr#8=[86400:BIGINT], expr#9=[*($t7, $t8)], expr#10=[-($t3, $t9)], expr#11=['HOUR'], expr#12=[EXTRACT($t11, $t0)], expr#13=[3600:BIGINT], expr#14=[*($t12, $t13)], expr#15=['MINUTE'], expr#16=[EXTRACT($t15, $t0)], expr#17=[60:BIGINT], expr#18=[*($t16, $t17)], expr#19=[+($t14, $t18)], expr#20=['SECOND'], expr#21=[EXTRACT($t20, $t0)], expr#22=[+($t19, $t21)], expr#23=[-($t10, $t22)], expr#24=[-($t3, $t23)], expr#25=['YEAR'], expr#26=[EXTRACT($t25, $t0)], expr#27=[12:BIGINT], expr#28=[*($t26, $t27)], expr#29=['MONTH'], expr#30=[EXTRACT($t29, $t0)], expr#31=[+($t28, $t30)], proj#0..1=[{exprs}], $2=[$t24], $3=[$t31]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(<=($0, '2024-07-03 18:00:00'), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},avg(cpu_usage)=AVG($0)), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":null,"to":"2024-07-03T18:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml new file mode 100644 index 00000000000..1775322b2c6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))], avg(cpu_usage)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600, MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL, 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 6, 'h')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[86400:BIGINT], expr#5=[MOD($t3, $t4)], expr#6=[-($t3, $t5)], expr#7=[+($t6, $t2)], expr#8=[FROM_UNIXTIME($t7)], expr#9=[1720029600:BIGINT], expr#10=[-($t9, $t3)], expr#11=[CAST($t10):DOUBLE NOT NULL], expr#12=[/($t11, $t4)], expr#13=[FLOOR($t12)], expr#14=[CAST($t13):BIGINT NOT NULL], expr#15=[-($t3, $t1)], expr#16=[/($t15, $t4)], expr#17=[1:BIGINT], expr#18=[+($t16, $t17)], @timestamp=[$t8], avg(cpu_usage)=[$t0], __base_offset__=[$t14], __period__=[$t18]) + EnumerableWindow(window#0=[window(aggs [MAX($1)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], expr#8=[UNIX_TIMESTAMP($t0)], expr#9=[CAST($t8):BIGINT NOT NULL], expr#10=[86400:BIGINT], expr#11=[MOD($t9, $t10)], avg(cpu_usage)=[$t7], $1=[$t9], $2=[$t11]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[6], expr#17=['h'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=['2024-07-03 18:00:00':EXPR_TIMESTAMP VARCHAR], expr#20=[<=($t1, $t19)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml new file mode 100644 index 00000000000..df3e91e740f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timewrap_month.yaml @@ -0,0 +1,22 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(-(CAST(UNIX_TIMESTAMP(MAX($0) OVER ())):BIGINT NOT NULL, *(-(EXTRACT('DAY', MAX($0) OVER ()), 1), 86400)), +(+(*(EXTRACT('HOUR', MAX($0) OVER ()), 3600), *(EXTRACT('MINUTE', MAX($0) OVER ()), 60)), EXTRACT('SECOND', MAX($0) OVER ()))), -(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, -(-(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, *(-(EXTRACT('DAY', $0), 1), 86400)), +(+(*(EXTRACT('HOUR', $0), 3600), *(EXTRACT('MINUTE', $0), 60)), EXTRACT('SECOND', $0))))))], avg(cpu_usage)=[$1], __base_offset__=[-(24295, +(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())))], __period__=[+(-(+(*(EXTRACT('YEAR', MAX($0) OVER ()), 12), EXTRACT('MONTH', MAX($0) OVER ())), +(*(EXTRACT('YEAR', $0), 12), EXTRACT('MONTH', $0))), 1)]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(@timestamp=[$0], avg(cpu_usage)=[$1]) + LogicalAggregate(group=[{1}], avg(cpu_usage)=[AVG($0)]) + LogicalProject(cpu_usage=[$7], @timestamp0=[SPAN($1, 1, 'd')]) + LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) + LogicalFilter(condition=[<=($1, TIMESTAMP('2024-07-03 18:00:00':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[UNIX_TIMESTAMP($t4)], expr#6=[CAST($t5):BIGINT NOT NULL], expr#7=['DAY'], expr#8=[EXTRACT($t7, $t4)], expr#9=[1:BIGINT], expr#10=[-($t8, $t9)], expr#11=[86400:BIGINT], expr#12=[*($t10, $t11)], expr#13=[-($t6, $t12)], expr#14=['HOUR'], expr#15=[EXTRACT($t14, $t4)], expr#16=[3600:BIGINT], expr#17=[*($t15, $t16)], expr#18=['MINUTE'], expr#19=[EXTRACT($t18, $t4)], expr#20=[60:BIGINT], expr#21=[*($t19, $t20)], expr#22=[+($t17, $t21)], expr#23=['SECOND'], expr#24=[EXTRACT($t23, $t4)], expr#25=[+($t22, $t24)], expr#26=[-($t13, $t25)], expr#27=[+($t26, $t2)], expr#28=[FROM_UNIXTIME($t27)], expr#29=[24295:BIGINT], expr#30=['YEAR'], expr#31=[EXTRACT($t30, $t4)], expr#32=[12:BIGINT], expr#33=[*($t31, $t32)], expr#34=['MONTH'], expr#35=[EXTRACT($t34, $t4)], expr#36=[+($t33, $t35)], expr#37=[-($t29, $t36)], expr#38=[-($t36, $t3)], expr#39=[+($t38, $t9)], @timestamp=[$t28], avg(cpu_usage)=[$t1], __base_offset__=[$t37], __period__=[$t39]) + EnumerableWindow(window#0=[window(aggs [MAX($0)])]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:DOUBLE], expr#6=[CASE($t4, $t5, $t1)], expr#7=[/($t6, $t2)], expr#8=[UNIX_TIMESTAMP($t0)], expr#9=[CAST($t8):BIGINT NOT NULL], expr#10=['DAY'], expr#11=[EXTRACT($t10, $t0)], expr#12=[1:BIGINT], expr#13=[-($t11, $t12)], expr#14=[86400:BIGINT], expr#15=[*($t13, $t14)], expr#16=[-($t9, $t15)], expr#17=['HOUR'], expr#18=[EXTRACT($t17, $t0)], expr#19=[3600:BIGINT], expr#20=[*($t18, $t19)], expr#21=['MINUTE'], expr#22=[EXTRACT($t21, $t0)], expr#23=[60:BIGINT], expr#24=[*($t22, $t23)], expr#25=[+($t20, $t24)], expr#26=['SECOND'], expr#27=[EXTRACT($t26, $t0)], expr#28=[+($t25, $t27)], expr#29=[-($t16, $t28)], expr#30=[-($t9, $t29)], expr#31=['YEAR'], expr#32=[EXTRACT($t31, $t0)], expr#33=[12:BIGINT], expr#34=[*($t32, $t33)], expr#35=['MONTH'], expr#36=[EXTRACT($t35, $t0)], expr#37=[+($t34, $t36)], @timestamp0=[$t0], avg(cpu_usage)=[$t7], $2=[$t30], $3=[$t37]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableAggregate(group=[{1}], agg#0=[$SUM0($0)], agg#1=[COUNT($0)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['d'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=['2024-07-03 18:00:00':EXPR_TIMESTAMP VARCHAR], expr#20=[<=($t1, $t19)], expr#21=[IS NOT NULL($t7)], expr#22=[AND($t20, $t21)], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/timewrap_test.json b/integ-test/src/test/resources/timewrap_test.json new file mode 100644 index 00000000000..5ba82f8fabf --- /dev/null +++ b/integ-test/src/test/resources/timewrap_test.json @@ -0,0 +1,66 @@ +{"index":{"_id":"1"}} +{"@timestamp":"2024-07-01T00:00:00","host":"web-01","requests":100,"errors":2} +{"index":{"_id":"2"}} +{"@timestamp":"2024-07-01T06:00:00","host":"web-01","requests":150,"errors":5} +{"index":{"_id":"3"}} +{"@timestamp":"2024-07-01T12:00:00","host":"web-01","requests":200,"errors":3} +{"index":{"_id":"4"}} +{"@timestamp":"2024-07-01T18:00:00","host":"web-01","requests":120,"errors":1} +{"index":{"_id":"5"}} +{"@timestamp":"2024-07-01T00:00:00","host":"web-02","requests":80,"errors":0} +{"index":{"_id":"6"}} +{"@timestamp":"2024-07-01T06:00:00","host":"web-02","requests":90,"errors":1} +{"index":{"_id":"7"}} +{"@timestamp":"2024-07-01T12:00:00","host":"web-02","requests":110,"errors":2} +{"index":{"_id":"8"}} +{"@timestamp":"2024-07-01T18:00:00","host":"web-02","requests":70,"errors":0} +{"index":{"_id":"9"}} +{"@timestamp":"2024-07-02T00:00:00","host":"web-01","requests":110,"errors":3} +{"index":{"_id":"10"}} +{"@timestamp":"2024-07-02T06:00:00","host":"web-01","requests":160,"errors":4} +{"index":{"_id":"11"}} +{"@timestamp":"2024-07-02T12:00:00","host":"web-01","requests":210,"errors":6} +{"index":{"_id":"12"}} +{"@timestamp":"2024-07-02T18:00:00","host":"web-01","requests":130,"errors":2} +{"index":{"_id":"13"}} +{"@timestamp":"2024-07-02T00:00:00","host":"web-02","requests":95,"errors":1} +{"index":{"_id":"14"}} +{"@timestamp":"2024-07-02T06:00:00","host":"web-02","requests":100,"errors":2} +{"index":{"_id":"15"}} +{"@timestamp":"2024-07-02T12:00:00","host":"web-02","requests":120,"errors":3} +{"index":{"_id":"16"}} +{"@timestamp":"2024-07-02T18:00:00","host":"web-02","requests":85,"errors":1} +{"index":{"_id":"17"}} +{"@timestamp":"2024-07-03T00:00:00","host":"web-01","requests":90,"errors":1} +{"index":{"_id":"18"}} +{"@timestamp":"2024-07-03T06:00:00","host":"web-01","requests":140,"errors":2} +{"index":{"_id":"19"}} +{"@timestamp":"2024-07-03T12:00:00","host":"web-01","requests":180,"errors":4} +{"index":{"_id":"20"}} +{"@timestamp":"2024-07-03T18:00:00","host":"web-01","requests":100,"errors":1} +{"index":{"_id":"21"}} +{"@timestamp":"2024-07-03T00:00:00","host":"web-02","requests":75,"errors":0} +{"index":{"_id":"22"}} +{"@timestamp":"2024-07-03T06:00:00","host":"web-02","requests":85,"errors":1} +{"index":{"_id":"23"}} +{"@timestamp":"2024-07-03T12:00:00","host":"web-02","requests":105,"errors":2} +{"index":{"_id":"24"}} +{"@timestamp":"2024-07-03T18:00:00","host":"web-02","requests":65,"errors":0} +{"index":{"_id":"25"}} +{"@timestamp":"2024-07-04T00:00:00","host":"web-01","requests":50,"errors":0} +{"index":{"_id":"26"}} +{"@timestamp":"2024-07-04T06:00:00","host":"web-01","requests":60,"errors":1} +{"index":{"_id":"27"}} +{"@timestamp":"2024-07-04T00:00:00","host":"web-02","requests":30,"errors":0} +{"index":{"_id":"28"}} +{"@timestamp":"2024-07-04T06:00:00","host":"web-02","requests":40,"errors":0} +{"index":{"_id":"29"}} +{"@timestamp":"2024-01-15T12:00:00","host":"web-01","requests":300,"errors":5} +{"index":{"_id":"30"}} +{"@timestamp":"2024-04-15T12:00:00","host":"web-01","requests":350,"errors":3} +{"index":{"_id":"31"}} +{"@timestamp":"2024-06-15T12:00:00","host":"web-01","requests":200,"errors":2} +{"index":{"_id":"32"}} +{"@timestamp":"2024-08-15T12:00:00","host":"web-01","requests":250,"errors":4} +{"index":{"_id":"33"}} +{"@timestamp":"2025-01-15T12:00:00","host":"web-01","requests":400,"errors":6} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 3a1fa9fe78d..e8c7cfc7c68 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -44,6 +44,7 @@ import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.TimewrapPivot; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; @@ -432,6 +433,23 @@ private QueryResponse buildResultSet( } columns.add(new Column(columnName, null, exprType)); } + // Timewrap post-processing: pivot unpivoted rows into period columns. The pivot is shared with + // the analytics route (AnalyticsExecutionEngine) so both engines produce identical output. + if (TimewrapPivot.isTimewrap()) { + try { + TimewrapPivot.Result pivoted = + TimewrapPivot.pivot( + columns, + values, + CalcitePlanContext.timewrapUnitName.get(), + CalcitePlanContext.timewrapSeries.get()); + columns = pivoted.columns(); + values = pivoted.values(); + } finally { + CalcitePlanContext.clearTimewrapSignals(); + } + } + Schema schema = new Schema(columns); QueryResponse response = new QueryResponse(schema, values, null); return response; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 5efc7b57baa..f8214096e41 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -244,6 +244,12 @@ private void doExecute( } } catch (Exception e) { closingListener.onFailure(e); + } finally { + // visitTimewrap (run inside planner.plan) sets timewrap thread-locals on this + // worker thread. execute()/executeWithProfile() capture-and-clear them on the + // happy path, but a planning exception bypasses that — clear here so the + // signals never leak onto the next query reusing this pooled thread. + CalcitePlanContext.clearTimewrapSignals(); } }), new TimeValue(0), @@ -291,6 +297,11 @@ private void doExplain( analyticsEngine.explain(plan, mode, planContext, listener); } catch (Exception e) { listener.onFailure(e); + } finally { + // explain plans a timewrap query (visitTimewrap sets thread-locals) but never + // executes, so nothing captures-and-clears them — clear here to avoid leaking + // onto the next query on this pooled thread. + CalcitePlanContext.clearTimewrapSignals(); } }), new TimeValue(0), diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4bc69a8f295..4f712042ed0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -55,6 +55,12 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +TIMEWRAP: 'TIMEWRAP'; +ALIGN: 'ALIGN'; +SERIES: 'SERIES'; +RELATIVE: 'RELATIVE'; +SHORT: 'SHORT'; +EXACT: 'EXACT'; EDGE: 'EDGE'; MAX_DEPTH: 'MAXDEPTH'; DEPTH_FIELD: 'DEPTHFIELD'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 98f85b08282..14655542062 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -98,6 +98,7 @@ commands | nomvCommand | graphLookupCommand | unionCommand + | timewrapCommand ; commandName @@ -149,6 +150,7 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | TIMEWRAP ; searchCommand @@ -355,6 +357,27 @@ transposeParameter | (COLUMN_NAME EQUAL stringLiteral) ; +timewrapCommand + : TIMEWRAP spanLiteral timewrapParameter* + ; + +timewrapParameter + : ALIGN EQUAL timewrapAlign + | SERIES EQUAL timewrapSeries + | TIME_FORMAT EQUAL stringLiteral + ; + +timewrapAlign + : NOW + | END + ; + +timewrapSeries + : RELATIVE + | SHORT + | EXACT + ; + timechartParameter : LIMIT EQUAL integerLiteral diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index f8604a352ce..88a9b9e1793 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -64,6 +64,7 @@ import org.opensearch.sql.ast.expression.SearchAnd; import org.opensearch.sql.ast.expression.SearchExpression; import org.opensearch.sql.ast.expression.SearchGroup; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.WindowFrame; @@ -118,6 +119,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; @@ -823,6 +825,39 @@ public UnresolvedPlan visitTimechartCommand(OpenSearchPPLParser.TimechartCommand .build(); } + /** Timewrap command. */ + @Override + public UnresolvedPlan visitTimewrapCommand(OpenSearchPPLParser.TimewrapCommandContext ctx) { + Literal spanLiteral = (Literal) expressionBuilder.visit(ctx.spanLiteral()); + String spanText = spanLiteral.getValue().toString(); + String valueStr = spanText.replaceAll("[^0-9]", ""); + String unitStr = spanText.replaceAll("[0-9]", ""); + int value = valueStr.isEmpty() ? 1 : Integer.parseInt(valueStr); + SpanUnit unit = SpanUnit.of(unitStr); + if (unit == SpanUnit.UNKNOWN || unit == SpanUnit.NONE) { + throw new SemanticCheckException("Invalid timewrap span unit: " + unitStr); + } + String align = "end"; + String series = "relative"; + String timeFormat = null; + for (var param : ctx.timewrapParameter()) { + if (param.timewrapAlign() != null) { + align = param.timewrapAlign().getText().toLowerCase(); + } else if (param.timewrapSeries() != null) { + series = param.timewrapSeries().getText().toLowerCase(); + } else if (param.TIME_FORMAT() != null) { + timeFormat = param.stringLiteral().getText(); + // Strip surrounding quotes + if (timeFormat.length() >= 2 + && ((timeFormat.startsWith("\"") && timeFormat.endsWith("\"")) + || (timeFormat.startsWith("'") && timeFormat.endsWith("'")))) { + timeFormat = timeFormat.substring(1, timeFormat.length() - 1); + } + } + } + return new Timewrap(unit, value, align, series, timeFormat, spanLiteral); + } + /** Eval command. */ @Override public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 4b75d444467..eb99a4b4381 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -105,6 +105,7 @@ import org.opensearch.sql.ast.tree.StreamWindow; import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.Timewrap; import org.opensearch.sql.ast.tree.Transpose; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Union; @@ -626,6 +627,25 @@ public String visitReverse(Reverse node, String context) { return StringUtils.format("%s | reverse", child); } + @Override + public String visitTimewrap(Timewrap node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder command = new StringBuilder(); + // span magnitude is masked like other span literals (see visitChart); align/series are + // constrained keywords, not user data, so they are rendered verbatim. + command.append(" | timewrap ").append(MASK_LITERAL); + if (node.getAlign() != null) { + command.append(" align=").append(node.getAlign()); + } + if (node.getSeries() != null) { + command.append(" series=").append(node.getSeries()); + } + if (node.getTimeFormat() != null) { + command.append(" time_format=").append(MASK_LITERAL); + } + return StringUtils.format("%s%s", child, command); + } + @Override public String visitChart(Chart node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java new file mode 100644 index 00000000000..66027839f8e --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.DataContext; +import org.apache.calcite.config.CalciteConnectionConfig; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Linq4j; +import org.apache.calcite.plan.RelTraitDef; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelProtoDataType; +import org.apache.calcite.schema.ScannableTable; +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.schema.Statistic; +import org.apache.calcite.schema.Statistics; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.Programs; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Test; + +/** + * Unit tests for the {@code timewrap} command's Calcite plan construction. Timewrap reshapes {@code + * timechart} output, so every query pipes through {@code timechart span=...} first. The pivot + * itself is post-processing in the execution engine (see {@code TimewrapPivot}); these tests cover + * the {@link org.opensearch.sql.calcite.CalciteRelNodeVisitor#visitTimewrap} lowering — the + * unpivoted RelNode and its generated Spark SQL. + */ +public class CalcitePPLTimewrapTest extends CalcitePPLAbstractTest { + + public CalcitePPLTimewrapTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Override + protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpecs) { + final SchemaPlus rootSchema = Frameworks.createRootSchema(true); + final SchemaPlus schema = CalciteAssert.addSchema(rootSchema, schemaSpecs); + ImmutableList rows = + ImmutableList.of( + new Object[] {java.sql.Timestamp.valueOf("2024-07-01 00:00:00"), 180}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-01 06:00:00"), 240}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-02 00:00:00"), 205}, + new Object[] {java.sql.Timestamp.valueOf("2024-07-03 00:00:00"), 165}); + schema.add("events", new EventsTable(rows)); + return Frameworks.newConfigBuilder() + .parserConfig(SqlParser.Config.DEFAULT) + .defaultSchema(schema) + .traitDefs((List) null) + .programs(Programs.heuristicJoinOrder(Programs.RULE_SET, true, 2)); + } + + // align=end query with a deterministic WHERE upper bound (2024-07-03 18:00:00 = epoch + // 1720029600), so the base_offset reference is stable across runs rather than the query clock. + private static final String TIMEWRAP_DAY = + "source=events | where @timestamp >= '2024-07-01 00:00:00' and @timestamp <=" + + " '2024-07-03 18:00:00' | timechart span=6h sum(value) | timewrap 1day align=end"; + + @Test + public void testTimewrapDayProducesUnpivotedPlan() { + RelNode root = getRelNode(TIMEWRAP_DAY); + // The pivot is post-processing in the execution engine; the RelNode is intentionally unpivoted: + // [display_ts, value, __base_offset__, __period__], sorted by (display_ts, period). base_offset + // uses FLOOR(.../span) (not truncating integer divide) so future-dated align=now stays correct. + String expectedLogical = + "LogicalSort(sort0=[$0], sort1=[$3], dir0=[ASC], dir1=[ASC])\n" + + " LogicalProject(@timestamp=[FROM_UNIXTIME(+(-(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT" + + " NOT NULL) OVER (), MOD(MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ()," + + " 86400:BIGINT)), MOD(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL, 86400:BIGINT)))]," + + " sum(value)=[$1], __base_offset__=[CAST(FLOOR(/(CAST(-(1720029600," + + " MAX(CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL) OVER ())):DOUBLE NOT NULL," + + " 86400:BIGINT))):BIGINT NOT NULL], __period__=[+(/(-(MAX(CAST(UNIX_TIMESTAMP($0))" + + ":BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + + " LogicalProject(@timestamp=[$0], sum(value)=[$1])\n" + + " LogicalAggregate(group=[{1}], sum(value)=[SUM($0)])\n" + + " LogicalProject(value=[$1], @timestamp0=[SPAN($0, 6, 'h')])\n" + + " LogicalFilter(condition=[AND(>=($0, TIMESTAMP('2024-07-01" + + " 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2024-07-03 18:00:00':VARCHAR)), IS NOT" + + " NULL($1))])\n" + + " LogicalTableScan(table=[[scott, events]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testTimewrapDaySparkSql() { + RelNode root = getRelNode(TIMEWRAP_DAY); + String expectedSparkSql = + "SELECT FROM_UNIXTIME((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " MOD(MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING), 86400) + MOD(CAST(UNIX_TIMESTAMP(`@timestamp`)" + + " AS BIGINT), 86400)) `@timestamp`, `sum(value)`, CAST(FLOOR(CAST(1720029600 -" + + " (MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING)) AS DOUBLE) / 86400) AS BIGINT)" + + " `__base_offset__`, ((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + + " CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) / 86400 + 1 `__period__`\n" + + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`, SUM(`value`) `sum(value)`\n" + + "FROM `scott`.`events`\n" + + "WHERE `@timestamp` >= TIMESTAMP('2024-07-01 00:00:00') AND `@timestamp` <=" + + " TIMESTAMP('2024-07-03 18:00:00') AND `value` IS NOT NULL\n" + + "GROUP BY SPAN(`@timestamp`, 6, 'h')\n" + + "ORDER BY 1 NULLS LAST) `t3`\n" + + "ORDER BY 1 NULLS LAST, 4 NULLS LAST"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + /** Minimal time-series table: a nullable timestamp and an integer measure. */ + public static class EventsTable implements ScannableTable { + private final ImmutableList rows; + + public EventsTable(ImmutableList rows) { + this.rows = rows; + } + + protected final RelProtoDataType protoRowType = + factory -> + factory + .builder() + .add("@timestamp", SqlTypeName.TIMESTAMP) + .nullable(true) + .add("value", SqlTypeName.INTEGER) + .nullable(true) + .build(); + + @Override + public Enumerable<@Nullable Object[]> scan(DataContext root) { + return Linq4j.asEnumerable(rows); + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return protoRowType.apply(typeFactory); + } + + @Override + public Statistic getStatistic() { + return Statistics.of(0d, ImmutableList.of(), RelCollations.createSingleton(0)); + } + + @Override + public Schema.TableType getJdbcTableType() { + return Schema.TableType.TABLE; + } + + @Override + public boolean isRolledUp(String column) { + return false; + } + + @Override + public boolean rolledUpColumnValidInsideAgg( + String column, + SqlCall call, + @Nullable SqlNode parent, + @Nullable CalciteConnectionConfig config) { + return false; + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 6756cdc198a..9ea21684edb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -283,6 +283,17 @@ public void testTimechartCommand() { anonymize("source=t | timechart timefield=month max(revenue)")); } + @Test + public void testTimewrapCommand() { + assertEquals( + "source=table | timechart count() | timewrap *** align=end series=relative", + anonymize("source=t | timechart count() | timewrap 1day")); + + assertEquals( + "source=table | timechart count() | timewrap *** align=now series=short", + anonymize("source=t | timechart count() | timewrap 1week align=now series=short")); + } + @Test public void testChartCommand() { assertEquals( From 4985a39efbc4399e19962d7a16291927c9121876 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:50:45 -0700 Subject: [PATCH 28/41] Widen narrow integer operands in PPL +/-/* to prevent overflow (#5603) PPL integer arithmetic (`+`, `-`, `*` and the named add/subtract/multiply functions) inferred `SMALLINT op SMALLINT -> SMALLINT` (and likewise for TINYINT) because the operators were registered with Calcite's stock `SqlStdOperatorTable.PLUS/MINUS/MULTIPLY`, whose return-type inference (`ReturnTypes.NULLABLE_SUM` / `PRODUCT_NULLABLE`) falls through to `LEAST_RESTRICTIVE` for non-decimal integers. As a result the product/sum of two narrow-integer columns overflowed the inferred type on every backend, just differently: - DataFusion / analytics-engine: silently wraps the i16 result, so e.g. `eval area = ResolutionWidth * ResolutionHeight | where area > 2000000` returned 0 rows instead of the matching row. - Calcite Enumerable engine: throws `ArithmeticException: value out of range`. - v2 legacy engine: `ExprShortValue` narrows via `shortValue()`, wrapping. Fix in the SQL-plugin lowering so all backends are corrected at once: widen the operands (byte/short -> INTEGER, any int/long -> BIGINT) before applying the operator. Casting the operands rather than only relabelling the result type is required, otherwise DataFusion still computes the narrow multiply and wraps before any outer cast. Non-integral operands (float/double/decimal/ datetime/mixed) are left untouched and defer to Calcite's default inference. The string-concat `ADD` variant and the DATETIME-DATETIME `SUBTRACT` variant are unchanged. mvindex's internal array-index arithmetic now uses the raw Calcite PLUS/MINUS operators so array indices stay INTEGER for ITEM/ARRAY_SLICE codegen (the widened result would otherwise be rejected as a long index). Note: this changes user-visible result column types for integer arithmetic (int-operand expressions now report bigint), which is the intended trade-off for overflow-safe, backend-consistent results. Adds CalcitePPLBuiltinFunctionIT coverage for the short->int and int->bigint widening tiers and updates the affected logical-plan / Spark-SQL snapshots. Signed-off-by: Kai Huang --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 4 +- .../function/CollectionUDF/MVAppendCore.java | 45 ++++- .../CollectionUDF/MVAppendFunctionImpl.java | 17 +- .../CollectionUDF/MVIndexFunctionImp.java | 36 ++-- .../expression/function/PPLFuncImpTable.java | 167 +++++++++++++++++- .../remote/CalcitePPLBuiltinFunctionIT.java | 65 ++++++- .../sql/ppl/MathematicalFunctionIT.java | 43 ++++- .../calcite/clickbench/q30.yaml | 8 +- .../calcite/clickbench/q36.yaml | 4 +- .../calcite/explain_agg_group_merge.yaml | 2 +- .../explain_agg_paginating_having3.yaml | 4 +- .../calcite/explain_agg_with_script.yaml | 2 +- .../explain_agg_with_sum_enhancement.yaml | 2 +- ...complex_sort_expr_no_expr_output_push.yaml | 4 +- ...n_complex_sort_expr_project_then_sort.yaml | 6 +- .../explain_complex_sort_expr_push.yaml | 6 +- ...lex_sort_expr_single_expr_output_push.yaml | 6 +- .../explain_complex_sort_nested_expr.yaml | 6 +- .../explain_complex_sort_then_field_sort.yaml | 4 +- .../calcite/explain_limit_push.yaml | 2 +- ...scalar_uncorrelated_subquery_in_where.yaml | 6 +- .../explain_simple_sort_expr_push.json | 4 +- ...ain_simple_sort_expr_pushdown_for_smj.yaml | 6 +- ...ple_sort_expr_single_expr_output_push.json | 2 +- .../explain_sort_complex_and_simple_expr.yaml | 6 +- .../explain_agg_group_merge.yaml | 2 +- .../explain_agg_with_script.yaml | 2 +- .../explain_agg_with_sum_enhancement.yaml | 2 +- ...complex_sort_expr_no_expr_output_push.yaml | 4 +- ...n_complex_sort_expr_project_then_sort.yaml | 4 +- .../explain_complex_sort_expr_push.yaml | 4 +- ...lex_sort_expr_single_expr_output_push.yaml | 4 +- .../explain_complex_sort_nested_expr.yaml | 4 +- .../explain_complex_sort_then_field_sort.yaml | 4 +- .../explain_filter_script_push.yaml | 2 +- .../explain_limit_push.yaml | 2 +- ...scalar_uncorrelated_subquery_in_where.yaml | 6 +- .../explain_simple_sort_expr_push.json | 6 +- ...ain_simple_sort_expr_pushdown_for_smj.yaml | 6 +- ...ple_sort_expr_single_expr_output_push.json | 4 +- .../explain_sort_complex_and_simple_expr.yaml | 4 +- .../ppl/calcite/CalcitePPLAppendPipeTest.java | 8 +- .../sql/ppl/calcite/CalcitePPLAppendTest.java | 8 +- .../calcite/CalcitePPLArrayFunctionTest.java | 4 +- .../sql/ppl/calcite/CalcitePPLDedupTest.java | 29 +-- .../sql/ppl/calcite/CalcitePPLEvalTest.java | 13 +- .../calcite/CalcitePPLExistsSubqueryTest.java | 6 +- .../calcite/CalcitePPLFieldFormatTest.java | 2 +- 48 files changed, 446 insertions(+), 141 deletions(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 064eff32d76..3320391c0d2 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -272,7 +272,7 @@ public void selectExpressionWithoutFrom() { givenQuery("SELECT 1 + 1") .assertPlan( """ - LogicalProject(1 + 1=[+(1, 1)]) + LogicalProject(1 + 1=[+(1:BIGINT, 1:BIGINT)]) LogicalValues(tuples=[[{ 0 }]]) """); } @@ -404,7 +404,7 @@ public void testArithmeticOnAggregates() { givenQuery("SELECT MAX(age) + MIN(age) AS range_sum FROM catalog.employees") .assertPlan( """ - LogicalProject(range_sum=[+($0, $1)]) + LogicalProject(range_sum=[+(CAST($0):BIGINT, CAST($1):BIGINT)]) LogicalAggregate(group=[{}], MAX(age)=[MAX($0)], MIN(age)=[MIN($0)]) LogicalProject(age=[$2]) LogicalTableScan(table=[[catalog, employees]]) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java index f9a67e4d6d8..c37189cb20b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendCore.java @@ -5,37 +5,72 @@ package org.opensearch.sql.expression.function.CollectionUDF; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; +import org.apache.calcite.sql.type.SqlTypeName; /** Core logic for `mvappend` command to collect elements from list of args */ public class MVAppendCore { /** * Collect non-null elements from `args`. If an item is a list, it will collect non-null elements - * of the list. See {@ref MVAppendFunctionImplTest} for detailed behavior. + * of the list. Each collected element is coerced to {@code elementType} so a heterogeneously + * boxed input (e.g. an {@code array(int_col)} operand contributing {@code Integer} cells to a + * {@code BIGINT}-typed result) does not throw {@code ClassCastException} when the array is later + * materialized by Avatica's per-type accessor. See {@ref MVAppendFunctionImplTest} for detailed + * behavior. */ + /** Untyped overload — collects without element coercion (used by map-append and unit tests). */ public static List collectElements(Object... args) { + return collectElements((SqlTypeName) null, args); + } + + public static List collectElements(SqlTypeName elementType, Object... args) { List elements = new ArrayList<>(); for (Object arg : args) { if (arg == null) { continue; } else if (arg instanceof List) { - addListElements((List) arg, elements); + addListElements((List) arg, elements, elementType); } else { - elements.add(arg); + elements.add(coerce(arg, elementType)); } } return elements.isEmpty() ? null : elements; } - private static void addListElements(List list, List elements) { + private static void addListElements( + List list, List elements, SqlTypeName elementType) { for (Object item : list) { if (item != null) { - elements.add(item); + elements.add(coerce(item, elementType)); } } } + + /** + * Align a boxed numeric element to the array's target element type. Only numeric widenings that + * arise from operand widening (e.g. INTEGER cells into a BIGINT array) are handled; non-numeric + * or null-typed targets pass the value through unchanged so mixed / ANY-typed arrays keep their + * existing {@code Object[]} runtime semantics. + */ + private static Object coerce(Object value, SqlTypeName elementType) { + if (elementType == null || !(value instanceof Number)) { + return value; + } + Number num = (Number) value; + return switch (elementType) { + case TINYINT -> num.byteValue(); + case SMALLINT -> num.shortValue(); + case INTEGER -> num.intValue(); + case BIGINT -> num.longValue(); + case FLOAT, REAL -> num.floatValue(); + case DOUBLE -> num.doubleValue(); + case DECIMAL -> num instanceof BigDecimal ? num : BigDecimal.valueOf(num.doubleValue()); + default -> value; + }; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java index bafbeb09c43..dc3402e464d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVAppendFunctionImpl.java @@ -127,12 +127,27 @@ public Expression implement( coerced.add(EnumUtils.convert(op, elementClass)); } } + // Pass the target element SqlTypeName so the runtime can align the elements flattened out of + // ARRAY operands. Calcite does not element-wise cast inside an array operand, so + // `mvappend(array(int_col), int_col * 2)` — where operand widening makes the result element + // type BIGINT while `array(int_col)` still yields Integer cells — would otherwise throw + // `Integer cannot be cast to Long` when the array is materialized. Scalars are already + // pre-cast above; the runtime coercion is a no-op for them. + SqlTypeName targetType = elementType == null ? null : elementType.getSqlTypeName(); return Expressions.call( - Types.lookupMethod(MVAppendFunctionImpl.class, "mvappend", Object[].class), + Types.lookupMethod( + MVAppendFunctionImpl.class, "mvappendTyped", SqlTypeName.class, Object[].class), + Expressions.constant(targetType, SqlTypeName.class), Expressions.newArrayInit(Object.class, coerced)); } } + /** Codegen entry point: coerces flattened elements to {@code elementType}. */ + public static Object mvappendTyped(SqlTypeName elementType, Object... args) { + return MVAppendCore.collectElements(elementType, args); + } + + /** Untyped entry point used by unit tests; performs no element coercion. */ public static Object mvappend(Object... args) { return MVAppendCore.collectElements(args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java index 24e4b489632..a244e290ae1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/MVIndexFunctionImp.java @@ -5,17 +5,16 @@ package org.opensearch.sql.expression.function.CollectionUDF; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDFUNCTION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_LENGTH; import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_SLICE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; import java.math.BigDecimal; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.opensearch.sql.expression.function.PPLFuncImpTable; /** @@ -37,6 +36,10 @@ *
  • Range access uses Calcite's ARRAY_SLICE operator (0-based indexing with length parameter) *
  • Index conversion handles the difference between PPL's 0-based indexing and Calcite's * conventions + *
  • Index arithmetic uses Calcite's raw {@code PLUS}/{@code MINUS} rather than PPL's widening + * {@code +}/{@code -} operators: array indices are int-domain and {@code ITEM}/{@code + * ARRAY_SLICE} require an INTEGER index, so the deliberate integer-overflow widening applied + * to user arithmetic must not leak into these internal, bounded computations. * */ public class MVIndexFunctionImp implements PPLFuncImpTable.FunctionImp { @@ -59,6 +62,16 @@ public RexNode resolve(RexBuilder builder, RexNode... args) { } } + /** Non-widening integer addition for internal, int-domain array-index math. */ + private static RexNode add(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.PLUS, left, right); + } + + /** Non-widening integer subtraction for internal, int-domain array-index math. */ + private static RexNode subtract(RexBuilder builder, RexNode left, RexNode right) { + return builder.makeCall(SqlStdOperatorTable.MINUS, left, right); + } + /** * Resolves single element access: mvindex(array, index) * @@ -72,11 +85,9 @@ private RexNode resolveSingleElement( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode sumArrayLenStart = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); - RexNode negativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, sumArrayLenStart, one); - RexNode positiveCase = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, startIdx, one); + RexNode sumArrayLenStart = add(builder, arrayLen, startIdx); + RexNode negativeCase = add(builder, sumArrayLenStart, one); + RexNode positiveCase = add(builder, startIdx, one); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isNegative, negativeCase, positiveCase); @@ -97,21 +108,18 @@ private RexNode resolveRange( RexNode one = builder.makeExactLiteral(BigDecimal.ONE); RexNode isStartNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, startIdx, zero); - RexNode startNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, startIdx); + RexNode startNegativeCase = add(builder, arrayLen, startIdx); RexNode normalizedStart = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isStartNegative, startNegativeCase, startIdx); RexNode isEndNegative = PPLFuncImpTable.INSTANCE.resolve(builder, LESS, endIdx, zero); - RexNode endNegativeCase = - PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, arrayLen, endIdx); + RexNode endNegativeCase = add(builder, arrayLen, endIdx); RexNode normalizedEnd = PPLFuncImpTable.INSTANCE.resolve(builder, IF, isEndNegative, endNegativeCase, endIdx); // Calculate length: (normalizedEnd - normalizedStart) + 1 - RexNode diff = - PPLFuncImpTable.INSTANCE.resolve(builder, SUBTRACT, normalizedEnd, normalizedStart); - RexNode length = PPLFuncImpTable.INSTANCE.resolve(builder, ADDFUNCTION, diff, one); + RexNode diff = subtract(builder, normalizedEnd, normalizedStart); + RexNode length = add(builder, diff, one); // Call ARRAY_SLICE(array, normalizedStart, length) return PPLFuncImpTable.INSTANCE.resolve(builder, ARRAY_SLICE, array, normalizedStart, length); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 5750bf6cae8..b9350d18d84 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -286,9 +286,12 @@ import javax.annotation.Nullable; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlLibraryOperators; @@ -572,6 +575,10 @@ public RexNode resolve( // return type of the lambda function. compulsoryCast(builder, functionName, args); + // Align integer operand widths when a comparison has a scalar subquery on one side; see + // coerceNumericComparisonOperands for why this is needed for correct decorrelated join keys. + args = coerceNumericComparisonOperands(builder, functionName, args); + List argTypes = Arrays.stream(args).map(RexNode::getType).toList(); try { for (Map.Entry implement : implementList) { @@ -606,6 +613,62 @@ public RexNode resolve( functionName, allowedSignatures, PlanUtils.getActualSignature(argTypes))); } + /** + * Widens the operands of a binary comparison against a scalar subquery to a common integer type + * when the two operands are integers of differing width (e.g. INT vs BIGINT, SMALLINT vs INT). + * Returns the original array unchanged for every other case, so only the ambiguous + * integer-width-against-a-subquery case is touched. + * + *

    Calcite's comparison operators accept mixed integer widths by family, so {@code makeCall(=, + * int, bigint)} type-checks and evaluates correctly as an ordinary scalar predicate (against a + * column or literal), which is why those comparisons are left alone. But when one side is a + * scalar subquery, Calcite's decorrelator turns the comparison into a join whose keys keep their + * original widths; the generated key extractors then box the two sides as different Java types + * (e.g. {@code Integer} vs {@code Long}) that never compare equal, so the join silently drops + * every row. Casting both sides to their least-restrictive common integer type keeps the boolean + * result identical while making the derived join keys share a single type. This surfaces once + * integer arithmetic widens (e.g. {@code min(salary) + 1000} is BIGINT) so a subquery result is + * compared against a narrower column. + */ + private static RexNode[] coerceNumericComparisonOperands( + RexBuilder builder, BuiltinFunctionName functionName, RexNode... args) { + if (args.length != 2 || !BuiltinFunctionName.COMPARATORS.contains(functionName)) { + return args; + } + if (!containsSubQuery(args[0]) && !containsSubQuery(args[1])) { + return args; + } + RelDataType leftType = args[0].getType(); + RelDataType rightType = args[1].getType(); + if (!SqlTypeName.INT_TYPES.contains(leftType.getSqlTypeName()) + || !SqlTypeName.INT_TYPES.contains(rightType.getSqlTypeName()) + || leftType.getSqlTypeName() == rightType.getSqlTypeName()) { + return args; + } + RelDataType commonType = + builder.getTypeFactory().leastRestrictive(List.of(leftType, rightType)); + if (commonType == null) { + return args; + } + return new RexNode[] { + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, leftType.isNullable()), args[0]), + builder.makeCast( + TYPE_FACTORY.createTypeWithNullability(commonType, rightType.isNullable()), args[1]) + }; + } + + /** Whether {@code node} is, or transitively contains, a {@link RexSubQuery}. */ + private static boolean containsSubQuery(RexNode node) { + if (node instanceof RexSubQuery) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(PPLFuncImpTable::containsSubQuery); + } + return false; + } + /** * Ad-hoc coercion for some functions that require specific casting of arguments. Now it only * applies to the REDUCE function. @@ -727,6 +790,89 @@ protected void registerDivideFunction(BuiltinFunctionName functionName) { PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); } + /** + * Register an arithmetic operator ({@code +}, {@code -}, {@code *}) that widens narrow integer + * operands before applying the operation, deriving the type checker from the operator. + * + *

    Calcite infers {@code SMALLINT op SMALLINT -> SMALLINT} (via {@code ReturnTypes.PLUS} / + * {@code PRODUCT_NULLABLE}, which fall through to {@code LEAST_RESTRICTIVE}), so the product of + * two {@code short} columns overflows: DataFusion silently wraps the {@code i16} result while + * the Calcite Enumerable engine throws {@code ArithmeticException: value out of range}. Casting + * the operands up (byte/short -> int, int -> long) makes the arithmetic compute at a width that + * cannot overflow for the widened tier, matching v2 arithmetic intent across all backends. + */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator) { + PPLTypeChecker typeChecker = + wrapSqlOperandTypeChecker(operator.getOperandTypeChecker(), operator.getName(), false); + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + /** Same as above but with an explicit {@link PPLTypeChecker}. */ + protected void registerWideningIntegerOperator( + BuiltinFunctionName functionName, SqlOperator operator, PPLTypeChecker typeChecker) { + register(functionName, wideningIntegerArithmetic(operator), typeChecker); + } + + private static FunctionImp wideningIntegerArithmetic(SqlOperator operator) { + // +, -, * are strictly binary; FunctionImp2 documents and enforces the two-operand contract. + return (FunctionImp2) + (builder, left, right) -> + builder.makeCall(operator, widenIntegerOperands(builder, left, right)); + } + + private static RexNode[] widenIntegerOperands(RexBuilder builder, RexNode... args) { + SqlTypeName promoted = promotedIntegerType(args); + if (promoted == null) { + return args; + } + // Skip widening arithmetic inside higher-order-function lambda bodies (reduce/mvmap/...). + // Those operands reference lambda parameters whose types are placeholders resolved later by + // the HOF's own return-type inference (see LambdaUtils.inferReturnTypeFromLambda); wrapping a + // RexLambdaRef in a CAST breaks that index-based resolution. They also execute JVM-side via + // linq4j (operands already promote to int), so they never hit the backend wrap path. + for (RexNode arg : args) { + if (referencesLambdaParameter(arg)) { + return args; + } + } + RexNode[] widened = new RexNode[args.length]; + for (int i = 0; i < args.length; i++) { + RelDataType target = TYPE_FACTORY.createSqlType(promoted, args[i].getType().isNullable()); + widened[i] = builder.makeCast(target, args[i]); + } + return widened; + } + + private static boolean referencesLambdaParameter(RexNode node) { + if (node instanceof RexLambdaRef) { + return true; + } + if (node instanceof RexCall call) { + return call.getOperands().stream().anyMatch(AbstractBuilder::referencesLambdaParameter); + } + return false; + } + + /** + * Target integer type that all operands should widen to, or {@code null} to leave the call + * untouched (any non-integral operand, e.g. FLOAT/DOUBLE/DECIMAL/DATETIME, defers to Calcite's + * default inference). byte/short -> INTEGER; anything involving int/long -> BIGINT. + */ + private static SqlTypeName promotedIntegerType(RexNode... args) { + boolean needsLong = false; + for (RexNode arg : args) { + switch (arg.getType().getSqlTypeName()) { + case TINYINT, SMALLINT -> {} + case INTEGER, BIGINT -> needsLong = true; + default -> { + return null; + } + } + } + return needsLong ? SqlTypeName.BIGINT : SqlTypeName.INTEGER; + } + void populate() { // register operators for comparison registerOperator(NOTEQUAL, PPLBuiltinOperators.NOT_EQUALS_IP, SqlStdOperatorTable.NOT_EQUALS); @@ -741,23 +887,25 @@ void populate() { registerOperator(OR, SqlStdOperatorTable.OR); registerOperator(NOT, SqlStdOperatorTable.NOT); - // Register ADDFUNCTION for numeric addition only - registerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); - registerOperator( + // Register ADDFUNCTION for numeric addition only. Widen narrow integer operands so the sum + // cannot overflow the (mis-)inferred SMALLINT/TINYINT result type; see + // registerWideningIntegerOperator. + registerWideningIntegerOperator(ADDFUNCTION, SqlStdOperatorTable.PLUS); + registerWideningIntegerOperator( SUBTRACTFUNCTION, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - registerOperator( + registerWideningIntegerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.wrapFamily((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC)); - // Add DATETIME-DATETIME variant for timestamp binning support + // Add DATETIME-DATETIME variant for timestamp binning support (no integer widening) registerOperator( SUBTRACT, SqlStdOperatorTable.MINUS, PPLTypeChecker.family(SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME)); - registerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); - registerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLY, SqlStdOperatorTable.MULTIPLY); + registerWideningIntegerOperator(MULTIPLYFUNCTION, SqlStdOperatorTable.MULTIPLY); registerOperator(TRUNCATE, SqlStdOperatorTable.TRUNCATE); registerOperator(ASCII, SqlStdOperatorTable.ASCII); registerOperator(LENGTH, SqlStdOperatorTable.CHAR_LENGTH); @@ -1125,8 +1273,9 @@ void populate() { SqlStdOperatorTable.CONCAT, PPLTypeChecker.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER)); // Register ADD (+ symbol) for numeric addition - // Replace type checker since PLUS also supports binary addition - registerOperator( + // Replace type checker since PLUS also supports binary addition. Widen narrow integer + // operands so the sum cannot overflow the (mis-)inferred SMALLINT/TINYINT result type. + registerWideningIntegerOperator( ADD, SqlStdOperatorTable.PLUS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java index cbd94683fd1..82debdb0310 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBuiltinFunctionIT.java @@ -267,7 +267,8 @@ public void testModFloatAndNegative() throws IOException { "source=%s | eval f = mod(float_number, 2), n = -1 * short_number %% 2, nd = -1 *" + " double_number %% 2 | fields f, n, nd", TEST_INDEX_DATATYPE_NUMERIC)); - verifySchema(actual, schema("f", "float"), schema("n", "int"), schema("nd", "double")); + // -1 * short_number widens the integer operands to bigint (overflow-safe widening). + verifySchema(actual, schema("f", "float"), schema("n", "bigint"), schema("nd", "double")); verifyDataRows(actual, closeTo(0.2, -1, -1.1)); } @@ -414,4 +415,66 @@ public void testDivideShouldReturnNull() throws IOException { schema("r6", "double")); verifyDataRows(actual, rows(null, null, null, null, null)); } + + /** + * Integer arithmetic must widen narrow operands so the result cannot overflow the (mis-)inferred + * SMALLINT/TINYINT result type. Historically {@code short * short} stayed SHORT, silently + * wrapping on the analytics-engine backend and throwing "value out of range" on the Calcite path. + * byte/short now widen to INT and int/long widen to BIGINT for {@code +}, {@code -}, {@code *}. + */ + @Test + public void testIntegerArithmeticWidensResultType() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval ss = short_number * short_number, bb = byte_number *" + + " byte_number, ii = integer_number * integer_number, sp = short_number +" + + " short_number, sm = short_number - byte_number | fields ss, bb, ii, sp, sm", + TEST_INDEX_DATATYPE_NUMERIC)); + // short/byte products widen to int; int product widens to bigint; +/- widen likewise. + verifySchema( + actual, + schema("ss", "int"), + schema("bb", "int"), + schema("ii", "bigint"), + schema("sp", "int"), + schema("sm", "int")); + // short_number=3, byte_number=4, integer_number=2. + verifyDataRows(actual, rows(9, 16, 4, 6, -1)); + } + + /** + * Regression for the reported overflow: {@code short * } whose product exceeds the 16-bit + * SHORT range (32767) must produce the correct widened value rather than a wrapped one. + */ + @Test + public void testShortArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval area = short_number * 20000 | where area > 32767 | fields area", + TEST_INDEX_DATATYPE_NUMERIC)); + // 3 * 20000 = 60000, which overflows SHORT (max 32767) but is exact once widened. The INTEGER + // literal 20000 promotes the product to BIGINT (any int operand -> long). + verifySchema(actual, schema("area", "bigint")); + verifyDataRows(actual, rows(60000)); + } + + /** + * The INTEGER->BIGINT tier: {@code integer_number * } whose product exceeds + * the 32-bit INT range must widen to BIGINT and stay exact rather than wrapping i32. + */ + @Test + public void testIntegerArithmeticDoesNotOverflow() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval big = integer_number * 2000000000 | where big > 2147483647 |" + + " fields big", + TEST_INDEX_DATATYPE_NUMERIC)); + // integer_number=2; 2 * 2,000,000,000 = 4,000,000,000 overflows INT (max 2,147,483,647) but is + // exact once widened to BIGINT. + verifySchema(actual, schema("big", "bigint")); + verifyDataRows(actual, rows(4000000000L)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java index 42f69010270..0a81ed92ad6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MathematicalFunctionIT.java @@ -571,7 +571,12 @@ public void testEvalSumMultipleIntegers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + // Calcite widens integer arithmetic operands to avoid overflow, so the result is bigint. + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(6), rows(6), rows(6), rows(6), rows(6)); } @@ -590,7 +595,11 @@ public void testEvalSumWithFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, 10) | fields f | head 7", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(42), rows(46), rows(38), rows(43), rows(46), rows(49), rows(44)); } @@ -600,7 +609,11 @@ public void testEvalSumMultipleNumericArguments() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(1, 2, 3, 4, 5) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(15), rows(15), rows(15), rows(15), rows(15)); } @@ -681,7 +694,11 @@ public void testEvalSumAndAvgComparison() throws IOException { "source=%s | eval sum_val = sum(10, 20, 30), avg_val = avg(10, 20, 30) | fields" + " sum_val, avg_val | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + if (isCalciteEnabled()) { + verifySchema(result, schema("sum_val", null, "bigint"), schema("avg_val", null, "double")); + } else { + verifySchema(result, schema("sum_val", null, "int"), schema("avg_val", null, "double")); + } verifyDataRows( result, rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0), rows(60, 20.0)); } @@ -693,7 +710,11 @@ public void testEvalSumInWhereClause() throws IOException { String.format( "source=%s | where sum(age, 10) > 40 | eval f = sum(age, 10) | fields f | head 6", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // Should return rows where age + 10 > 40, so age > 30 verifyDataRows(result, rows(42), rows(46), rows(43), rows(46), rows(49), rows(44)); } @@ -740,7 +761,11 @@ public void testEvalSumWithMultipleFields() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(age, age, 10) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } // sum(age, age, 10) = age + age + 10 = 2*age + 10 verifyDataRows(result, rows(74), rows(82), rows(66), rows(76), rows(82)); } @@ -762,7 +787,11 @@ public void testEvalSumWithNegativeNumbers() throws IOException { executeQuery( String.format( "source=%s | eval f = sum(-5, 10, -3) | fields f | head 5", TEST_INDEX_BANK)); - verifySchema(result, schema("f", null, "int")); + if (isCalciteEnabled()) { + verifySchema(result, schema("f", null, "bigint")); + } else { + verifySchema(result, schema("f", null, "int")); + } verifyDataRows(result, rows(2), rows(2), rows(2), rows(2), rows(2)); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index de1f6d31f0d..d50a9ec47ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -2,8 +2,10 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) - LogicalProject(ResolutionWidth=[$80], $f90=[+($80, 1)], $f91=[+($80, 2)], $f92=[+($80, 3)], $f93=[+($80, 4)], $f94=[+($80, 5)], $f95=[+($80, 6)], $f96=[+($80, 7)], $f97=[+($80, 8)], $f98=[+($80, 9)], $f99=[+($80, 10)], $f100=[+($80, 11)], $f101=[+($80, 12)], $f102=[+($80, 13)], $f103=[+($80, 14)], $f104=[+($80, 15)], $f105=[+($80, 16)], $f106=[+($80, 17)], $f107=[+($80, 18)], $f108=[+($80, 19)], $f109=[+($80, 20)], $f110=[+($80, 21)], $f111=[+($80, 22)], $f112=[+($80, 23)], $f113=[+($80, 24)], $f114=[+($80, 25)], $f115=[+($80, 26)], $f116=[+($80, 27)], $f117=[+($80, 28)], $f118=[+($80, 29)], $f119=[+($80, 30)], $f120=[+($80, 31)], $f121=[+($80, 32)], $f122=[+($80, 33)], $f123=[+($80, 34)], $f124=[+($80, 35)], $f125=[+($80, 36)], $f126=[+($80, 37)], $f127=[+($80, 38)], $f128=[+($80, 39)], $f129=[+($80, 40)], $f130=[+($80, 41)], $f131=[+($80, 42)], $f132=[+($80, 43)], $f133=[+($80, 44)], $f134=[+($80, 45)], $f135=[+($80, 46)], $f136=[+($80, 47)], $f137=[+($80, 48)], $f138=[+($80, 49)], $f139=[+($80, 50)], $f140=[+($80, 51)], $f141=[+($80, 52)], $f142=[+($80, 53)], $f143=[+($80, 54)], $f144=[+($80, 55)], $f145=[+($80, 56)], $f146=[+($80, 57)], $f147=[+($80, 58)], $f148=[+($80, 59)], $f149=[+($80, 60)], $f150=[+($80, 61)], $f151=[+($80, 62)], $f152=[+($80, 63)], $f153=[+($80, 64)], $f154=[+($80, 65)], $f155=[+($80, 66)], $f156=[+($80, 67)], $f157=[+($80, 68)], $f158=[+($80, 69)], $f159=[+($80, 70)], $f160=[+($80, 71)], $f161=[+($80, 72)], $f162=[+($80, 73)], $f163=[+($80, 74)], $f164=[+($80, 75)], $f165=[+($80, 76)], $f166=[+($80, 77)], $f167=[+($80, 78)], $f168=[+($80, 79)], $f169=[+($80, 80)], $f170=[+($80, 81)], $f171=[+($80, 82)], $f172=[+($80, 83)], $f173=[+($80, 84)], $f174=[+($80, 85)], $f175=[+($80, 86)], $f176=[+($80, 87)], $f177=[+($80, 88)], $f178=[+($80, 89)]) + LogicalProject(ResolutionWidth=[$80], $f90=[+(CAST($80):BIGINT, 1)], $f91=[+(CAST($80):BIGINT, 2)], $f92=[+(CAST($80):BIGINT, 3)], $f93=[+(CAST($80):BIGINT, 4)], $f94=[+(CAST($80):BIGINT, 5)], $f95=[+(CAST($80):BIGINT, 6)], $f96=[+(CAST($80):BIGINT, 7)], $f97=[+(CAST($80):BIGINT, 8)], $f98=[+(CAST($80):BIGINT, 9)], $f99=[+(CAST($80):BIGINT, 10)], $f100=[+(CAST($80):BIGINT, 11)], $f101=[+(CAST($80):BIGINT, 12)], $f102=[+(CAST($80):BIGINT, 13)], $f103=[+(CAST($80):BIGINT, 14)], $f104=[+(CAST($80):BIGINT, 15)], $f105=[+(CAST($80):BIGINT, 16)], $f106=[+(CAST($80):BIGINT, 17)], $f107=[+(CAST($80):BIGINT, 18)], $f108=[+(CAST($80):BIGINT, 19)], $f109=[+(CAST($80):BIGINT, 20)], $f110=[+(CAST($80):BIGINT, 21)], $f111=[+(CAST($80):BIGINT, 22)], $f112=[+(CAST($80):BIGINT, 23)], $f113=[+(CAST($80):BIGINT, 24)], $f114=[+(CAST($80):BIGINT, 25)], $f115=[+(CAST($80):BIGINT, 26)], $f116=[+(CAST($80):BIGINT, 27)], $f117=[+(CAST($80):BIGINT, 28)], $f118=[+(CAST($80):BIGINT, 29)], $f119=[+(CAST($80):BIGINT, 30)], $f120=[+(CAST($80):BIGINT, 31)], $f121=[+(CAST($80):BIGINT, 32)], $f122=[+(CAST($80):BIGINT, 33)], $f123=[+(CAST($80):BIGINT, 34)], $f124=[+(CAST($80):BIGINT, 35)], $f125=[+(CAST($80):BIGINT, 36)], $f126=[+(CAST($80):BIGINT, 37)], $f127=[+(CAST($80):BIGINT, 38)], $f128=[+(CAST($80):BIGINT, 39)], $f129=[+(CAST($80):BIGINT, 40)], $f130=[+(CAST($80):BIGINT, 41)], $f131=[+(CAST($80):BIGINT, 42)], $f132=[+(CAST($80):BIGINT, 43)], $f133=[+(CAST($80):BIGINT, 44)], $f134=[+(CAST($80):BIGINT, 45)], $f135=[+(CAST($80):BIGINT, 46)], $f136=[+(CAST($80):BIGINT, 47)], $f137=[+(CAST($80):BIGINT, 48)], $f138=[+(CAST($80):BIGINT, 49)], $f139=[+(CAST($80):BIGINT, 50)], $f140=[+(CAST($80):BIGINT, 51)], $f141=[+(CAST($80):BIGINT, 52)], $f142=[+(CAST($80):BIGINT, 53)], $f143=[+(CAST($80):BIGINT, 54)], $f144=[+(CAST($80):BIGINT, 55)], $f145=[+(CAST($80):BIGINT, 56)], $f146=[+(CAST($80):BIGINT, 57)], $f147=[+(CAST($80):BIGINT, 58)], $f148=[+(CAST($80):BIGINT, 59)], $f149=[+(CAST($80):BIGINT, 60)], $f150=[+(CAST($80):BIGINT, 61)], $f151=[+(CAST($80):BIGINT, 62)], $f152=[+(CAST($80):BIGINT, 63)], $f153=[+(CAST($80):BIGINT, 64)], $f154=[+(CAST($80):BIGINT, 65)], $f155=[+(CAST($80):BIGINT, 66)], $f156=[+(CAST($80):BIGINT, 67)], $f157=[+(CAST($80):BIGINT, 68)], $f158=[+(CAST($80):BIGINT, 69)], $f159=[+(CAST($80):BIGINT, 70)], $f160=[+(CAST($80):BIGINT, 71)], $f161=[+(CAST($80):BIGINT, 72)], $f162=[+(CAST($80):BIGINT, 73)], $f163=[+(CAST($80):BIGINT, 74)], $f164=[+(CAST($80):BIGINT, 75)], $f165=[+(CAST($80):BIGINT, 76)], $f166=[+(CAST($80):BIGINT, 77)], $f167=[+(CAST($80):BIGINT, 78)], $f168=[+(CAST($80):BIGINT, 79)], $f169=[+(CAST($80):BIGINT, 80)], $f170=[+(CAST($80):BIGINT, 81)], $f171=[+(CAST($80):BIGINT, 82)], $f172=[+(CAST($80):BIGINT, 83)], $f173=[+(CAST($80):BIGINT, 84)], $f174=[+(CAST($80):BIGINT, 85)], $f175=[+(CAST($80):BIGINT, 86)], $f176=[+(CAST($80):BIGINT, 87)], $f177=[+(CAST($80):BIGINT, 88)], $f178=[+(CAST($80):BIGINT, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t1):BIGINT], expr#3=[+($t0, $t2)], expr#4=[2], expr#5=[*($t1, $t4)], expr#6=[+($t0, $t5)], expr#7=[3], expr#8=[*($t1, $t7)], expr#9=[+($t0, $t8)], expr#10=[4], expr#11=[*($t1, $t10)], expr#12=[+($t0, $t11)], expr#13=[5], expr#14=[*($t1, $t13)], expr#15=[+($t0, $t14)], expr#16=[6], expr#17=[*($t1, $t16)], expr#18=[+($t0, $t17)], expr#19=[7], expr#20=[*($t1, $t19)], expr#21=[+($t0, $t20)], expr#22=[8], expr#23=[*($t1, $t22)], expr#24=[+($t0, $t23)], expr#25=[9], expr#26=[*($t1, $t25)], expr#27=[+($t0, $t26)], expr#28=[10], expr#29=[*($t1, $t28)], expr#30=[+($t0, $t29)], expr#31=[11], expr#32=[*($t1, $t31)], expr#33=[+($t0, $t32)], expr#34=[12], expr#35=[*($t1, $t34)], expr#36=[+($t0, $t35)], expr#37=[13], expr#38=[*($t1, $t37)], expr#39=[+($t0, $t38)], expr#40=[14], expr#41=[*($t1, $t40)], expr#42=[+($t0, $t41)], expr#43=[15], expr#44=[*($t1, $t43)], expr#45=[+($t0, $t44)], expr#46=[16], expr#47=[*($t1, $t46)], expr#48=[+($t0, $t47)], expr#49=[17], expr#50=[*($t1, $t49)], expr#51=[+($t0, $t50)], expr#52=[18], expr#53=[*($t1, $t52)], expr#54=[+($t0, $t53)], expr#55=[19], expr#56=[*($t1, $t55)], expr#57=[+($t0, $t56)], expr#58=[20], expr#59=[*($t1, $t58)], expr#60=[+($t0, $t59)], expr#61=[21], expr#62=[*($t1, $t61)], expr#63=[+($t0, $t62)], expr#64=[22], expr#65=[*($t1, $t64)], expr#66=[+($t0, $t65)], expr#67=[23], expr#68=[*($t1, $t67)], expr#69=[+($t0, $t68)], expr#70=[24], expr#71=[*($t1, $t70)], expr#72=[+($t0, $t71)], expr#73=[25], expr#74=[*($t1, $t73)], expr#75=[+($t0, $t74)], expr#76=[26], expr#77=[*($t1, $t76)], expr#78=[+($t0, $t77)], expr#79=[27], expr#80=[*($t1, $t79)], expr#81=[+($t0, $t80)], expr#82=[28], expr#83=[*($t1, $t82)], expr#84=[+($t0, $t83)], expr#85=[29], expr#86=[*($t1, $t85)], expr#87=[+($t0, $t86)], expr#88=[30], expr#89=[*($t1, $t88)], expr#90=[+($t0, $t89)], expr#91=[31], expr#92=[*($t1, $t91)], expr#93=[+($t0, $t92)], expr#94=[32], expr#95=[*($t1, $t94)], expr#96=[+($t0, $t95)], expr#97=[33], expr#98=[*($t1, $t97)], expr#99=[+($t0, $t98)], expr#100=[34], expr#101=[*($t1, $t100)], expr#102=[+($t0, $t101)], expr#103=[35], expr#104=[*($t1, $t103)], expr#105=[+($t0, $t104)], expr#106=[36], expr#107=[*($t1, $t106)], expr#108=[+($t0, $t107)], expr#109=[37], expr#110=[*($t1, $t109)], expr#111=[+($t0, $t110)], expr#112=[38], expr#113=[*($t1, $t112)], expr#114=[+($t0, $t113)], expr#115=[39], expr#116=[*($t1, $t115)], expr#117=[+($t0, $t116)], expr#118=[40], expr#119=[*($t1, $t118)], expr#120=[+($t0, $t119)], expr#121=[41], expr#122=[*($t1, $t121)], expr#123=[+($t0, $t122)], expr#124=[42], expr#125=[*($t1, $t124)], expr#126=[+($t0, $t125)], expr#127=[43], expr#128=[*($t1, $t127)], expr#129=[+($t0, $t128)], expr#130=[44], expr#131=[*($t1, $t130)], expr#132=[+($t0, $t131)], expr#133=[45], expr#134=[*($t1, $t133)], expr#135=[+($t0, $t134)], expr#136=[46], expr#137=[*($t1, $t136)], expr#138=[+($t0, $t137)], expr#139=[47], expr#140=[*($t1, $t139)], expr#141=[+($t0, $t140)], expr#142=[48], expr#143=[*($t1, $t142)], expr#144=[+($t0, $t143)], expr#145=[49], expr#146=[*($t1, $t145)], expr#147=[+($t0, $t146)], expr#148=[50], expr#149=[*($t1, $t148)], expr#150=[+($t0, $t149)], expr#151=[51], expr#152=[*($t1, $t151)], expr#153=[+($t0, $t152)], expr#154=[52], expr#155=[*($t1, $t154)], expr#156=[+($t0, $t155)], expr#157=[53], expr#158=[*($t1, $t157)], expr#159=[+($t0, $t158)], expr#160=[54], expr#161=[*($t1, $t160)], expr#162=[+($t0, $t161)], expr#163=[55], expr#164=[*($t1, $t163)], expr#165=[+($t0, $t164)], expr#166=[56], expr#167=[*($t1, $t166)], expr#168=[+($t0, $t167)], expr#169=[57], expr#170=[*($t1, $t169)], expr#171=[+($t0, $t170)], expr#172=[58], expr#173=[*($t1, $t172)], expr#174=[+($t0, $t173)], expr#175=[59], expr#176=[*($t1, $t175)], expr#177=[+($t0, $t176)], expr#178=[60], expr#179=[*($t1, $t178)], expr#180=[+($t0, $t179)], expr#181=[61], expr#182=[*($t1, $t181)], expr#183=[+($t0, $t182)], expr#184=[62], expr#185=[*($t1, $t184)], expr#186=[+($t0, $t185)], expr#187=[63], expr#188=[*($t1, $t187)], expr#189=[+($t0, $t188)], expr#190=[64], expr#191=[*($t1, $t190)], expr#192=[+($t0, $t191)], expr#193=[65], expr#194=[*($t1, $t193)], expr#195=[+($t0, $t194)], expr#196=[66], expr#197=[*($t1, $t196)], expr#198=[+($t0, $t197)], expr#199=[67], expr#200=[*($t1, $t199)], expr#201=[+($t0, $t200)], expr#202=[68], expr#203=[*($t1, $t202)], expr#204=[+($t0, $t203)], expr#205=[69], expr#206=[*($t1, $t205)], expr#207=[+($t0, $t206)], expr#208=[70], expr#209=[*($t1, $t208)], expr#210=[+($t0, $t209)], expr#211=[71], expr#212=[*($t1, $t211)], expr#213=[+($t0, $t212)], expr#214=[72], expr#215=[*($t1, $t214)], expr#216=[+($t0, $t215)], expr#217=[73], expr#218=[*($t1, $t217)], expr#219=[+($t0, $t218)], expr#220=[74], expr#221=[*($t1, $t220)], expr#222=[+($t0, $t221)], expr#223=[75], expr#224=[*($t1, $t223)], expr#225=[+($t0, $t224)], expr#226=[76], expr#227=[*($t1, $t226)], expr#228=[+($t0, $t227)], expr#229=[77], expr#230=[*($t1, $t229)], expr#231=[+($t0, $t230)], expr#232=[78], expr#233=[*($t1, $t232)], expr#234=[+($t0, $t233)], expr#235=[79], expr#236=[*($t1, $t235)], expr#237=[+($t0, $t236)], expr#238=[80], expr#239=[*($t1, $t238)], expr#240=[+($t0, $t239)], expr#241=[81], expr#242=[*($t1, $t241)], expr#243=[+($t0, $t242)], expr#244=[82], expr#245=[*($t1, $t244)], expr#246=[+($t0, $t245)], expr#247=[83], expr#248=[*($t1, $t247)], expr#249=[+($t0, $t248)], expr#250=[84], expr#251=[*($t1, $t250)], expr#252=[+($t0, $t251)], expr#253=[85], expr#254=[*($t1, $t253)], expr#255=[+($t0, $t254)], expr#256=[86], expr#257=[*($t1, $t256)], expr#258=[+($t0, $t257)], expr#259=[87], expr#260=[*($t1, $t259)], expr#261=[+($t0, $t260)], expr#262=[88], expr#263=[*($t1, $t262)], expr#264=[+($t0, $t263)], expr#265=[89], expr#266=[*($t1, $t265)], expr#267=[+($t0, $t266)], sum(ResolutionWidth)=[$t0], sum(ResolutionWidth+1)=[$t3], sum(ResolutionWidth+2)=[$t6], sum(ResolutionWidth+3)=[$t9], sum(ResolutionWidth+4)=[$t12], sum(ResolutionWidth+5)=[$t15], sum(ResolutionWidth+6)=[$t18], sum(ResolutionWidth+7)=[$t21], sum(ResolutionWidth+8)=[$t24], sum(ResolutionWidth+9)=[$t27], sum(ResolutionWidth+10)=[$t30], sum(ResolutionWidth+11)=[$t33], sum(ResolutionWidth+12)=[$t36], sum(ResolutionWidth+13)=[$t39], sum(ResolutionWidth+14)=[$t42], sum(ResolutionWidth+15)=[$t45], sum(ResolutionWidth+16)=[$t48], sum(ResolutionWidth+17)=[$t51], sum(ResolutionWidth+18)=[$t54], sum(ResolutionWidth+19)=[$t57], sum(ResolutionWidth+20)=[$t60], sum(ResolutionWidth+21)=[$t63], sum(ResolutionWidth+22)=[$t66], sum(ResolutionWidth+23)=[$t69], sum(ResolutionWidth+24)=[$t72], sum(ResolutionWidth+25)=[$t75], sum(ResolutionWidth+26)=[$t78], sum(ResolutionWidth+27)=[$t81], sum(ResolutionWidth+28)=[$t84], sum(ResolutionWidth+29)=[$t87], sum(ResolutionWidth+30)=[$t90], sum(ResolutionWidth+31)=[$t93], sum(ResolutionWidth+32)=[$t96], sum(ResolutionWidth+33)=[$t99], sum(ResolutionWidth+34)=[$t102], sum(ResolutionWidth+35)=[$t105], sum(ResolutionWidth+36)=[$t108], sum(ResolutionWidth+37)=[$t111], sum(ResolutionWidth+38)=[$t114], sum(ResolutionWidth+39)=[$t117], sum(ResolutionWidth+40)=[$t120], sum(ResolutionWidth+41)=[$t123], sum(ResolutionWidth+42)=[$t126], sum(ResolutionWidth+43)=[$t129], sum(ResolutionWidth+44)=[$t132], sum(ResolutionWidth+45)=[$t135], sum(ResolutionWidth+46)=[$t138], sum(ResolutionWidth+47)=[$t141], sum(ResolutionWidth+48)=[$t144], sum(ResolutionWidth+49)=[$t147], sum(ResolutionWidth+50)=[$t150], sum(ResolutionWidth+51)=[$t153], sum(ResolutionWidth+52)=[$t156], sum(ResolutionWidth+53)=[$t159], sum(ResolutionWidth+54)=[$t162], sum(ResolutionWidth+55)=[$t165], sum(ResolutionWidth+56)=[$t168], sum(ResolutionWidth+57)=[$t171], sum(ResolutionWidth+58)=[$t174], sum(ResolutionWidth+59)=[$t177], sum(ResolutionWidth+60)=[$t180], sum(ResolutionWidth+61)=[$t183], sum(ResolutionWidth+62)=[$t186], sum(ResolutionWidth+63)=[$t189], sum(ResolutionWidth+64)=[$t192], sum(ResolutionWidth+65)=[$t195], sum(ResolutionWidth+66)=[$t198], sum(ResolutionWidth+67)=[$t201], sum(ResolutionWidth+68)=[$t204], sum(ResolutionWidth+69)=[$t207], sum(ResolutionWidth+70)=[$t210], sum(ResolutionWidth+71)=[$t213], sum(ResolutionWidth+72)=[$t216], sum(ResolutionWidth+73)=[$t219], sum(ResolutionWidth+74)=[$t222], sum(ResolutionWidth+75)=[$t225], sum(ResolutionWidth+76)=[$t228], sum(ResolutionWidth+77)=[$t231], sum(ResolutionWidth+78)=[$t234], sum(ResolutionWidth+79)=[$t237], sum(ResolutionWidth+80)=[$t240], sum(ResolutionWidth+81)=[$t243], sum(ResolutionWidth+82)=[$t246], sum(ResolutionWidth+83)=[$t249], sum(ResolutionWidth+84)=[$t252], sum(ResolutionWidth+85)=[$t255], sum(ResolutionWidth+86)=[$t258], sum(ResolutionWidth+87)=[$t261], sum(ResolutionWidth+88)=[$t264], sum(ResolutionWidth+89)=[$t267]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(ResolutionWidth)=SUM($0),sum(ResolutionWidth+1)_COUNT=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(ResolutionWidth)":{"sum":{"field":"ResolutionWidth"}},"sum(ResolutionWidth+1)_COUNT":{"value_count":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[1:BIGINT], expr#3=[+($t1, $t2)], expr#4=[2:BIGINT], expr#5=[+($t1, $t4)], expr#6=[3:BIGINT], expr#7=[+($t1, $t6)], expr#8=[4:BIGINT], expr#9=[+($t1, $t8)], expr#10=[5:BIGINT], expr#11=[+($t1, $t10)], expr#12=[6:BIGINT], expr#13=[+($t1, $t12)], expr#14=[7:BIGINT], expr#15=[+($t1, $t14)], expr#16=[8:BIGINT], expr#17=[+($t1, $t16)], expr#18=[9:BIGINT], expr#19=[+($t1, $t18)], expr#20=[10:BIGINT], expr#21=[+($t1, $t20)], expr#22=[11:BIGINT], expr#23=[+($t1, $t22)], expr#24=[12:BIGINT], expr#25=[+($t1, $t24)], expr#26=[13:BIGINT], expr#27=[+($t1, $t26)], expr#28=[14:BIGINT], expr#29=[+($t1, $t28)], expr#30=[15:BIGINT], expr#31=[+($t1, $t30)], expr#32=[16:BIGINT], expr#33=[+($t1, $t32)], expr#34=[17:BIGINT], expr#35=[+($t1, $t34)], expr#36=[18:BIGINT], expr#37=[+($t1, $t36)], expr#38=[19:BIGINT], expr#39=[+($t1, $t38)], expr#40=[20:BIGINT], expr#41=[+($t1, $t40)], expr#42=[21:BIGINT], expr#43=[+($t1, $t42)], expr#44=[22:BIGINT], expr#45=[+($t1, $t44)], expr#46=[23:BIGINT], expr#47=[+($t1, $t46)], expr#48=[24:BIGINT], expr#49=[+($t1, $t48)], expr#50=[25:BIGINT], expr#51=[+($t1, $t50)], expr#52=[26:BIGINT], expr#53=[+($t1, $t52)], expr#54=[27:BIGINT], expr#55=[+($t1, $t54)], expr#56=[28:BIGINT], expr#57=[+($t1, $t56)], expr#58=[29:BIGINT], expr#59=[+($t1, $t58)], expr#60=[30:BIGINT], expr#61=[+($t1, $t60)], expr#62=[31:BIGINT], expr#63=[+($t1, $t62)], expr#64=[32:BIGINT], expr#65=[+($t1, $t64)], expr#66=[33:BIGINT], expr#67=[+($t1, $t66)], expr#68=[34:BIGINT], expr#69=[+($t1, $t68)], expr#70=[35:BIGINT], expr#71=[+($t1, $t70)], expr#72=[36:BIGINT], expr#73=[+($t1, $t72)], expr#74=[37:BIGINT], expr#75=[+($t1, $t74)], expr#76=[38:BIGINT], expr#77=[+($t1, $t76)], expr#78=[39:BIGINT], expr#79=[+($t1, $t78)], expr#80=[40:BIGINT], expr#81=[+($t1, $t80)], expr#82=[41:BIGINT], expr#83=[+($t1, $t82)], expr#84=[42:BIGINT], expr#85=[+($t1, $t84)], expr#86=[43:BIGINT], expr#87=[+($t1, $t86)], expr#88=[44:BIGINT], expr#89=[+($t1, $t88)], expr#90=[45:BIGINT], expr#91=[+($t1, $t90)], expr#92=[46:BIGINT], expr#93=[+($t1, $t92)], expr#94=[47:BIGINT], expr#95=[+($t1, $t94)], expr#96=[48:BIGINT], expr#97=[+($t1, $t96)], expr#98=[49:BIGINT], expr#99=[+($t1, $t98)], expr#100=[50:BIGINT], expr#101=[+($t1, $t100)], expr#102=[51:BIGINT], expr#103=[+($t1, $t102)], expr#104=[52:BIGINT], expr#105=[+($t1, $t104)], expr#106=[53:BIGINT], expr#107=[+($t1, $t106)], expr#108=[54:BIGINT], expr#109=[+($t1, $t108)], expr#110=[55:BIGINT], expr#111=[+($t1, $t110)], expr#112=[56:BIGINT], expr#113=[+($t1, $t112)], expr#114=[57:BIGINT], expr#115=[+($t1, $t114)], expr#116=[58:BIGINT], expr#117=[+($t1, $t116)], expr#118=[59:BIGINT], expr#119=[+($t1, $t118)], expr#120=[60:BIGINT], expr#121=[+($t1, $t120)], expr#122=[61:BIGINT], expr#123=[+($t1, $t122)], expr#124=[62:BIGINT], expr#125=[+($t1, $t124)], expr#126=[63:BIGINT], expr#127=[+($t1, $t126)], expr#128=[64:BIGINT], expr#129=[+($t1, $t128)], expr#130=[65:BIGINT], expr#131=[+($t1, $t130)], expr#132=[66:BIGINT], expr#133=[+($t1, $t132)], expr#134=[67:BIGINT], expr#135=[+($t1, $t134)], expr#136=[68:BIGINT], expr#137=[+($t1, $t136)], expr#138=[69:BIGINT], expr#139=[+($t1, $t138)], expr#140=[70:BIGINT], expr#141=[+($t1, $t140)], expr#142=[71:BIGINT], expr#143=[+($t1, $t142)], expr#144=[72:BIGINT], expr#145=[+($t1, $t144)], expr#146=[73:BIGINT], expr#147=[+($t1, $t146)], expr#148=[74:BIGINT], expr#149=[+($t1, $t148)], expr#150=[75:BIGINT], expr#151=[+($t1, $t150)], expr#152=[76:BIGINT], expr#153=[+($t1, $t152)], expr#154=[77:BIGINT], expr#155=[+($t1, $t154)], expr#156=[78:BIGINT], expr#157=[+($t1, $t156)], expr#158=[79:BIGINT], expr#159=[+($t1, $t158)], expr#160=[80:BIGINT], expr#161=[+($t1, $t160)], expr#162=[81:BIGINT], expr#163=[+($t1, $t162)], expr#164=[82:BIGINT], expr#165=[+($t1, $t164)], expr#166=[83:BIGINT], expr#167=[+($t1, $t166)], expr#168=[84:BIGINT], expr#169=[+($t1, $t168)], expr#170=[85:BIGINT], expr#171=[+($t1, $t170)], expr#172=[86:BIGINT], expr#173=[+($t1, $t172)], expr#174=[87:BIGINT], expr#175=[+($t1, $t174)], expr#176=[88:BIGINT], expr#177=[+($t1, $t176)], expr#178=[89:BIGINT], expr#179=[+($t1, $t178)], ResolutionWidth=[$t0], $f90=[$t3], $f91=[$t5], $f92=[$t7], $f93=[$t9], $f94=[$t11], $f95=[$t13], $f96=[$t15], $f97=[$t17], $f98=[$t19], $f99=[$t21], $f100=[$t23], $f101=[$t25], $f102=[$t27], $f103=[$t29], $f104=[$t31], $f105=[$t33], $f106=[$t35], $f107=[$t37], $f108=[$t39], $f109=[$t41], $f110=[$t43], $f111=[$t45], $f112=[$t47], $f113=[$t49], $f114=[$t51], $f115=[$t53], $f116=[$t55], $f117=[$t57], $f118=[$t59], $f119=[$t61], $f120=[$t63], $f121=[$t65], $f122=[$t67], $f123=[$t69], $f124=[$t71], $f125=[$t73], $f126=[$t75], $f127=[$t77], $f128=[$t79], $f129=[$t81], $f130=[$t83], $f131=[$t85], $f132=[$t87], $f133=[$t89], $f134=[$t91], $f135=[$t93], $f136=[$t95], $f137=[$t97], $f138=[$t99], $f139=[$t101], $f140=[$t103], $f141=[$t105], $f142=[$t107], $f143=[$t109], $f144=[$t111], $f145=[$t113], $f146=[$t115], $f147=[$t117], $f148=[$t119], $f149=[$t121], $f150=[$t123], $f151=[$t125], $f152=[$t127], $f153=[$t129], $f154=[$t131], $f155=[$t133], $f156=[$t135], $f157=[$t137], $f158=[$t139], $f159=[$t141], $f160=[$t143], $f161=[$t145], $f162=[$t147], $f163=[$t149], $f164=[$t151], $f165=[$t153], $f166=[$t155], $f167=[$t157], $f168=[$t159], $f169=[$t161], $f170=[$t163], $f171=[$t165], $f172=[$t167], $f173=[$t169], $f174=[$t171], $f175=[$t173], $f176=[$t175], $f177=[$t177], $f178=[$t179]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[ResolutionWidth]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["ResolutionWidth"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml index 5f1b457eaa1..d964a1422a1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q36.yaml @@ -6,8 +6,8 @@ calcite: LogicalAggregate(group=[{0, 1, 2, 3}], c=[COUNT()]) LogicalProject(ClientIP=[$76], ClientIP - 1=[$111], ClientIP - 2=[$112], ClientIP - 3=[$113]) LogicalFilter(condition=[AND(IS NOT NULL($76), IS NOT NULL($111), IS NOT NULL($112), IS NOT NULL($113))]) - LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-($76, 1)], ClientIP - 2=[-($76, 2)], ClientIP - 3=[-($76, 3)]) + LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], ClientIP - 1=[-(CAST($76):BIGINT, 1)], ClientIP - 2=[-(CAST($76):BIGINT, 2)], ClientIP - 3=[-(CAST($76):BIGINT, 3)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[-($t0, $t2)], expr#4=[2], expr#5=[-($t0, $t4)], expr#6=[3], expr#7=[-($t0, $t6)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t3], ClientIP - 2=[$t5], ClientIP - 3=[$t7]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[1:BIGINT], expr#4=[-($t2, $t3)], expr#5=[2:BIGINT], expr#6=[-($t2, $t5)], expr#7=[3:BIGINT], expr#8=[-($t2, $t7)], c=[$t1], ClientIP=[$t0], ClientIP - 1=[$t4], ClientIP - 2=[$t6], ClientIP - 3=[$t8]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"ClientIP":{"terms":{"field":"ClientIP","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml index acd95f0ec63..841e131df29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(age1=[*($8, 10)], age2=[+($8, 10)], age3=[10], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml index e7589d8109d..4f57bbf4cd4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -8,5 +8,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[>($t6, $t5)], expr#9=[OR($t7, $t8)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t9]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1:BIGINT], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[1], expr#9=[>($t6, $t8)], expr#10=[OR($t7, $t9)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index c543431c519..bc65d5c4c29 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index c17bd10e18a..1d664d5cd43 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index 4a9a143cba3..c42ecef2132 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,7 +3,7 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index e8e9ac1f4f2..ffd55ffb1fb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -2,8 +2,8 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index 5aa37ee3296..64e868f9f8c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index d80ebc5735b..8f60e23e491 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age2=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age2=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index 31efd3c688c..7ad040f826d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[-($t13, $t10)], proj#0..14=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+($10, $7), $10) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCsHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[-($t14, $t13)], proj#0..12=[{exprs}], age2=[$t14], age3=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEzHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml index 3fd07a9682e..5aef351b780 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_then_field_sort.yaml @@ -5,10 +5,10 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[ABS($t7)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml index 690c3ce24e7..27110780ce4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], ageMinus=[$t2]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30:BIGINT], expr#2=[-($t0, $t1)], ageMinus=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml index 042787a458b..e98a407d378 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(CAST($2):BIGINT, +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,6 +12,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(CAST($1):BIGINT, +($2, 999))], joinType=[inner]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"name","boost":1.0}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json index 066b678a960..2c5f37e6813 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], age=[$t0], age2=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[2:BIGINT], expr#3=[+($t1, $t2)], age=[$t0], age2=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml index d0e66b1d14f..e4274781967 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -12,14 +12,14 @@ calcite: EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[1:BIGINT], expr#15=[+($t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ "age" : { "order" : "asc", "missing" : "_last" } }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20:BIGINT], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ "balance" : { "order" : "asc", diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json index 0a36ec4648d..7494f2453c3 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1:BIGINT], expr#2=[+($t0, $t1)], b=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index 08e75fbbdeb..f5834404f19 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -3,8 +3,8 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[1], expr#15=[+($t7, $t14)], proj#0..13=[{exprs}], balance2=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[1:BIGINT], expr#16=[+($t7, $t15)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml index a694c63b2ca..28bee6245d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_group_merge.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10:BIGINT], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], expr#5=[10], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t5], age=[$t0]) EnumerableAggregate(group=[{8}], count()=[COUNT()]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 285d0b221e1..1db12fc013f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index bf861c337b9..655e16839ed 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml index 5c479c6867e..575e40c0f77 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_no_expr_output_push.yaml @@ -3,11 +3,11 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], age=[$t0]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml index a95c277b40e..2eb653bfbb6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_project_then_sort.yaml @@ -2,10 +2,10 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) - LogicalProject(age=[$10], age2=[+($10, $7)]) + LogicalProject(age=[$10], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml index ef4ea5fc43e..723ca37b3e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age=[$10], age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age=[$t10], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age=[$t10], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml index 7df4a4d7f4e..87baf3c8399 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_single_expr_output_push.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(age2=[$19]) LogicalSort(sort0=[$19], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml index 711608264eb..1a9c720ee8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_nested_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$14], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], age3=[$20]) LogicalSort(sort0=[$20], dir0=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], age3=[-(+($10, $7), $10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], age3=[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$14], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[-($t19, $t10)], proj#0..12=[{exprs}], age2=[$t19], age3=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[-($t20, $t19)], proj#0..12=[{exprs}], age2=[$t20], age3=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml index 362f847ae6e..156ec314c53 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_then_field_sort.yaml @@ -5,11 +5,11 @@ calcite: LogicalSort(sort0=[$10], dir0=[ASC-nulls-first]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[$19], balance2=[ABS($7)]) LogicalSort(sort0=[$19], sort1=[$10], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..13=[{inputs}], expr#14=[ABS($t7)], proj#0..14=[{exprs}]) EnumerableSort(sort0=[$10], dir0=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], proj#0..12=[{exprs}], age2=[$t19]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], proj#0..12=[{exprs}], age2=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml index 90492abbaf8..a1b5e3d3ed5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_push.yaml @@ -6,5 +6,5 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['Amber':VARCHAR], expr#18=[=($t1, $t17)], expr#19=[2:BIGINT], expr#20=[-($t8, $t19)], expr#21=[30], expr#22=[=($t20, $t21)], expr#23=[AND($t18, $t22)], firstname=[$t1], age=[$t8], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml index fb3daa06769..2ee55737ac2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml @@ -7,6 +7,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30:BIGINT], expr#18=[-($t8, $t17)], ageMinus=[$t18]) EnumerableLimit(fetch=[5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml index ba13359c44d..0c4b5d3f606 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_uncorrelated_subquery_in_where.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(name=[$0]) - LogicalFilter(condition=[>($2, +($SCALAR_QUERY({ + LogicalFilter(condition=[>(CAST($2):BIGINT, +($SCALAR_QUERY({ LogicalAggregate(group=[{}], count(name)=[COUNT($0)]) LogicalProject(name=[$0]) LogicalFilter(condition=[IS NOT NULL($0)]) @@ -12,9 +12,9 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..2=[{inputs}], name=[$t0]) - EnumerableNestedLoopJoin(condition=[>($1, +($2, 999))], joinType=[inner]) + EnumerableNestedLoopJoin(condition=[>(CAST($1):BIGINT, +($2, 999))], joinType=[inner]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t0)], proj#0..9=[{exprs}], $condition=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json index adb4cb6244d..b48850334a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[2], expr#20=[+($t10, $t19)], age=[$t10], age2=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[2:BIGINT], expr#21=[+($t19, $t20)], age=[$t10], age2=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml index 8897a1023cc..30be0be9ca7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) - LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalJoin(condition=[=(+(CAST($10):BIGINT, 1), -($20, 20))], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) @@ -13,9 +13,9 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[1:BIGINT], expr#21=[+($t19, $t20)], proj#0..12=[{exprs}], $f13=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$13], dir0=[ASC]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20:BIGINT], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) EnumerableLimit(fetch=[50000]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json index 67cf82580e9..2fffb64bef5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:BIGINT], expr#20=[+($t7, $t19)], b=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml index 873a778f979..df4e941dfbe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_complex_and_simple_expr.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], age2=[$19], balance2=[$20]) LogicalSort(sort0=[$19], sort1=[$20], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)], balance2=[+($7, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)], balance2=[+($7, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$13], sort1=[$14], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[+($t10, $t7)], expr#20=[1], expr#21=[+($t7, $t20)], proj#0..12=[{exprs}], age2=[$t19], balance2=[$t21]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CAST($t10):BIGINT], expr#20=[+($t19, $t7)], expr#21=[1:BIGINT], expr#22=[+($t7, $t21)], proj#0..12=[{exprs}], age2=[$t20], balance2=[$t22]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java index 56ed409b4d7..dc476b77cff 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendPipeTest.java @@ -44,18 +44,18 @@ public void testAppendPipeWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java index a163af186d5..027062485c6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAppendTest.java @@ -211,18 +211,18 @@ public void testAppendWithMergedColumns() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalUnion(all=[true])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:INTEGER])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[null:BIGINT])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+($7, 10)])\n" + + " LogicalProject(DEPTNO=[$7], DEPTNO_PLUS=[+(CAST($7):BIGINT, 10)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); verifyResultCount(root, 28); String expectedSparkSql = - "SELECT `DEPTNO`, CAST(NULL AS INTEGER) `DEPTNO_PLUS`\n" + "SELECT `DEPTNO`, CAST(NULL AS BIGINT) `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "UNION ALL\n" - + "SELECT `DEPTNO`, `DEPTNO` + 10 `DEPTNO_PLUS`\n" + + "SELECT `DEPTNO`, CAST(`DEPTNO` AS BIGINT) + 10 `DEPTNO_PLUS`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java index 1d6792b0990..fbbee80ff79 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLArrayFunctionTest.java @@ -656,8 +656,8 @@ public void testMvmapWithNestedFunction() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr` * 10)" - + " `result`\n" + "SELECT TRANSFORM(ARRAY_SLICE(ARRAY(1, 2, 3, 4, 5), 1, 3 - 1 + 1), `arr` -> `arr`" + + " * 10) `result`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java index 5f32c1b85bb..3c13297f8f0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java @@ -198,7 +198,7 @@ public void testDedupExpr() { + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4)])\n" + " LogicalFilter(condition=[IS NOT NULL($4)])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], DEPTNO=[$7]," - + " NEW_DEPTNO=[+($7, 1)])\n" + + " NEW_DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -216,7 +216,8 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -229,10 +230,10 @@ public void testDedupExpr() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } @@ -277,10 +278,10 @@ public void testSortThenDedupWithEval() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); // After fix, the sort order (NEW_DEPTNO ASC) must be preserved through dedup. @@ -304,7 +305,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -317,7 +319,8 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $3)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); ppl = @@ -330,10 +333,10 @@ public void testRenameDedup() { + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3])\n" + " LogicalFilter(condition=[<=($4, 1)])\n" + " LogicalProject(NEW_DEPTNO=[$0], EMPNO=[$1], ENAME=[$2], JOB=[$3]," - + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS" - + " FIRST)])\n" + + " _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $0 NULLS FIRST)])\n" + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalProject(NEW_DEPTNO=[+($7, 1)], EMPNO=[$0], ENAME=[$1], JOB=[$2])\n" + + " LogicalProject(NEW_DEPTNO=[+(CAST($7):BIGINT, 1)], EMPNO=[$0], ENAME=[$1]," + + " JOB=[$2])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java index 9b37ab5b407..3cd9e417a58 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java @@ -104,7 +104,7 @@ public void testEvalSum() { String ppl = "source=EMP | eval total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -131,7 +131,8 @@ public void testEvalAvg() { String ppl = "source=EMP | eval average = avg(10, 20, 30) | fields EMPNO, average"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20, 30)), 3.0E0:DOUBLE)])\n" + "LogicalProject(EMPNO=[$0], average=[DIVIDE(+(10, +(20:BIGINT, 30:BIGINT))," + + " 3.0E0:DOUBLE)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -210,7 +211,7 @@ public void testEvalUsingExistingFields() { "LogicalProject(EMPNO=[$0], EMPNO_PLUS=[$8])\n" + " LogicalSort(sort0=[$8], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+($0, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], EMPNO_PLUS=[+(CAST($0):BIGINT NOT NULL, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -223,7 +224,7 @@ public void testEvalUsingExistingFields() { String expectedSparkSql = "SELECT `EMPNO`, `EMPNO_PLUS`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " `EMPNO` + 1 `EMPNO_PLUS`\n" + + " CAST(`EMPNO` AS BIGINT) + 1 `EMPNO_PLUS`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY 9 DESC\n" + "LIMIT 3) `t0`"; @@ -239,7 +240,7 @@ public void testEvalOverridingExistingFields() { "LogicalProject(EMPNO=[$0], SAL=[$7])\n" + " LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[3])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " COMM=[$6], DEPTNO=[$7], SAL=[+($7, 10000)])\n" + + " COMM=[$6], DEPTNO=[$7], SAL=[+(CAST($7):BIGINT, 10000)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -248,7 +249,7 @@ public void testEvalOverridingExistingFields() { String expectedSparkSql = "" - + "SELECT `EMPNO`, `DEPTNO` + 10000 `SAL`\n" + + "SELECT `EMPNO`, CAST(`DEPTNO` AS BIGINT) + 10000 `SAL`\n" + "FROM `scott`.`EMP`\n" + "ORDER BY `EMPNO` DESC\n" + "LIMIT 3"; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java index 76c280db92f..84b509d16e9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java @@ -490,14 +490,14 @@ public void testCorrelatedExistsSubqueryWithOverridingFields() { + " LogicalTableScan(table=[[scott, DEPT]])\n" + "})], variablesSet=[[$cor0]])\n" + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," - + " SAL=[$5], COMM=[$6], DEPTNO=[+($7, 1)])\n" + + " SAL=[$5], COMM=[$6], DEPTNO=[+(CAST($7):BIGINT, 1)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT *\n" - + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO` + 1" - + " `DEPTNO`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`," + + " CAST(`DEPTNO` AS BIGINT) + 1 `DEPTNO`\n" + "FROM `scott`.`EMP`) `t`\n" + "WHERE EXISTS (SELECT *\n" + "FROM `scott`.`DEPT`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java index 5bef9c397eb..85dfb3eb650 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -177,7 +177,7 @@ public void testFieldFormatSum() { "source=EMP |sort EMPNO | head 3| fieldformat total = sum(1, 2, 3) | fields EMPNO, total"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + "LogicalProject(EMPNO=[$0], total=[+(1, +(2:BIGINT, 3:BIGINT))])\n" + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; From c0d75c9c627bc644cb2f5c061034ec4fe34f7016 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 7 Jul 2026 16:20:56 -0400 Subject: [PATCH 29/41] Onboard new backport-pr re-usable github workflow (sql) (#5586) - Replace old backport workflow with reusable workflow from opensearch-build - Remove obsolete backport-related workflows Signed-off-by: Peter Zhu --- .github/workflows/backport.yml | 30 ++++---------------- .github/workflows/delete_backport_branch.yml | 22 -------------- 2 files changed, 6 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/delete_backport_branch.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 55002cf6dc5..9e7c5c2136e 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,30 +1,12 @@ +--- name: Backport on: pull_request_target: - types: - - closed - - labeled + types: [closed, labeled] jobs: backport: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - name: Backport - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - - name: Backport - uses: VachaShah/backport@142d3b8a8c70dc54db515e653e5ed3c3fac64100 # v2.2.0 - with: - github_token: ${{ steps.github_app_token.outputs.token }} - head_template: backport/backport-<%= number %>-to-<%= base %> - failure_labels: backport-failed + if: github.repository == 'opensearch-project/sql' + uses: opensearch-project/opensearch-build/.github/workflows/backport-pr.yml@main + secrets: + OPENSEARCH_CI_BOT_TOKEN: ${{ secrets.OPENSEARCH_CI_BOT_TOKEN }} diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml deleted file mode 100644 index 61089ed1334..00000000000 --- a/.github/workflows/delete_backport_branch.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Delete merged branch of the backport PRs -on: - pull_request: - types: - - closed - -jobs: - delete-branch: - runs-on: ubuntu-latest - permissions: - pull-requests: write - if: startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/') - steps: - - name: Delete merged branch - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - github.rest.git.deleteRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: `heads/${context.payload.pull_request.head.ref}`, - }) From 3d4938a0aad4978fe8e4952f47d735f402f01c0d Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 13 Jul 2026 11:47:21 -0700 Subject: [PATCH 30/41] fix: Gracefully handle malformed documents in result scanning (#5618) Signed-off-by: Simeon Widdis --- .../value/OpenSearchExprValueFactory.java | 17 ++- .../response/OpenSearchResponse.java | 21 +++- .../value/OpenSearchExprValueFactoryTest.java | 101 ++++++------------ .../response/OpenSearchResponseTest.java | 35 ++++++ 4 files changed, 99 insertions(+), 75 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index d772b3e603b..b4fa498e2ac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -209,7 +209,11 @@ private ExprValue parse( final ExprType type = fieldType.get(); if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))) { - return parseGeoPoint(content, supportArrays); + try { + return parseGeoPoint(content, supportArrays); + } catch (Exception e) { + return ExprNullValue.of(); + } } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) || content.isArray()) { return parseArray(content, field, type, supportArrays); @@ -217,9 +221,14 @@ private ExprValue parse( || type == STRUCT) { return parseStruct(content, field, supportArrays); } else if (typeActionMap.containsKey(type)) { - return content.isArray() - ? parseArray(content, field, type, supportArrays) - : typeActionMap.get(type).apply(content, type); + if (content.isArray()) { + return parseArray(content, field, type, supportArrays); + } + try { + return typeActionMap.get(type).apply(content, type); + } catch (Exception e) { + return ExprNullValue.of(); + } } else { throw new IllegalStateException( String.format( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 0a47dc64a5e..03e2d546418 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -18,10 +18,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.TestOnly; import org.opensearch.action.search.SearchResponse; import org.opensearch.core.common.text.Text; @@ -42,6 +45,8 @@ @ToString public class OpenSearchResponse implements Iterable { + private static final Logger LOG = LogManager.getLogger(); + public static final OpenSearchResponse EMPTY = empty(); /** Search query result (non-aggregation). */ @@ -155,12 +160,18 @@ public Iterator iterator() { return Arrays.stream(hits.getHits()) .map( hit -> { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - addParsedHitsToBuilder(builder, hit); - addMetaDataFieldsToBuilder(builder, hit); - addHighlightsToBuilder(builder, hit); - return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); + try { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + addParsedHitsToBuilder(builder, hit); + addMetaDataFieldsToBuilder(builder, hit); + addHighlightsToBuilder(builder, hit); + return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); + } catch (Exception e) { + LOG.warn("Failed to parse document {}, skipping", hit.getId(), e); + return null; + } }) + .filter(Objects::nonNull) .iterator(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 031b9243f38..7c27c995e4c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -48,7 +48,6 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.junit.jupiter.api.Test; -import org.opensearch.OpenSearchParseException; import org.opensearch.geometry.utils.Geohash; import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; @@ -230,6 +229,25 @@ public void constructIp() { constructFromObject("ipV", "2001:db7::ff00:42:8329"))); } + @Test + public void constructIpFromInvalidString_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"ipV\":\"not-an-ip\"}").get("ipV")); + assertEquals(nullValue(), constructFromObject("ipV", "garbage")); + } + + @Test + public void constructNumericFromObjectNode_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"intV\":{\"nested\":\"value\"}}").get("intV")); + assertEquals(nullValue(), tupleValue("{\"longV\":{\"nested\":\"value\"}}").get("longV")); + assertEquals(nullValue(), tupleValue("{\"floatV\":{\"nested\":\"value\"}}").get("floatV")); + assertEquals(nullValue(), tupleValue("{\"doubleV\":{\"nested\":\"value\"}}").get("doubleV")); + } + + @Test + public void constructBooleanFromObjectNode_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"boolV\":{\"nested\":\"value\"}}").get("boolV")); + } + @Test public void constructBoolean() { assertAll( @@ -374,13 +392,7 @@ public void constructDatetime_fromCustomFormat() { new ExprTimestampValue("2015-01-01 12:10:30"), constructFromObject("customFormatV", "2015-01-01-12-10-30")); - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("customFormatV", "2015-01-01 12-10-30")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12-10-30\" failed, unsupported format.", - exception.getMessage()); + assertEquals(nullValue(), constructFromObject("customFormatV", "2015-01-01 12-10-30")); assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), @@ -388,52 +400,21 @@ public void constructDatetime_fromCustomFormat() { } @Test - public void constructDatetimeFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("timestampV", "2015-01-01 12:10")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, unsupported format.", - exception.getMessage()); - - // fail with missing seconds - exception = - assertThrows( - IllegalArgumentException.class, - () -> constructFromObject("dateOrEpochMillisV", "2015-01-01 12:10")); - assertEquals( - "Construct TIMESTAMP from \"2015-01-01 12:10\" failed, unsupported format.", - exception.getMessage()); + public void constructDatetimeFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("timestampV", "2015-01-01 12:10")); + assertEquals(nullValue(), constructFromObject("dateOrEpochMillisV", "2015-01-01 12:10")); } @Test - public void constructTimeFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("timeV", "2015-01-01")); - assertEquals( - "Construct TIME from \"2015-01-01\" failed, unsupported format.", exception.getMessage()); - - exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("timeStringV", "10:10")); - assertEquals( - "Construct TIME from \"10:10\" failed, unsupported format.", exception.getMessage()); + public void constructTimeFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("timeV", "2015-01-01")); + assertEquals(nullValue(), constructFromObject("timeStringV", "10:10")); } @Test - public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { - IllegalArgumentException exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("dateV", "12:10:10")); - assertEquals( - "Construct DATE from \"12:10:10\" failed, unsupported format.", exception.getMessage()); - - exception = - assertThrows( - IllegalArgumentException.class, () -> constructFromObject("dateStringV", "abc")); - assertEquals("Construct DATE from \"abc\" failed, unsupported format.", exception.getMessage()); + public void constructDateFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), constructFromObject("dateV", "12:10:10")); + assertEquals(nullValue(), constructFromObject("dateStringV", "abc")); } @Test @@ -803,24 +784,12 @@ public void constructGeoPoint() { } @Test - public void constructGeoPointFromUnsupportedFormatShouldThrowException() { - OpenSearchParseException exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\": [42.60355556, false]}").get("geoV")); - assertEquals("lat must be a number, got false", exception.getMessage()); - - exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV")); - assertEquals("field [lat] missing", exception.getMessage()); - - exception = - assertThrows( - OpenSearchParseException.class, - () -> tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV")); - assertEquals("lat must be a number", exception.getMessage()); + public void constructGeoPointFromUnsupportedFormat_ReturnsNull() { + assertEquals(nullValue(), tupleValue("{\"geoV\": [42.60355556, false]}").get("geoV")); + assertEquals(nullValue(), tupleValue("{\"geoV\":{\"lon\":-97.25263889}}").get("geoV")); + assertEquals( + nullValue(), tupleValue("{\"geoV\":{\"lat\":true,\"lon\":-97.25263889}}").get("geoV")); + assertEquals(nullValue(), tupleValue("{\"geoV\":\"not-a-geo-point\"}").get("geoV")); } @Test diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index f9897f48dd2..938fb56f47b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.response; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,8 +18,10 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -340,4 +343,36 @@ void highlight_iterator() { assertTrue(expected.equals(result)); } } + + @Test + void iterator_skipsDocumentWhenConstructThrows() { + when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit1, searchHit2}, + new TotalHits(2L, TotalHits.Relation.EQUAL_TO), + 1.0F)); + + when(searchHit1.getSourceAsString()).thenReturn("{\"id1\": 1}"); + when(searchHit1.getInnerHits()).thenReturn(null); + when(searchHit1.getId()).thenReturn("doc1"); + when(searchHit2.getSourceAsString()).thenReturn("{\"id1\": 2}"); + when(searchHit2.getInnerHits()).thenReturn(null); + + when(factory.construct(any(), anyBoolean())) + .thenThrow(new RuntimeException("simulated parse failure")) + .thenReturn(exprTupleValue2); + + List results = new ArrayList<>(); + assertDoesNotThrow( + () -> { + Iterator it = + OpenSearchResponse.of(searchResponse, factory, List.of("id1")).iterator(); + while (it.hasNext()) { + results.add(it.next()); + } + }); + assertEquals(1, results.size()); + assertEquals(exprTupleValue2, results.get(0)); + } } From 3a837689cb204809d2d723f7d76f37b537ef7a4c Mon Sep 17 00:00:00 2001 From: Radhakrishnan Pachyappan Date: Tue, 14 Jul 2026 23:54:20 +0530 Subject: [PATCH 31/41] fix(dedup): use Map> for fieldNameMapping to handle alias collision (#5593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dedup): use Map> in fieldNameMapping to handle alias collision (#5197) When `rename` and `eval` column-ref both resolve to the same source field (e.g. `eval nm2 = name | rename name as nm`), the previous Map approach silently dropped one mapping on collision. This commit implements the fix from scratch (PR #5192 was closed unmerged): * TopHitsParser: add an optional `Map> fieldNameMapping` field and a new 4-arg constructor; the 3-arg constructor delegates with null (no-op). `applyFieldNameMapping()` copies the source-field value to every output alias and removes the original key only when it is not itself an expected output name. * AggregateAnalyzer: in the LITERAL_AGG (dedup) branch, build fieldNameMapping by iterating over the projection args; pass it to TopHitsParser when non-empty. * Unit tests (OpenSearchAggregationResponseParserTest): - `top_hits_field_name_mapping_single_rename_should_pass` – regression for #5150 - `top_hits_field_name_mapping_collision_should_duplicate_value` – regression for #5197 * Integration tests (CalcitePPLDedupIT): - `testDedupWithRenamedField` – dedup after rename, single alias - `testDedupWithRenamedFieldMappingCollision` – dedup after both rename and eval alias Fixes #5197 Signed-off-by: Radhakrishnan Pachyappan * review: address PR feedback — null guard and map.get optimisation * AggregateAnalyzer: guard against a null return from inferNamedField before calling getRootName() (defensive; the method returns non-null for RexInputRef today but the null check makes the contract explicit). * TopHitsParser.applyFieldNameMapping: replace the containsKey+get double lookup with a single map.get() call; null value + absent key is distinguished via containsKey only when value is null, eliminating the redundant containsKey in the common (non-null value) path. Fixes #5150 Fixes #5197 Signed-off-by: Radhakrishnan Pachyappan * test: fix testDedupWithRenamedField* expected rows for category Y The test data (duplication_nullable.json) has category=Y rows in this order: A (id 2), A (id 3), null (id 8), A (id 12), B (id 15). 'dedup 1 category' keeps the FIRST occurrence per category, which has name=A for category Y, not B. Update expected rows: rows("Y","B") -> rows("Y","A") and rows("Y","B","B") -> rows("Y","A","A"). Signed-off-by: RadhaKrishnan Rajendran Signed-off-by: Radhakrishnan Pachyappan --------- Signed-off-by: Radhakrishnan Pachyappan Signed-off-by: RadhaKrishnan Rajendran --- .../sql/calcite/remote/CalcitePPLDedupIT.java | 39 ++++++++ .../opensearch/request/AggregateAnalyzer.java | 27 +++++- .../response/agg/TopHitsParser.java | 51 +++++++++++ ...enSearchAggregationResponseParserTest.java | 90 +++++++++++++++++++ 4 files changed, 206 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java index 9c93b12e6ac..4177d108440 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java @@ -348,6 +348,45 @@ public void testMultiColumnSortThenDedup() throws IOException { verifyDataRows(actual, rows("M", "AK", 20, 23), rows("F", "AK", 21, 334)); } + /** + * Regression test for https://github.com/opensearch-project/sql/issues/5150 + * + *

    A renamed field that is not the dedup key must retain its value after dedup aggregation + * pushdown. Previously the top_hits response returned the original index field name ({@code + * name}), which the enumerator could not resolve to the renamed output name ({@code nm}), + * yielding null. + */ + @Test + public void testDedupWithRenamedField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | rename name as nm | dedup 1 category | fields category, nm", + TEST_INDEX_DUPLICATION_NULLABLE)); + // One representative row per category; nm must not be null + verifyDataRows(actual, rows("X", "A"), rows("Z", "B"), rows("Y", "A")); + } + + /** + * Regression test for https://github.com/opensearch-project/sql/issues/5197 + * + *

    When both a {@code rename} and an {@code eval} column-reference resolve to the same original + * index field, the old {@code Map<String,String>} mapping silently dropped one alias on + * collision. With {@code Map<String,List<String>>} both aliases must appear in the + * result with correct values. + */ + @Test + public void testDedupWithRenamedFieldMappingCollision() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval nm2 = name | rename name as nm | dedup 1 category" + + " | fields category, nm, nm2", + TEST_INDEX_DUPLICATION_NULLABLE)); + // Both nm (from rename) and nm2 (from eval col-ref) must carry the same non-null name value + verifyDataRows(actual, rows("X", "A", "A"), rows("Z", "B", "B"), rows("Y", "A", "A")); + } + /** Regression test for https://github.com/opensearch-project/sql/issues/3922 */ @Test public void testSortThenDedupKeepEmpty() throws IOException { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index f919fdc0e30..775b0278683 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -618,7 +619,31 @@ yield switch (functionName) { topHitsAggregationBuilder.sort( SortBuilders.fieldSort(key.field()).order(order).missing(missing)); } - yield Pair.of(topHitsAggregationBuilder, new TopHitsParser(aggName, false, false)); + // Build a mapping from original index field name to output (renamed) field names. + // top_hits returns _source / fields keyed by the original index name, but the Calcite + // row-type uses the renamed output name. A single original field may map to multiple + // output names when both a rename and an eval column-ref resolve to the same source + // field (issue #5197), so the value is a List rather than a single String. + Map> fieldNameMapping = new HashMap<>(); + for (Pair arg : args) { + if (arg.getKey() instanceof RexInputRef) { + NamedFieldExpression namedField = helper.inferNamedField(arg.getKey()); + if (namedField == null) { + continue; + } + String originalName = namedField.getRootName(); + String outputName = arg.getValue(); + if (!originalName.equals(outputName)) { + fieldNameMapping + .computeIfAbsent(originalName, k -> new ArrayList<>()) + .add(outputName); + } + } + } + yield Pair.of( + topHitsAggregationBuilder, + new TopHitsParser( + aggName, false, false, fieldNameMapping.isEmpty() ? null : fieldNameMapping)); } default -> throw new AggregateAnalyzer.AggregateAnalyzerException( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java index f9c3d5bb5d2..9fa3589f1fb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import org.opensearch.common.document.DocumentField; @@ -27,10 +28,29 @@ public class TopHitsParser implements MetricParser { private final boolean returnSingleValue; private final boolean returnMergeValue; + /** + * Maps each original OpenSearch field name to one or more output (renamed) field names. Used by + * the dedup aggregation pushdown path when {@code rename} creates aliases that differ from the + * index field name: top_hits returns {@code _source} / {@code fields} entries keyed by the + * original name, while the Calcite row-type expects the renamed name. A single original field may + * map to multiple output names when both {@code rename} and an {@code eval} column reference + * resolve to the same source field (issue #5197). + */ + @Nullable private final Map> fieldNameMapping; + public TopHitsParser(String name, boolean returnSingleValue, boolean returnMergeValue) { + this(name, returnSingleValue, returnMergeValue, null); + } + + public TopHitsParser( + String name, + boolean returnSingleValue, + boolean returnMergeValue, + @Nullable Map> fieldNameMapping) { this.name = name; this.returnSingleValue = returnSingleValue; this.returnMergeValue = returnMergeValue; + this.fieldNameMapping = fieldNameMapping; } @Override @@ -129,12 +149,43 @@ public List> parse(Aggregation agg) { ? new LinkedHashMap<>() : new LinkedHashMap<>(hit.getSourceAsMap()); hit.getFields().values().forEach(f -> map.put(f.getName(), f.getValue())); + applyFieldNameMapping(map); return map; }) .toList(); } } + /** + * Applies {@link #fieldNameMapping} to a parsed hit map in-place. + * + *

    For each {@code (originalName → [outputName1, outputName2, ...])} entry: the value stored + * under {@code originalName} is copied to every output name, and {@code originalName} is removed + * unless it is itself one of the expected output names. This handles both the single-rename case + * (issue #5150) and the many-to-one collision case where two aliases resolve to the same source + * field (issue #5197). + */ + private void applyFieldNameMapping(Map map) { + if (fieldNameMapping == null || fieldNameMapping.isEmpty()) { + return; + } + for (Map.Entry> entry : fieldNameMapping.entrySet()) { + String originalName = entry.getKey(); + Object value = map.get(originalName); + if (value == null && !map.containsKey(originalName)) { + continue; + } + List outputNames = entry.getValue(); + for (String outputName : outputNames) { + map.put(outputName, value); + } + // Remove the original key only when it is not itself one of the expected output names. + if (!outputNames.contains(originalName)) { + map.remove(originalName); + } + } + } + private boolean isEmptyHits(SearchHit[] hits) { return isFieldsEmpty(hits) && isSourceEmpty(hits); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 7ba64eaa475..1147fccf063 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -570,6 +570,96 @@ void two_bucket_percentiles_should_pass() { ImmutableMap.of("percentiles", List.of(21.0, 27.0, 30.0, 35.0, 55.0, 58.0, 60.0)))); } + /** + * Dedup pushdown (LITERAL_AGG) with a renamed field: {@code rename value as val | dedup + * category}. The top_hits response returns {@code _source: {category, value}} but the output + * schema expects {@code val}. The {@code fieldNameMapping} must translate {@code value -> val}. + * + *

    Regression test for https://github.com/opensearch-project/sql/issues/5150 + */ + @Test + void top_hits_field_name_mapping_single_rename_should_pass() { + String response = + "{\n" + + " \"composite#composite_buckets\": {\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": { \"category\": \"A\" },\n" + + " \"doc_count\": 2,\n" + + " \"top_hits#dedup\": {\n" + + " \"hits\": {\n" + + " \"total\": { \"value\": 2, \"relation\": \"eq\" },\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"idx\",\n" + + " \"_id\": \"1\",\n" + + " \"fields\": { \"value\": [10.5] }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + + Map> mapping = Map.of("value", List.of("val")); + OpenSearchAggregationResponseParser parser = + new CompositeAggregationParser(new TopHitsParser("dedup", false, false, mapping)); + assertThat( + parse(parser, response), + contains(ImmutableMap.of("category", "A"), ImmutableMap.of("val", 10.5))); + } + + /** + * Dedup pushdown (LITERAL_AGG) where two output names ({@code pay} from rename, {@code pay2} from + * eval column-ref) both resolve to the same original field {@code salary}. The old {@code + * Map} approach silently dropped one mapping on collision; with {@code + * Map>} both aliases must appear in the result. + * + *

    Regression test for https://github.com/opensearch-project/sql/issues/5197 + */ + @Test + void top_hits_field_name_mapping_collision_should_duplicate_value() { + String response = + "{\n" + + " \"composite#composite_buckets\": {\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": { \"dept_id\": \"eng\" },\n" + + " \"doc_count\": 3,\n" + + " \"top_hits#dedup\": {\n" + + " \"hits\": {\n" + + " \"total\": { \"value\": 3, \"relation\": \"eq\" },\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"idx\",\n" + + " \"_id\": \"1\",\n" + + " \"fields\": { \"salary\": [50000] }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + + // salary -> [pay, pay2]: both rename and eval-column-ref resolve to the same source field + Map> mapping = Map.of("salary", List.of("pay", "pay2")); + OpenSearchAggregationResponseParser parser = + new CompositeAggregationParser(new TopHitsParser("dedup", false, false, mapping)); + + List> result = parse(parser, response); + // Bucket key row + assertThat(result.get(0), org.hamcrest.Matchers.hasEntry("dept_id", "eng")); + // Hit row must contain both aliases with the same value; original key must be absent + Map hitRow = result.get(1); + assertEquals(50000, hitRow.get("pay")); + assertEquals(50000, hitRow.get("pay2")); + assertNull(hitRow.get("salary"), "original field name must be removed after mapping"); + } + public List> parse(OpenSearchAggregationResponseParser parser, String json) { return parser.parse(fromJson(json)); } From 6a07689a70fcb9016f3328fdb811adabab9c4ff3 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:30:47 -0700 Subject: [PATCH 32/41] Bump Apache Calcite 1.41.0 -> 1.42.0 (CVE-2026-46718) (#5619) * Bump Apache Calcite 1.41.0 -> 1.42.0 (CVE-2026-46718) Calcite 1.5.0 through 1.41.x are affected by CVE-2026-46718 (GHSA-c2rv-hwqm-wjpg, CWE-470 unsafe reflection), fixed in 1.42.0. Bump calcite-core, calcite-linq4j, calcite-babel, and calcite-testkit to 1.42.0. This transitively bumps avatica-core 1.27.0 -> 1.28.0. In 1.42.0, RelDataTypeSystemImpl.getMaxNumericPrecision() and getMaxNumericScale() became final; they now delegate to getMaxPrecision(DECIMAL)/getMaxScale(DECIMAL). OpenSearchTypeSystem overrode the former pair to keep Spark-aligned DECIMAL precision/scale of 38, so move those values into getMaxPrecision/getMaxScale for the DECIMAL case to preserve identical behavior. Update PPL Calcite unit-test golden strings to match 1.42.0's cosmetic plan and Spark-SQL rendering changes (explicit literal type suffixes, self-reference alias disambiguation, MAP/ARRAY type spelling, removal of redundant parentheses, explicit no-op casts). No query semantics change. Signed-off-by: Kai Huang * Align json-path and joou-java-6 with Calcite 1.42.0 transitive deps Calcite 1.42.0 bumps two of its runtime transitive dependencies: com.jayway.jsonpath:json-path 2.9.0 -> 2.10.0 and org.jooq:joou-java-6 0.9.4 -> 0.9.5. Modules built under the OpenSearch Gradle plugin's strict failOnVersionConflict (plugin, doc, integ, security-it, bwc) fail to resolve because core pinned json-path to 2.9.0 and the old joou 0.9.4 remained in the tree. Bump the explicit json-path pin in core to 2.10.0 and force both modules to the Calcite-1.42.0 versions in the root configurations.all block, alongside the existing transitive-conflict forces. Verified with the CI unit command (build -x integTest -x yamlRestTest -x doctest): BUILD SUCCESSFUL, no conflicts. Signed-off-by: Kai Huang * Fix FROM_UNIXTIME method resolution for boxed numeric operands Calcite 1.42.0 passes a nullable numeric operand to FROM_UNIXTIME as a boxed java.lang.Double, so method resolution could not match the primitive fromUnixTime(double) overload and codegen failed with NoSuchMethodException. This surfaced as a 500 SQLException on bin span queries over timestamp fields (e.g. bin @timestamp span=1h). Box the numeric operand and accept it as Number, mirroring SecToTimeFunction, so resolution succeeds for both primitive and boxed inputs. Signed-off-by: Kai Huang * Fix tostring() method resolution for boxed numeric operands Same Calcite 1.42.0 boxing change as FROM_UNIXTIME: a nullable numeric operand (e.g. a BIGINT field) now reaches tostring() boxed as java.lang.Long, which matched none of the primitive double/int overloads and failed codegen with NoSuchMethodException (500 SQLException on tostring(, ), e.g. tostring(balance, 'hex')). Box the numeric operand and resolve to a single Number overload, preserving BigDecimal precision by passing DECIMAL values through unchanged. String operands keep their existing path. Signed-off-by: Kai Huang * Update Calcite 1.42.0 explain-plan golden files Calcite 1.42.0 changes cosmetic plan rendering and some cost-based physical-plan choices. Regenerate the CalciteExplainIT and CalcitePPLClickBenchIT expected outputs (pushdown and no-pushdown variants) to match. Changes are limited to expected-output resources: - String literals now render with a type annotation ('x' -> 'x':VARCHAR). - Null-check operand references shift with equivalent projection indices (IS NOT NULL($t8) -> IS NOT NULL($t11)). - Anonymous projected columns render with their real alias. - streamstats reset: the planner now prefers HashJoin/TopK over MergeJoin + Sort + Limit. The pushdown request bodies are unchanged and CalciteStreamstatsCommandIT result assertions still pass, so query semantics are preserved. Signed-off-by: Kai Huang * Centralize Calcite dependency version Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- api/build.gradle | 4 +- build.gradle | 4 ++ core/build.gradle | 6 +- .../sql/executor/OpenSearchTypeSystem.java | 13 ++--- .../function/udf/ToStringFunction.java | 24 +++++--- .../udf/datetime/FromUnixTimeFunction.java | 16 ++--- .../calcite/chart_multiple_group_keys.yaml | 4 +- .../calcite/chart_null_str.yaml | 4 +- .../chart_timestamp_span_and_category.yaml | 2 +- .../calcite/chart_use_other.yaml | 4 +- .../calcite/clickbench/q21.yaml | 4 +- .../calcite/clickbench/q22.yaml | 4 +- .../calcite/clickbench/q23.yaml | 4 +- .../calcite/clickbench/q24.yaml | 6 +- .../explain_exists_correlated_subquery.yaml | 10 ++-- .../explain_in_correlated_subquery.yaml | 13 +++-- .../explain_keyword_ilike_function.yaml | 4 +- .../explain_keyword_like_function.yaml | 4 +- ...eyword_like_function_case_insensitive.yaml | 4 +- ..._scalar_correlated_subquery_in_select.yaml | 6 +- .../calcite/explain_streamstats_global.yaml | 2 +- .../calcite/explain_streamstats_reset.yaml | 42 +++++++------- .../calcite/explain_text_ilike_function.yaml | 4 +- .../calcite/explain_text_like_function.yaml | 4 +- ...n_text_like_function_case_insensitive.yaml | 4 +- .../calcite/explain_timechart.yaml | 2 +- .../calcite/explain_timechart_count.yaml | 4 +- .../chart_multiple_group_keys.yaml | 2 +- .../calcite_no_pushdown/chart_null_str.yaml | 2 +- .../chart_timestamp_span_and_category.yaml | 2 +- .../calcite_no_pushdown/chart_use_other.yaml | 2 +- .../explain_eventstats_earliest_latest.json | 2 +- ...ventstats_earliest_latest_custom_time.json | 2 +- ...n_eventstats_earliest_latest_no_group.json | 2 +- .../explain_exists_correlated_subquery.yaml | 13 ++--- .../explain_in_correlated_subquery.yaml | 20 +++---- .../explain_keyword_ilike_function.yaml | 6 +- .../explain_keyword_like_function.yaml | 6 +- ...eyword_like_function_case_insensitive.yaml | 6 +- .../calcite_no_pushdown/explain_output.yaml | 2 +- ..._scalar_correlated_subquery_in_select.yaml | 6 +- .../explain_streamstats_dc.yaml | 2 +- ..._streamstats_earliest_latest_no_group.yaml | 4 +- .../explain_streamstats_global.yaml | 2 +- .../explain_streamstats_reset.yaml | 31 +++++----- .../explain_text_ilike_function.yaml | 6 +- .../explain_text_like_function.yaml | 6 +- ...n_text_like_function_case_insensitive.yaml | 6 +- .../explain_timechart.yaml | 2 +- .../explain_timechart_count.yaml | 2 +- ppl/build.gradle | 2 +- .../calcite/CalcitePPLCaseFunctionTest.java | 2 +- .../calcite/CalcitePPLExistsSubqueryTest.java | 10 ++-- .../ppl/calcite/CalcitePPLFillnullTest.java | 12 ++-- .../ppl/calcite/CalcitePPLInSubqueryTest.java | 2 +- .../ppl/calcite/CalcitePPLPatternsTest.java | 20 +++---- .../calcite/CalcitePPLScalarSubqueryTest.java | 58 +++++++++---------- .../calcite/CalcitePPLStringFunctionTest.java | 4 +- 58 files changed, 230 insertions(+), 216 deletions(-) diff --git a/api/build.gradle b/api/build.gradle index 638aa8700ce..0ac7ed81880 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -14,13 +14,13 @@ plugins { dependencies { api project(':ppl') api project(':sql') - api group: 'org.apache.calcite', name: 'calcite-babel', version: '1.41.0' + api group: 'org.apache.calcite', name: 'calcite-babel', version: "${calcite_version}" testImplementation testFixtures(project(':api')) testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" - testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}" testFixturesApi group: 'junit', name: 'junit', version: '4.13.2' testFixturesApi group: 'org.hamcrest', name: 'hamcrest', version: "${hamcrest_version}" diff --git a/build.gradle b/build.gradle index 7b76532aad0..9047a9c3feb 100644 --- a/build.gradle +++ b/build.gradle @@ -52,6 +52,7 @@ buildscript { // TODO: Migrate following to Gradle version catalog || Read from OpenSearch BOM in the future. // See: https://github.com/opensearch-project/sql/issues/3257 aws_java_sdk_version = "1.12.651" + calcite_version = "1.42.0" guava_version = "33.3.0-jre" resilience4j_version = "1.5.0" hamcrest_version = "2.1" @@ -162,6 +163,9 @@ allprojects { resolutionStrategy.force 'org.apache.commons:commons-text:1.11.0' resolutionStrategy.force 'commons-io:commons-io:2.15.0' resolutionStrategy.force 'org.yaml:snakeyaml:2.2' + // Align Calcite 1.42.0's bumped transitive deps under strict conflict resolution + resolutionStrategy.force 'com.jayway.jsonpath:json-path:2.10.0' + resolutionStrategy.force 'org.jooq:joou-java-6:0.9.5' resolutionStrategy.dependencySubstitution { substitute module('commons-lang:commons-lang') using module('org.apache.commons:commons-lang3:3.18.0') because 'CVE-2025-48924: commons-lang 2.x vulnerable to StackOverflowError' } diff --git a/core/build.gradle b/core/build.gradle index 4d0d98edb29..f4ccaa0d6e1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -60,10 +60,10 @@ dependencies { api group: 'com.google.code.gson', name: 'gson', version: '2.8.9' api group: 'com.tdunning', name: 't-digest', version: '3.3' api "net.minidev:json-smart:${versions.json_smart}" - api('org.apache.calcite:calcite-core:1.41.0') { + api("org.apache.calcite:calcite-core:${calcite_version}") { exclude group: 'net.minidev', module: 'json-smart' } - api 'org.apache.calcite:calcite-linq4j:1.41.0' + api "org.apache.calcite:calcite-linq4j:${calcite_version}" api project(':common') compileOnly 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' // Needed because analytics-api's QueryPlanExecutor signature uses @@ -72,7 +72,7 @@ dependencies { testImplementation 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" - implementation "com.jayway.jsonpath:json-path:2.9.0" + implementation "com.jayway.jsonpath:json-path:2.10.0" annotationProcessor('org.immutables:value:2.8.8') compileOnly 'org.immutables:value-annotations:2.8.8' diff --git a/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java b/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java index 941f42de46c..7eb1e3cacd4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java +++ b/core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java @@ -27,14 +27,10 @@ public class OpenSearchTypeSystem extends RelDataTypeSystemImpl { private OpenSearchTypeSystem() {} - @Override - public int getMaxNumericPrecision() { - return MAX_PRECISION; - } - @Override public int getMaxPrecision(SqlTypeName typeName) { return switch (typeName) { + case DECIMAL -> MAX_PRECISION; case TIME, TIME_WITH_LOCAL_TIME_ZONE, TIME_TZ, @@ -47,8 +43,11 @@ public int getMaxPrecision(SqlTypeName typeName) { } @Override - public int getMaxNumericScale() { - return MAX_SCALE; + public int getMaxScale(SqlTypeName typeName) { + return switch (typeName) { + case DECIMAL -> MAX_SCALE; + default -> super.getMaxScale(typeName); + }; } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index e6e8dd01df0..6be0576bada 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -20,6 +20,8 @@ import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.*; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.utils.MathUtils; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -67,6 +69,11 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { Expression fieldValue = translatedOperands.get(0); Expression format = translatedOperands.get(1); + // Box numeric operands and pass them as Number so method resolution succeeds whether the + // upstream expression yields a primitive or a boxed value (e.g. a nullable long/double). + if (SqlTypeUtil.isNumeric(call.getOperands().get(0).getType())) { + fieldValue = Expressions.convert_(Expressions.box(fieldValue), Number.class); + } return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } } @@ -97,13 +104,16 @@ public static String toString(BigDecimal num, String format) { } @Strict - public static String toString(double num, String format) { - return toString(BigDecimal.valueOf(num), format); - } - - @Strict - public static String toString(int num, String format) { - return toString(BigDecimal.valueOf(num), format); + public static String toString(Number num, String format) { + BigDecimal bd; + if (num instanceof BigDecimal decimal) { + bd = decimal; + } else if (MathUtils.isIntegral(num)) { + bd = BigDecimal.valueOf(num.longValue()); + } else { + bd = BigDecimal.valueOf(num.doubleValue()); + } + return toString(bd, format); } @Strict diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java index 00fa9f690da..36fb718dbd3 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/FromUnixTimeFunction.java @@ -10,7 +10,7 @@ import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTime; import static org.opensearch.sql.expression.datetime.DateTimeFunctions.exprFromUnixTimeFormat; -import java.math.BigDecimal; +import java.util.ArrayList; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -61,18 +61,18 @@ public static class FromUnixTimeImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", translatedOperands); + // Box the numeric operand and pass it as Number so method resolution succeeds whether the + // upstream expression yields a primitive or a boxed value (e.g. a nullable double). + List operands = new ArrayList<>(translatedOperands); + operands.set(0, Expressions.convert_(Expressions.box(operands.getFirst()), Number.class)); + return Expressions.call(FromUnixTimeImplementor.class, "fromUnixTime", operands); } - public static String fromUnixTime(double unixTime) { + public static String fromUnixTime(Number unixTime) { return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite(); } - public static String fromUnixTime(BigDecimal unixTime) { - return (String) exprFromUnixTime(new ExprDoubleValue(unixTime)).valueForCalcite(); - } - - public static String fromUnixTime(double unixTime, String format) { + public static String fromUnixTime(Number unixTime, String format) { return (String) exprFromUnixTimeFormat(new ExprDoubleValue(unixTime), new ExprStringValue(format)) .valueForCalcite(); diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml index 2f539057b66..63c7ea798df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_multiple_group_keys.yaml @@ -28,8 +28,8 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 656118ca892..726eeedc429 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -30,10 +30,10 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml index 9267e6faab1..895d30e06e0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_timestamp_span_and_category.yaml @@ -25,7 +25,7 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[timestamp0, category, max(value)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},max(value)=MAX($1)), PROJECT->[category, max(value)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"timestamp","boost":1.0}},{"exists":{"field":"value","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"category","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"timestamp0":{"date_histogram":{"field":"timestamp","missing_bucket":false,"order":"asc","calendar_interval":"1w"}}}]},"aggregations":{"max(value)":{"max":{"field":"value"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml index 8b9f72596b1..aec30183545 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_use_other.yaml @@ -24,7 +24,7 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($2)), FILTER->IS NOT NULL($1), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},max(severityNumber)=MAX($2)), PROJECT->[severityText, max(severityNumber)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"flags","boost":1.0}},{"exists":{"field":"severityNumber","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"severityText","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"flags":{"terms":{"field":"flags","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"severityText":{"terms":{"field":"severityText","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"max(severityNumber)":{"max":{"field":"severityNumber"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml index ea3d4d6863e..fdaf8ad1db8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q21.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalAggregate(group=[{}], count()=[COUNT()]) - LogicalFilter(condition=[LIKE($26, '%google%', '\')]) + LogicalFilter(condition=[LIKE($26, '%google%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->LIKE($0, '%google%', '\'), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->LIKE($0, '%google%':VARCHAR, '\'), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml index edd3dabd8d8..a1ae1ea9ab6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q22.yaml @@ -6,7 +6,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()]) LogicalProject(SearchPhrase=[$63]) LogicalFilter(condition=[IS NOT NULL($63)]) - LogicalFilter(condition=[AND(LIKE($26, '%google%', '\'), <>($63, ''))]) + LogicalFilter(condition=[AND(LIKE($26, '%google%':VARCHAR, '\'), <>($63, ''))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($0, '%google%', '\'), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT()), PROJECT->[c, SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($0, '%google%':VARCHAR, '\'), <>($1, '')), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT()), PROJECT->[c, SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"_count":"desc"},{"_key":"asc"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml index 6f6b5056a9f..ada01f0ebef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q23.yaml @@ -6,7 +6,7 @@ calcite: LogicalAggregate(group=[{0}], c=[COUNT()], dc(UserID)=[COUNT(DISTINCT $1)]) LogicalProject(SearchPhrase=[$63], UserID=[$84]) LogicalFilter(condition=[IS NOT NULL($63)]) - LogicalFilter(condition=[AND(LIKE($97, '%Google%', '\'), <>($63, ''), NOT(LIKE($26, '%.google.%', '\')))]) + LogicalFilter(condition=[AND(LIKE($97, '%Google%':VARCHAR, '\'), <>($63, ''), NOT(LIKE($26, '%.google.%':VARCHAR, '\')))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($3, '%Google%', '\'), <>($1, ''), NOT(LIKE($0, '%.google.%', '\'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),dc(UserID)=COUNT(DISTINCT $2)), PROJECT->[c, dc(UserID), SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"Title":{"wildcard":"*Google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"wildcard":{"URL":{"wildcard":"*.google.*","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(LIKE($3, '%Google%':VARCHAR, '\'), <>($1, ''), NOT(LIKE($0, '%.google.%':VARCHAR, '\'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),dc(UserID)=COUNT(DISTINCT $2)), PROJECT->[c, dc(UserID), SearchPhrase], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"wildcard":{"Title":{"wildcard":"*Google*","boost":1.0}}},{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"wildcard":{"URL":{"wildcard":"*.google.*","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchPhrase":{"terms":{"field":"SearchPhrase","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml index f24aabcab1a..0c83653d72d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q24.yaml @@ -3,12 +3,12 @@ calcite: LogicalSystemLimit(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104]) LogicalSort(sort0=[$17], dir0=[ASC-nulls-first], fetch=[10]) - LogicalFilter(condition=[LIKE($26, '%google%', '\')]) + LogicalFilter(condition=[LIKE($26, '%google%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage, CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID, HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm, AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash, URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel, SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload, IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits, CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink, IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming, FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength, IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent, UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency, Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming, ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID, WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID, OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID, FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth], FILTER->LIKE($26, '%google%', '\'), SORT->[{ + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, URLRegionID, HasGCLID, Income, Interests, Robotness, BrowserLanguage, CounterClass, BrowserCountry, OriginalURL, ClientTimeZone, RefererHash, TraficSourceID, HitColor, RefererRegionID, URLCategoryID, LocalEventTime, EventTime, UTMTerm, AdvEngineID, UserAgentMinor, UserAgentMajor, RemoteIP, Sex, JavaEnable, URLHash, URL, ParamOrderID, OpenstatSourceID, HTTPError, SilverlightVersion3, MobilePhoneModel, SilverlightVersion4, SilverlightVersion1, SilverlightVersion2, IsDownload, IsParameter, CLID, FlashMajor, FlashMinor, UTMMedium, WatchID, DontCountHits, CookieEnable, HID, SocialAction, WindowName, ConnectTiming, PageCharset, IsLink, IsArtifical, JavascriptEnable, ClientEventTime, DNSTiming, CodeVersion, ResponseEndTiming, FUniqID, WindowClientHeight, OpenstatServiceName, UTMContent, HistoryLength, IsOldCounter, MobilePhone, SearchPhrase, FlashMinor2, SearchEngineID, IsEvent, UTMSource, RegionID, OpenstatAdID, UTMCampaign, GoodEvent, IsRefresh, ParamCurrency, Params, ResolutionHeight, ClientIP, FromTag, ParamCurrencyID, ResponseStartTiming, ResolutionWidth, SendTiming, RefererCategoryID, OpenstatCampaignID, UserID, WithHash, UserAgent, ParamPrice, ResolutionDepth, IsMobile, Age, SocialSourceNetworkID, OpenerName, OS, IsNotBounce, Referer, NetMinor, Title, NetMajor, IPNetworkID, FetchTiming, SocialNetwork, SocialSourcePage, CounterID, WindowClientWidth], FILTER->LIKE($26, '%google%':VARCHAR, '\'), SORT->[{ "EventTime" : { "order" : "asc", "missing" : "_first" } - }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file + }], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","query":{"wildcard":{"URL":{"wildcard":"*google*","boost":1.0}}},"_source":{"includes":["EventDate","URLRegionID","HasGCLID","Income","Interests","Robotness","BrowserLanguage","CounterClass","BrowserCountry","OriginalURL","ClientTimeZone","RefererHash","TraficSourceID","HitColor","RefererRegionID","URLCategoryID","LocalEventTime","EventTime","UTMTerm","AdvEngineID","UserAgentMinor","UserAgentMajor","RemoteIP","Sex","JavaEnable","URLHash","URL","ParamOrderID","OpenstatSourceID","HTTPError","SilverlightVersion3","MobilePhoneModel","SilverlightVersion4","SilverlightVersion1","SilverlightVersion2","IsDownload","IsParameter","CLID","FlashMajor","FlashMinor","UTMMedium","WatchID","DontCountHits","CookieEnable","HID","SocialAction","WindowName","ConnectTiming","PageCharset","IsLink","IsArtifical","JavascriptEnable","ClientEventTime","DNSTiming","CodeVersion","ResponseEndTiming","FUniqID","WindowClientHeight","OpenstatServiceName","UTMContent","HistoryLength","IsOldCounter","MobilePhone","SearchPhrase","FlashMinor2","SearchEngineID","IsEvent","UTMSource","RegionID","OpenstatAdID","UTMCampaign","GoodEvent","IsRefresh","ParamCurrency","Params","ResolutionHeight","ClientIP","FromTag","ParamCurrencyID","ResponseStartTiming","ResolutionWidth","SendTiming","RefererCategoryID","OpenstatCampaignID","UserID","WithHash","UserAgent","ParamPrice","ResolutionDepth","IsMobile","Age","SocialSourceNetworkID","OpenerName","OS","IsNotBounce","Referer","NetMinor","Title","NetMajor","IPNetworkID","FetchTiming","SocialNetwork","SocialSourcePage","CounterID","WindowClientWidth"]},"sort":[{"EventTime":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml index 3830e23a87b..1026165ed7e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_exists_correlated_subquery.yaml @@ -12,10 +12,10 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableHashJoin(condition=[=($1, $3)], joinType=[semi]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[true], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t0)], i=[$t1], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000, PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10000], expr#3=[<=($t1, $t2)], expr#4=[IS NOT NULL($t0)], expr#5=[AND($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml index 23bd0d6df69..421d36454a2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_in_correlated_subquery.yaml @@ -12,11 +12,12 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) CalciteEnumerableTopK(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id, salary]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id","salary"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['Tom':VARCHAR], expr#2=[CAST($t1):VARCHAR], name=[$t2], uid=[$t0]) EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["name","uid"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10000], expr#3=[<=($t1, $t2)], expr#4=[IS NOT NULL($t0)], expr#5=[AND($t3, $t4)], proj#0..1=[{exprs}], $condition=[$t5]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[PROJECT->[name, uid], FILTER->=($0, 'Tom'), PROJECT->[uid]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"term":{"name":{"value":"Tom","boost":1.0}}},"_source":{"includes":["uid"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml index 0651ff30dbd..97fd56f0c94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_ilike_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml index 98b5bbb2f34..fb337d54483 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[LIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->LIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->LIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml index 0651ff30dbd..97fd56f0c94 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function_case_insensitive.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"wildcard":{"firstname.keyword":{"wildcard":"*mbe*","case_insensitive":true,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml index 7fde9c0d05e..a704f1588d9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_scalar_correlated_subquery_in_select.yaml @@ -12,9 +12,9 @@ calcite: physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) EnumerableLimit(fetch=[10000]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[PROJECT->[name, id], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["name","id"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"id":{"terms":{"field":"id","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},count(name)=COUNT($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"uid","boost":1.0}},{"exists":{"field":"name","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"uid":{"terms":{"field":"uid","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(name)":{"value_count":{"field":"name"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 124539b9d4c..0478b24369c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -13,7 +13,7 @@ calcite: EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableNestedLoopJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..2=[{inputs}], __r_seq__=[$t2], __r_gender__=[$t0], __r_age__=[$t1]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 72f8f4d6ca7..324960f28dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -15,26 +15,24 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) - EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) - EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) - EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + CalciteEnumerableTopK(sort0=[$11], dir0=[ASC], fetch=[10000]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $14), =($11, $15), =($12, $16), =($13, $17))], joinType=[left]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml index 0b13885f73f..3c8bd027e33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_ilike_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml index 9ca9c104e89..ce83c3e9cc6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[LIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->LIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCBnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMSUtFIiwKICAgICJraW5kIjogIkxJS0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->LIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCBnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMSUtFIiwKICAgICJraW5kIjogIkxJS0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml index 0b13885f73f..3c8bd027e33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function_case_insensitive.yaml @@ -2,7 +2,7 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%':VARCHAR, '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCB3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index 4f14b591721..0818c18eabb 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], cpu_usage=[$t2], @timestamp0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index 6e68f5335d1..f26cb9e5822 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -29,7 +29,7 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp0=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp","host"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index 95e83cdcd19..fe925e0a80a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index 274186e377e..beb3275a6c6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml index 76b833ce3f1..f47e62e81d0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_timestamp_span_and_category.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..9=[{inputs}], expr#10=[1], expr#11=['w'], expr#12=[SPAN($t3, $t10, $t11)], expr#13=[IS NOT NULL($t3)], expr#14=[IS NOT NULL($t2)], expr#15=[AND($t13, $t14)], category=[$t1], value=[$t2], timestamp0=[$t12], $condition=[$t15]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], category=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{0, 2}], max(value)=[MAX($1)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml index 027d0e30124..2d716443c5a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_use_other.yaml @@ -29,7 +29,7 @@ calcite: EnumerableCalc(expr#0..171=[{inputs}], expr#172=[IS NOT NULL($t23)], expr#173=[IS NOT NULL($t163)], expr#174=[AND($t172, $t173)], proj#0..171=[{exprs}], $condition=[$t174]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_otel_logs]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], severityText=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableAggregate(group=[{7, 23}], max(severityNumber)=[MAX($163)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json index e2ea6b3ddb5..f17a86f9a0c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER (PARTITION BY $1)], latest_message=[ARG_MAX($3, $2) OVER (PARTITION BY $1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(partition {1} aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(partition {1} aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json index 27849cce681..d5626e8129c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_custom_time.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $0) OVER (PARTITION BY $4)], latest_message=[ARG_MAX($3, $0) OVER (PARTITION BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(partition {4} aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(partition {4} aggs [ARG_MIN($3, $0), ARG_MAX($3, $0)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json index 034699c80e9..181d34a5474 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eventstats_earliest_latest_no_group.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(created_at=[$0], server=[$1], @timestamp=[$2], message=[$3], level=[$4], earliest_message=[ARG_MIN($3, $2) OVER ()], latest_message=[ARG_MAX($3, $2) OVER ()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12])\n EnumerableWindow(window#0=[window(aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12])\n EnumerableWindow(window#0=[window(aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml index 400bd549ee8..18146bd8e9c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_exists_correlated_subquery.yaml @@ -13,13 +13,12 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) + EnumerableHashJoin(condition=[=($1, $4)], joinType=[semi]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[true], expr#3=[$cor0], expr#4=[$t3.id], expr#5=[=($t4, $t1)], i=[$t2], $condition=[$t5]) - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10000], expr#12=[<=($t10, $t11)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t13)], proj#0..10=[{exprs}], $condition=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml index cb17a67d1cd..f64f0d94dbe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_in_correlated_subquery.yaml @@ -12,15 +12,15 @@ calcite: })], variablesSet=[[$cor0]]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) physical: | - EnumerableCalc(expr#0..3=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], id=[$t1], name=[$t0], salary=[$t2]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$2], dir0=[DESC-nulls-last]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[=($t0, $t3)], proj#0..3=[{exprs}], $condition=[$t4]) - EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}]) - EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) - EnumerableAggregate(group=[{0}]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[$cor0], expr#3=[$t2.id], expr#4=[=($t3, $t1)], proj#0..1=[{exprs}], $condition=[$t4]) - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..1=[{exprs}], $condition=[$t11]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + EnumerableHashJoin(condition=[AND(=($1, $4), =($0, $3))], joinType=[semi]) + EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2], salary=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['Tom':VARCHAR], expr#2=[CAST($t1):VARCHAR], name=[$t2], uid=[$t0]) + EnumerableAggregate(group=[{1}]) + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10000], expr#12=[<=($t10, $t11)], expr#13=[IS NOT NULL($t1)], expr#14=[AND($t12, $t13)], proj#0..10=[{exprs}], $condition=[$t14]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=['Tom':VARCHAR], expr#11=[=($t0, $t10)], proj#0..9=[{exprs}], $condition=[$t11]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml index f8b576cb814..13eb24eadfe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_ilike_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml index 2d164b50d29..68a1125b16b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[LIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[LIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[LIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml index f8b576cb814..13eb24eadfe 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_keyword_like_function_case_insensitive.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($1, '%mbe%', '\')]) + LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%'], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%mbe%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t1, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index 42e82eca514..f781995261c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml index 5e76c380ff2..87249bc2d06 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_scalar_correlated_subquery_in_select.yaml @@ -12,13 +12,13 @@ calcite: physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t3)], expr#5=[0:BIGINT], expr#6=[CASE($t4, $t5, $t3)], id=[$t1], name=[$t0], count_dept=[$t6]) EnumerableLimit(fetch=[10000]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($1, $2)], joinType=[left]) EnumerableCalc(expr#0..10=[{inputs}], name=[$t0], id=[$t2]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t2)], expr#4=[0], expr#5=[CASE($t3, $t2, $t4)], uid=[$t0], count(name)=[$t5]) - EnumerableNestedLoopJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) + EnumerableHashJoin(condition=[IS NOT DISTINCT FROM($0, $1)], joinType=[left]) EnumerableAggregate(group=[{2}]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_worker]]) EnumerableAggregate(group=[{1}], count(name)=[COUNT($0)]) EnumerableCalc(expr#0..9=[{inputs}], expr#10=[IS NOT NULL($t1)], expr#11=[IS NOT NULL($t0)], expr#12=[AND($t10, $t11)], proj#0..9=[{exprs}], $condition=[$t12]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_work_information]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml index 6ffa5ad304c..95539d91aa7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_dc.yaml @@ -5,6 +5,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..17=[{inputs}], proj#0..10=[{exprs}], $11=[$t17]) + EnumerableCalc(expr#0..17=[{inputs}], proj#0..10=[{exprs}], distinct_states=[$t17]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [DISTINCT_COUNT_APPROX($7)])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml index 79dcbca7555..5ba8abf2038 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_earliest_latest_no_group.yaml @@ -5,6 +5,6 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], $5=[$t11], $6=[$t12]) + EnumerableCalc(expr#0..12=[{inputs}], proj#0..4=[{exprs}], earliest_message=[$t11], latest_message=[$t12]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ARG_MIN($3, $2), ARG_MAX($3, $2)])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index c56cd5d1bce..0bf9a2c50ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -14,7 +14,7 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) - EnumerableNestedLoopJoin(condition=[AND(>=($18, -($17, 1)), <=($18, $17), IS NOT DISTINCT FROM($4, $19))], joinType=[left]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) EnumerableCalc(expr#0..17=[{inputs}], __r_seq__=[$t17], __r_gender__=[$t4], __r_age__=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 5664cc6aa87..3ec98ba9382 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -16,25 +16,24 @@ calcite: physical: | EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) - EnumerableMergeJoin(condition=[AND(=($11, $15), =($12, $16), =($13, $17), IS NOT DISTINCT FROM($4, $14))], joinType=[left]) - EnumerableSort(sort0=[$11], sort1=[$12], sort2=[$13], dir0=[ASC], dir1=[ASC], dir2=[ASC]) + EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $14), =($11, $15), =($12, $16), =($13, $17))], joinType=[left]) + EnumerableSort(sort0=[$11], dir0=[ASC]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[0], expr#18=[COALESCE($t16, $t17)], expr#19=[+($t15, $t18)], proj#0..11=[{exprs}], __seg_id__=[$t19], $f16=[$t14]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($12)])], window#1=[window(rows between UNBOUNDED PRECEDING and $15 PRECEDING aggs [$SUM0($13)])], constants=[[1]]) EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) + EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml index 41638cd1b16..a79db7b3383 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_ilike_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml index 6be02086bb0..1a450917474 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[LIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[LIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[LIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[LIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml index 41638cd1b16..a79db7b3383 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_text_like_function_case_insensitive.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) - LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) + LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%'], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['%Holmes%':VARCHAR], expr#18=['\'], expr#19=[ILIKE($t2, $t17, $t18)], proj#0..10=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml index e982ce038e2..da0b6d37d16 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart.yaml @@ -31,7 +31,7 @@ calcite: EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], host=[$t4], cpu_usage=[$t7], @timestamp0=[$t18], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($2)]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml index 2979778506a..6556ef9022f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_timechart_count.yaml @@ -30,7 +30,7 @@ calcite: EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], expr#19=[IS NOT NULL($t1)], host=[$t4], @timestamp0=[$t18], $condition=[$t19]) CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], $1=[$t2]) + EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{4}], __grand_total__=[COUNT()]) EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t1)], expr#17=[IS NOT NULL($t4)], expr#18=[AND($t16, $t17)], proj#0..15=[{exprs}], $condition=[$t18]) diff --git a/ppl/build.gradle b/ppl/build.gradle index caf5223103c..e883c891fd3 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -63,7 +63,7 @@ dependencies { testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" - testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: '1.41.0' + testImplementation group: 'org.apache.calcite', name: 'calcite-testkit', version: "${calcite_version}" testImplementation(testFixtures(project(":core"))) } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java index 2788f7a9cfe..14d3476802d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCaseFunctionTest.java @@ -102,7 +102,7 @@ public void testCaseWhenInSubquery() { + "FROM `scott`.`EMP`\n" + "WHERE `DEPTNO` IN (SELECT CASE WHEN `DEPTNO` IN (20, 21) THEN 20 WHEN `DEPTNO` IN" + " (30, 31) THEN 30 ELSE 100 END `new_deptno`\n" - + "FROM `scott`.`EMP`)"; + + "FROM `scott`.`EMP` `EMP0`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java index 84b509d16e9..2a436d30bb0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExistsSubqueryTest.java @@ -326,7 +326,7 @@ public void testNestedCorrelatedExistsSubquery() { + "FROM (SELECT *\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `SAL` = `SALGRADE`.`HISAL`)) `t0`\n" + "WHERE `EMP`.`SAL` = `HISAL`)\n" + "ORDER BY `EMPNO` DESC"; @@ -372,7 +372,7 @@ public void testNestedUncorrelatedExistsSubquery() { + "FROM (SELECT *\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `SAL` > 1000.0)) `t0`\n" + "WHERE `HISAL` > 1000.0)\n" + "ORDER BY `EMPNO` DESC"; @@ -435,12 +435,12 @@ public void testNestedMixedExistsSubquery() { + "FROM `scott`.`SALGRADE`\n" + "WHERE EXISTS (SELECT *\n" + "FROM (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE EXISTS (SELECT *\n" + "FROM (SELECT *\n" - + "FROM `scott`.`SALGRADE`\n" + + "FROM `scott`.`SALGRADE` `SALGRADE0`\n" + "WHERE EXISTS (SELECT *\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP1`\n" + "WHERE `SAL` = `SALGRADE0`.`HISAL`)) `t0`\n" + "WHERE `EMP0`.`SAL` > 1000.0)) `t2`\n" + "WHERE `SAL` = `SALGRADE`.`HISAL`)) `t4`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java index e664b4f21db..ee7a1d5c2f1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFillnullTest.java @@ -114,7 +114,8 @@ public void testFillnullAll() { String ppl = "source=EMP | fields EMPNO, MGR, SAL, COMM, DEPTNO | fillnull with 0"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], MGR=[COALESCE($3, 0)], SAL=[COALESCE($5, 0)]," + "LogicalProject(EMPNO=[CAST($0):INTEGER NOT NULL], MGR=[COALESCE($3, 0)]," + + " SAL=[COALESCE($5, 0)]," + " COMM=[COALESCE($6, 0)], DEPTNO=[COALESCE($7, 0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -136,7 +137,8 @@ public void testFillnullAll() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0) `SAL`, COALESCE(`COMM`, 0)" + "SELECT CAST(`EMPNO` AS INTEGER) `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0)" + + " `SAL`, COALESCE(`COMM`, 0)" + " `COMM`, COALESCE(`DEPTNO`, 0) `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -194,7 +196,8 @@ public void testFillnullValueSyntaxAllFields() { String ppl = "source=EMP | fields EMPNO, MGR, SAL, COMM, DEPTNO | fillnull value=0"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], MGR=[COALESCE($3, 0)], SAL=[COALESCE($5, 0)]," + "LogicalProject(EMPNO=[CAST($0):INTEGER NOT NULL], MGR=[COALESCE($3, 0)]," + + " SAL=[COALESCE($5, 0)]," + " COMM=[COALESCE($6, 0)], DEPTNO=[COALESCE($7, 0)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); @@ -216,7 +219,8 @@ public void testFillnullValueSyntaxAllFields() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0) `SAL`, COALESCE(`COMM`, 0)" + "SELECT CAST(`EMPNO` AS INTEGER) `EMPNO`, COALESCE(`MGR`, 0) `MGR`, COALESCE(`SAL`, 0)" + + " `SAL`, COALESCE(`COMM`, 0)" + " `COMM`, COALESCE(`DEPTNO`, 0) `DEPTNO`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java index 5c26d70335b..e30fa9fd045 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java @@ -77,7 +77,7 @@ public void testSelfInSubquery() { + "SELECT `MGR`\n" + "FROM `scott`.`EMP`\n" + "WHERE `MGR` IN (SELECT `MGR`\n" - + "FROM `scott`.`EMP`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + "WHERE `DEPTNO` = 10)"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java index c272453b829..161a923738c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java @@ -68,7 +68,7 @@ public void testPatternsLabelMode_ShowNumberedToken_ForSimplePatternMethod() { + " '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['pattern'] AS" + " STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR" + " `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END," - + " `ENAME`)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " `ENAME`)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -95,7 +95,7 @@ public void testPatternsLabelModeWithCustomPattern_ShowNumberedToken_ForSimplePa + " '' ELSE REGEXP_REPLACE(`ENAME`, '[A-H]', '<*>') END, `ENAME`)['pattern'] AS STRING)" + " `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` =" + " '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[A-H]', '<*>') END, `ENAME`)['tokens'] AS" - + " MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -142,7 +142,7 @@ public void testPatternsLabelModeWithPartitionBy_ShowNumberedToken_SimplePattern + " = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END," + " `ENAME`)['pattern'] AS STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(CASE" + " WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`," - + " '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" + + " '[a-zA-Z0-9]+', '<*>') END, `ENAME`)['tokens'] AS MAP< STRING, ARRAY< STRING > >)" + " `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -187,7 +187,7 @@ public void testPatternsLabelMode_ShowNumberedToken_ForBrainMethod() { + " OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), TRUE)['pattern']" + " AS STRING) `patterns_field`, TRY_CAST(PATTERN_PARSER(`ENAME`, `pattern`(`ENAME`," + " 10, 100000, TRUE) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)," - + " TRUE)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " TRUE)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -234,7 +234,7 @@ public void testPatternsLabelModeWithPartitionBy_ShowNumberedToken_ForBrainMetho + " UNBOUNDED FOLLOWING), TRUE)['pattern'] AS STRING) `patterns_field`," + " TRY_CAST(PATTERN_PARSER(`ENAME`, `pattern`(`ENAME`, 10, 100000, TRUE) OVER" + " (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)," - + " TRUE)['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " TRUE)['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -287,7 +287,7 @@ public void testPatternsAggregationMode_ShowNumberedToken_ForSimplePatternMethod + " '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END) `pattern_count`," + " TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" + " REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `TAKE`(`ENAME`, 10))['tokens']" - + " AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`, `TAKE`(`ENAME`, 10) `sample_logs`\n" + + " AS MAP< STRING, ARRAY< STRING > >) `tokens`, `TAKE`(`ENAME`, 10) `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" + " REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END"; @@ -318,7 +318,7 @@ public void testPatternsAggregationModeWithGroupBy_ShowNumberedToken_ForSimplePa + " `ENAME` = '' THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END)" + " `pattern_count`, TRY_CAST(PATTERN_PARSER(CASE WHEN `ENAME` IS NULL OR `ENAME` = ''" + " THEN '' ELSE REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>') END, `TAKE`(`ENAME`," - + " 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`, `TAKE`(`ENAME`, 10)" + + " 10))['tokens'] AS MAP< STRING, ARRAY< STRING > >) `tokens`, `TAKE`(`ENAME`, 10)" + " `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, CASE WHEN `ENAME` IS NULL OR `ENAME` = '' THEN '' ELSE" @@ -410,7 +410,7 @@ public void testPatternsAggregationMode_ShowNumberedToken_ForBrainMethod() { String expectedSparkSql = "SELECT TRY_CAST(`t20`.`patterns_field`['pattern'] AS STRING) `patterns_field`," + " TRY_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT) `pattern_count`," - + " TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" + + " TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< STRING, ARRAY< STRING > >)" + " `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS ARRAY< STRING >)" + " `sample_logs`\n" + "FROM (SELECT `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" @@ -475,8 +475,8 @@ public void testPatternsAggregationModeWithGroupBy_ShowNumberedToken_ForBrainMet String expectedSparkSql = "SELECT `$cor0`.`DEPTNO`, TRY_CAST(`t20`.`patterns_field`['pattern'] AS STRING)" + " `patterns_field`, TRY_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT)" - + " `pattern_count`, TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR," - + " VARCHAR ARRAY >) `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS" + + " `pattern_count`, TRY_CAST(`t20`.`patterns_field`['tokens'] AS MAP< STRING," + + " ARRAY< STRING > >) `tokens`, TRY_CAST(`t20`.`patterns_field`['sample_logs'] AS" + " ARRAY< STRING >) `sample_logs`\n" + "FROM (SELECT `DEPTNO`, `pattern`(`ENAME`, 10, 100000, TRUE) `patterns_field`\n" + "FROM `scott`.`EMP`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java index e19d896283d..8844105f447 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLScalarSubqueryTest.java @@ -39,8 +39,8 @@ public void testUncorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`SAL`) `AVG(SAL)`\n" - + "FROM `scott`.`EMP`)))"; + + "WHERE `SAL` > (SELECT AVG(`SAL`) `AVG(SAL)`\n" + + "FROM `scott`.`EMP` `EMP0`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -66,9 +66,9 @@ public void testUncorrelatedScalarSubqueryInSelect() { String expectedSparkSql = "" - + "SELECT (((SELECT MIN(`EMPNO`) `min(EMPNO)`\n" - + "FROM `scott`.`EMP`))) `min_empno`, `SAL`\n" - + "FROM `scott`.`EMP`"; + + "SELECT (SELECT MIN(`EMPNO`) `min(EMPNO)`\n" + + "FROM `scott`.`EMP` `EMP0`) `min_empno`, `SAL`\n" + + "FROM `scott`.`EMP` `EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -107,11 +107,11 @@ public void testUncorrelatedScalarSubqueryInWhereAndSelect() { String expectedSparkSql = "SELECT `min_empno`, `SAL`\n" + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," - + " (((SELECT MIN(`EMPNO`) `min(EMPNO)`\n" - + "FROM `scott`.`EMP`))) `min_empno`\n" - + "FROM `scott`.`EMP`) `t1`\n" - + "WHERE `SAL` > (((SELECT AVG(`SAL`) `AVG(SAL)`\n" - + "FROM `scott`.`EMP`)))"; + + " (SELECT MIN(`EMPNO`) `min(EMPNO)`\n" + + "FROM `scott`.`EMP` `EMP0`) `min_empno`\n" + + "FROM `scott`.`EMP` `EMP`) `t1`\n" + + "WHERE `SAL` > (SELECT AVG(`SAL`) `AVG(SAL)`\n" + + "FROM `scott`.`EMP` `EMP1`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -140,9 +140,9 @@ public void testCorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + + "WHERE `SAL` > (SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`)))"; + + "WHERE `EMP`.`SAL` = `HISAL`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -170,10 +170,10 @@ public void testCorrelatedScalarSubqueryInSelect() { String expectedSparkSql = "" - + "SELECT (((SELECT MIN(`EMP`.`EMPNO`) `min(EMPNO)`\n" + + "SELECT (SELECT MIN(`EMP`.`EMPNO`) `min(EMPNO)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`))) `min_empno`, `SAL`\n" - + "FROM `scott`.`EMP`"; + + "WHERE `EMP`.`SAL` = `HISAL`) `min_empno`, `SAL`\n" + + "FROM `scott`.`EMP` `EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -201,9 +201,9 @@ public void testDisjunctiveCorrelatedScalarSubqueryInWhere() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE (((SELECT COUNT(*) `COUNT()`\n" + + "WHERE (SELECT COUNT(*) `COUNT()`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL` OR `HISAL` > 1000.0))) > 0"; + + "WHERE `EMP`.`SAL` = `HISAL` OR `HISAL` > 1000.0) > 0"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -230,10 +230,10 @@ public void testDisjunctiveCorrelatedScalarSubqueryInWhere2() { String expectedSparkSql = "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE (((SELECT COUNT(*) `COUNT()`\n" + + "WHERE (SELECT COUNT(*) `COUNT()`\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE `EMP`.`SAL` = `HISAL` AND `HISAL` > 1000.0 OR `EMP`.`SAL` = `HISAL` AND" - + " `LOSAL` > 1000.0))) > 0"; + + " `LOSAL` > 1000.0) > 0"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -268,14 +268,14 @@ public void testTwoScalarSubqueriesInOr() { String expectedSparkSql = "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` = (((SELECT MAX(`HISAL`) `max(HISAL)`\n" + + "WHERE `SAL` = (SELECT MAX(`HISAL`) `max(HISAL)`\n" + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" - + "ORDER BY `LOSAL`) `t0`))) OR `SAL` = (((SELECT MIN(`HISAL`) `min(HISAL)`\n" + + "ORDER BY `LOSAL`) `t0`) OR `SAL` = (SELECT MIN(`HISAL`) `min(HISAL)`\n" + "FROM (SELECT `HISAL`\n" + "FROM `scott`.`SALGRADE`\n" + "WHERE `LOSAL` > 1000.0\n" - + "ORDER BY `HISAL` DESC) `t4`)))"; + + "ORDER BY `HISAL` DESC) `t4`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -331,13 +331,13 @@ public void testNestedScalarSubquery() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` = (((SELECT MAX(`HISAL`) `max_hisal`\n" + + "WHERE `SAL` = (SELECT MAX(`HISAL`) `max_hisal`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `HISAL` = (((SELECT MAX(`SAL`) `max_sal`\n" - + "FROM `scott`.`EMP`\n" - + "GROUP BY `JOB`)))\n" + + "WHERE `HISAL` = (SELECT MAX(`SAL`) `max_sal`\n" + + "FROM `scott`.`EMP` `EMP0`\n" + + "GROUP BY `JOB`)\n" + "GROUP BY `GRADE`\n" - + "LIMIT 1)))"; + + "LIMIT 1)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -366,9 +366,9 @@ public void testCorrelatedScalarSubqueryInWhereMaxOut() { "" + "SELECT *\n" + "FROM `scott`.`EMP`\n" - + "WHERE `SAL` > (((SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + + "WHERE `SAL` > (SELECT AVG(`EMP`.`SAL`) `AVG(SAL)`\n" + "FROM `scott`.`SALGRADE`\n" - + "WHERE `EMP`.`SAL` = `HISAL`)))"; + + "WHERE `EMP`.`SAL` = `HISAL`)"; verifyPPLToSparkSQL(root, expectedSparkSql); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 43912b90572..4b19990ea1a 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -286,7 +286,7 @@ public void testLike() { String expectedLogical = "" + "LogicalAggregate(group=[{}], cnt=[COUNT()])\n" - + " LogicalFilter(condition=[LIKE($2, 'SALE%', '\\')])\n" + + " LogicalFilter(condition=[LIKE($2, 'SALE%':VARCHAR, '\\')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "cnt=4\n"; @@ -307,7 +307,7 @@ public void testILike() { String expectedLogical = "" + "LogicalAggregate(group=[{}], cnt=[COUNT()])\n" - + " LogicalFilter(condition=[ILIKE($2, 'SALE%', '\\')])\n" + + " LogicalFilter(condition=[ILIKE($2, 'SALE%':VARCHAR, '\\')])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "cnt=4\n"; From bf2111bc84a8ea0bab6ed28025923355f18bb15b Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:35:28 -0700 Subject: [PATCH 33/41] Fix protocol-dependent HTTP status assertion (#5623) Signed-off-by: Kai Huang --- .../sql/calcite/remote/CalcitePPLCaseFunctionIT.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index 8bba907e2e2..f40ca7b4f4e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -550,8 +550,9 @@ public void testCaseWithIncompatibleBranchTypesRejectsCleanly() { "source=%s | eval x = case(age > 30, 'old', age > 20, 1 else 0.0) | fields" + " x", TEST_INDEX_BANK))); - org.junit.Assert.assertTrue( + org.junit.Assert.assertEquals( "expected 400 status, got: " + e.getMessage(), - e.getMessage().contains("status line [HTTP/1.1 400")); + 400, + e.getResponse().getStatusLine().getStatusCode()); } } From 454ac4e7dd87d1f4120d5eed4bf0b0a8893d5838 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 14 Jul 2026 22:55:47 -0700 Subject: [PATCH 34/41] [Feature] Add PPL `rest` command (#5599) * [Feature] Add PPL `rest` command (Calcite system row source) Add a leading `rest ` command that exposes a curated, read-only, fixed-schema set of in-cluster management endpoints as a PPL table, modeled as a system row source bridged through visitRelation (the same seam as describe and the system-index family), so it runs on the Calcite engine without the unsupported table-function path. - Grammar/AST: REST/TIMEOUT tokens, restCommand rule, RestRelation, AstBuilder visit, query anonymizer. - Execution: RestSourceTable -> CalciteLogicalRestScan / CalciteEnumerableRestScan; RestEndpointRegistry (read-only allow-list + fixed schema + accepted args); RestEnumerator/RestRequest dispatch via OpenSearchClient (NodeClient in-cluster, RestClient standalone). - 9 endpoints: cluster health/state/settings, cat indices/nodes/cluster_manager/ plugins/shards, resolve/index. - Output shaping: numeric type normalization, id-to-name resolution, role-name expansion, structural flattening, graceful null degrade. - Args: count caps emitted rows; timeout reserved but rejected with 400; get-args applied server-side with per-arg value validation (local on health, health on cat/indices, expand_wildcards on resolve/index). Undeclared arg or out-of-domain value is rejected with a 400. level and include_defaults are deferred to a later release; flat_settings is dropped as redundant. - Error handling: blank endpoint, negative count, disallowed arg, and uncoercible value all surface clean 400s rather than 500s. Tests: CalcitePPLRestIT 25/25, RestEndpointRegistryTest 16, RestSourceTableTest 10. Signed-off-by: Louis Chu * Align rest '/_cluster/settings' redaction with native endpoint; CI fixes; comment cleanup - /_cluster/settings: run the persistent and transient tiers through the node SettingsFilter (published via RestSettingsFilterHolder from SQLPlugin#getRestHandlers) so Property.Filtered and plugin-registered pattern settings are redacted exactly as the native GET /_cluster/settings endpoint. Remove the dead secretFields column-filter, which was the wrong shape for the (setting, value, tier) rows. - Parser: add TIMEOUT to searchableKeyWord so a bare 'timeout' term still matches searchLiteral. - coerce(): narrow the catch to IllegalArgumentException | ClassCastException; add empty-string guards in toNumber/toBoolean. - spotlessApply formatting; drop outdated and redundant comments. Tests: RestEndpointRegistryTest, RestSourceTableTest, OpenSearchNodeClientClusterSettingsFilterTest green. Signed-off-by: Louis Chu * Address rest command bot-review findings; register rest doctest - clusterSettings: fail closed (throw IllegalStateException) when the node SettingsFilter is unavailable, instead of returning unredacted settings - collectSettings: handle list-type settings via getAsList fallback - decodeRestSpec: reject a blank/missing endpoint with a clear error - docs: correct rest.md allow-list table (9 endpoints + accepted args), quote endpoint literals, fix timeout/get-arg descriptions, add security note - register docs/user/ppl/cmd/rest.md in docs/category.json (deterministic single-node examples: number_of_nodes=1, cluster_manager count=1) Signed-off-by: Louis Chu * Harden decodeRestSpec: reject non-rest-source tokens with a clear error decodeRestSpec is only ever called behind an isRestSource gate today, but as a public decoder it must not assume its precondition. Without the guard a malformed token would surface an opaque StringIndexOutOfBoundsException from substring; now it throws a clear IllegalArgumentException instead. Addresses the PR #5599 Code Suggestions finding (importance 8). Signed-off-by: Louis Chu * Fix rest explain IT (JSON payload) and harden cluster-settings/state fetch - CalciteExplainIT.explainRestCommand: single-quote the endpoint literal; the explain harness inlines the query into a JSON body without escaping, so a double-quoted literal produced an invalid payload and a 400 (integration CI failure). - OpenSearchNodeClient.clusterSettings: resolve the SettingsFilter and fail closed BEFORE fetching cluster state, so settings are never read into memory when the redaction filter is unavailable. - OpenSearchRestClient.clusterState: narrow filter_path to nodes.*.name so node IPs/attributes are not over-fetched; manager-name resolution is preserved. Signed-off-by: Louis Chu * Rest command: analytics-engine coexistence IT + hardened source-token decode - AnalyticsEngineCompatIT: assert | rest '/_cluster/health' behaves identically with the analytics engine enabled (rest is never routed to DataFusion). - SystemIndexUtils.fromHex: reject an odd-length hex body so a crafted source name that passes the isRestSource suffix check fails clearly rather than silently dropping the trailing half-byte. Signed-off-by: Louis Chu * [Bugfix] Keep rest command on Calcite path under cluster-composite On a cluster started with cluster.pluggable.dataformat=composite, RestUnifiedQueryAction.isAnalyticsIndex() routed every non-system-catalog PPL query to the analytics engine. The rest command's reserved in-cluster source (REST...__REST_SOURCE) has no backing index and only resolves on the Calcite path, so it was routed to DataFusion and failed with "Table 'REST...__REST_SOURCE' not found". Fix: exclude isRestSource(name) alongside isSystemCatalog(name) so a rest source falls back to the default (Calcite) pipeline and is never routed to the analytics engine. - RestUnifiedQueryActionTest: unit repro under cluster-composite. - integ-test analyticsEngineCompat testcluster set composite-default so the existing rest coexistence IT exercises this routing exclusion. Signed-off-by: Louis Chu * [Refactor] Unify system-index and rest scans behind one catalog table Collapse the two near-duplicate Calcite scan hierarchies -- SHOW/DESCRIBE system tables and the rest command -- into one generic OpenSearchCatalogTable whose per-endpoint behavior is supplied by a pluggable CatalogSource. - OpenSearchSystemIndex + RestSourceTable -> one OpenSearchCatalogTable backed by SystemIndexCatalogSource / RestCatalogSource. - Two Abstract/Logical/Enumerable scans + two enumerators + two converter rules -> AbstractCalciteCatalogScan / CalciteLogicalCatalogScan / CalciteEnumerableCatalogScan (+ rest-only CalciteScannableCatalogScan) / OpenSearchCatalogEnumerator / EnumerableCatalogScanRule. - Concerns stay per-source: system tables keep the real V2 implement() path; rest is Calcite-only (implement() throws) and opts into Scannable for the collect short-circuit. No behavior change: dispatch, schemas, V2 support, and the Scannable marker are preserved; this is pure de-duplication of the Calcite scan plumbing. Verified: opensearch compileJava/compileTestJava green; ppl and integ-test test-compile green; affected unit tests pass. Signed-off-by: Louis Chu * [Feature] Add rest command response redaction and endpoint allow-list Two dynamic cluster settings gate the rest command per deployment: - plugins.ppl.rest.redaction.enabled (default false): mask network identifiers in _cat/* cells and availability-zone names in _cluster/settings values. - plugins.ppl.rest.allowed_endpoints (default all): restrict which endpoints are served; an empty list disables the rest command. Both default to open-source parity: all endpoints served, no masking. Signed-off-by: Louis Chu * [Refactor] Remove unused REST/TIMEOUT rules from shared language grammar The shared language-grammar carried REST and TIMEOUT lexer tokens and the restCommand/restArgument parser rules with no consumer: async-query-core has no rest visitor, and the rest command grammar lives in the ppl module. Remove the dead rules. Signed-off-by: Louis Chu * [Test] Add rest command security integration tests Verify the rest command is subject to the security plugin fine grained access control: a caller without cluster:monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege can run them, and the resolve index endpoint is filtered to the caller authorized indices. Test indices are created idempotently, and denials are asserted by the security denial reason in the response body because a denied action on the Calcite only rest path surfaces as a wrapped error carrying that reason. Calcite fallback is disabled so the denial reason is not replaced by an unsupported command error. Signed-off-by: Louis Chu * [Bugfix] Make rest redaction and allow-list settings node-level plugins.ppl.rest.redaction.enabled and plugins.ppl.rest.allowed_endpoints were dynamic cluster settings, so they could be changed at runtime through _cluster/settings or the _plugins/_query/settings endpoint. On a managed deployment that let a caller disable redaction or widen the allow-list an operator had configured. Drop Setting.Property.Dynamic so both are node-level settings set in the node config; the engine rejects runtime updates on both paths. Register them without an update consumer and read the node-configured value. Signed-off-by: Louis Chu * Enhance redaction logic Signed-off-by: Louis Chu * [Change] Disable all rest endpoints by default (empty allow-list) Flip plugins.ppl.rest.allowed_endpoints default from ["*"] to empty so open source ships with the rest command closed. Deployments opt specific endpoints in via the setting (AOS enables the ones it supports; AOSS leaves it empty and stays disabled). Enforcement already treats an empty or missing list as disabled. Opt the integration-test clusters into all endpoints so the rest ITs still exercise the enabled path. Signed-off-by: Louis Chu --------- Signed-off-by: Louis Chu Signed-off-by: Louis Chu --- .../sql/common/setting/Settings.java | 2 + .../opensearch/sql/ast/tree/RestRelation.java | 25 + .../sql/utils/SystemIndexUtils.java | 118 +++++ docs/category.json | 1 + docs/user/ppl/cmd/rest.md | 94 ++++ docs/user/ppl/index.md | 1 + doctest/build.gradle | 5 + integ-test/build.gradle | 10 + .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 9 + .../sql/calcite/remote/CalcitePPLRestIT.java | 210 +++++++++ .../sql/plugin/AnalyticsEngineCompatIT.java | 31 ++ .../sql/ppl/NewAddedCommandsIT.java | 13 + .../sql/security/RestCommandSecurityIT.java | 154 +++++++ .../opensearch/sql/sql/VectorSearchIT.java | 1 + .../opensearch/client/OpenSearchClient.java | 101 +++++ .../client/OpenSearchNodeClient.java | 273 +++++++++++ .../client/OpenSearchRestClient.java | 268 +++++++++++ .../rules/EnumerableCatalogScanRule.java | 63 +++ .../rules/EnumerableSystemIndexScanRule.java | 50 -- .../planner/rules/OpenSearchIndexRules.java | 6 +- .../setting/OpenSearchSettings.java | 27 +- .../storage/OpenSearchStorageEngine.java | 31 +- .../storage/rest/RestCatalogSource.java | 63 +++ .../storage/rest/RestEndpointRegistry.java | 428 ++++++++++++++++++ .../opensearch/storage/rest/RestRequest.java | 57 +++ .../storage/rest/RestResponseRedactor.java | 64 +++ .../rest/RestSettingsFilterHolder.java | 40 ++ ...n.java => AbstractCalciteCatalogScan.java} | 12 +- ...java => CalciteEnumerableCatalogScan.java} | 23 +- ...an.java => CalciteLogicalCatalogScan.java} | 20 +- .../system/CalciteScannableCatalogScan.java | 29 ++ .../storage/system/CatalogSource.java | 39 ++ ....java => OpenSearchCatalogEnumerator.java} | 9 +- .../system/OpenSearchCatalogTable.java | 67 +++ .../storage/system/OpenSearchSystemIndex.java | 105 ----- .../system/SystemIndexCatalogSource.java | 70 +++ ...chNodeClientClusterSettingsFilterTest.java | 99 ++++ .../setting/OpenSearchSettingsTest.java | 12 + .../storage/OpenSearchStorageEngineTest.java | 71 ++- .../storage/rest/RestCatalogSourceTest.java | 123 +++++ .../rest/RestEndpointRegistryTest.java | 290 ++++++++++++ .../rest/RestResponseRedactorTest.java | 98 ++++ ...t.java => OpenSearchCatalogTableTest.java} | 25 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 + .../plugin/rest/RestUnifiedQueryAction.java | 9 +- .../rest/RestUnifiedQueryActionTest.java | 6 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 14 + .../opensearch/sql/ppl/parser/AstBuilder.java | 33 ++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 19 + .../sql/ppl/calcite/CalcitePPLRestTest.java | 67 +++ .../sql/ppl/parser/AstBuilderTest.java | 28 ++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 12 + 54 files changed, 3226 insertions(+), 206 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java create mode 100644 docs/user/ppl/cmd/rest.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{AbstractCalciteSystemIndexScan.java => AbstractCalciteCatalogScan.java} (68%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteEnumerableSystemIndexScan.java => CalciteEnumerableCatalogScan.java} (80%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteLogicalSystemIndexScan.java => CalciteLogicalCatalogScan.java} (59%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexEnumerator.java => OpenSearchCatalogEnumerator.java} (90%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexTest.java => OpenSearchCatalogTableTest.java} (76%) create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index bf3e65d8741..5473aa8812e 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,8 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..7589cf522f6 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,9 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +37,121 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + StringBuilder h = new StringBuilder(); + for (byte b : s.getBytes(StandardCharsets.UTF_8)) { + h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return h.toString(); + } + + private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } + byte[] bytes = new byte[h.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = + (byte) + ((Character.digit(h.charAt(2 * i), 16) << 4) + + Character.digit(h.charAt(2 * i + 1), 16)); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/docs/category.json b/docs/category.json index 1e45f3e65bb..04f2bbae22e 100644 --- a/docs/category.json +++ b/docs/category.json @@ -34,6 +34,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md new file mode 100644 index 00000000000..a9761aa87e8 --- /dev/null +++ b/docs/user/ppl/cmd/rest.md @@ -0,0 +1,94 @@ +# rest + +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. + +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. + +## Enabling the command + +The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): + +```yaml +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] +``` + +Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. + +The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. + +## Syntax + +The `rest` command has the following syntax: + +```syntax +rest [count=] [timeout=] [= ...] +``` + +## Parameters + +The `rest` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | +| `count=` | Optional | Caps the number of emitted rows. | +| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | + +## Allow-list + +`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. + +| Endpoint | Output columns | Accepted args | +| --- | --- | --- | +| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | +| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | +| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | +| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | +| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | +| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | +| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | +| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | +| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | + +## Example 1: Counting the nodes in the cluster + +The following query reads cluster health and projects a column that is deterministic on a single-node cluster: + +```ppl +| rest '/_cluster/health' | fields number_of_nodes +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++-----------------+ +| number_of_nodes | +|-----------------| +| 1 | ++-----------------+ +``` + +`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. + +## Example 2: Composing downstream commands over a cat endpoint + +The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: + +```ppl +| rest '/_cat/cluster_manager' | stats count() as managers +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------+ +| managers | +|----------| +| 1 | ++----------+ +``` + +For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 37947113800..939684c0ecc 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,6 +74,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/doctest/build.gradle b/doctest/build.gradle index cce64170f56..1ac658457dc 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,6 +205,11 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' + // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into + // the registered endpoints so the rest.md examples run against an enabled command. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1435d1d499d..c18fa6e37f6 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,6 +387,11 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + // The rest command is disabled by default (empty allow-list). Opt this cluster into the + // registered endpoints so the rest integration tests exercise the enabled path. A literal + // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -405,6 +410,9 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" + // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. + setting 'plugins.ppl.rest.allowed_endpoints', + '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -419,6 +427,8 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" + // Composite-default cluster: PPL queries route to the analytics engine unless excluded. + setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index d46649d56d9..da434750a96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -57,6 +57,7 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, + CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 9244981125f..d4a9bdffd5a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -66,6 +66,15 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. + @Test + public void explainRestCommand() throws IOException { + String result = explainQueryToString("| rest '/_cluster/health' | fields status"); + Assert.assertTrue( + "Expected a rest scan node in the explain output, got: " + result, + result.contains("CatalogScan")); + } + @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..70ee4ef0de6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,210 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code + * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also + * verifies that a non-allow-listed / mutating endpoint is refused. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); + } + + @Test + public void testRestClusterHealthDataRows() throws IOException { + // Single-node test cluster: exactly one node, status is green or yellow. + JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() throws IOException { + assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() throws IOException { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() throws IOException { + assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); + } + + @Test + public void testRestRejectsTimeoutArg() throws IOException { + assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. Covers allow-list and bad-argument rejection. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } + + @Test + public void testRestCatIndicesSchema() throws IOException { + // Schema is fixed by the registry, independent of how many indices exist. + JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); + verifySchema(result, schema("index", "string"), schema("health", "string")); + } + + @Test + public void testRestCatIndicesReturnsCreatedIndex() throws IOException { + // Create a known index, then confirm rest surfaces it and downstream where/fields compose. + client().performRequest(new Request("PUT", "/rest_cat_test")); + JSONObject result = + executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); + verifyDataRows(result, rows("rest_cat_test")); + } + + @Test + public void testRestCatNodesSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); + verifySchema(result, schema("name", "string"), schema("cpu", "int")); + } + + @Test + public void testRestCatNodesSingleNode() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatClusterManagerSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); + verifySchema(result, schema("node", "string"), schema("id", "string")); + } + + @Test + public void testRestCatClusterManagerSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatPluginsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); + verifySchema(result, schema("component", "string"), schema("version", "string")); + } + + @Test + public void testRestCatShardsSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); + verifySchema( + result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); + } + + @Test + public void testRestClusterStateSchema() throws IOException { + // Assert the string columns; version is the LONG epoch column. + JSONObject result = + executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); + verifySchema( + result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); + } + + @Test + public void testRestClusterStateSingleRow() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterSettingsSchema() throws IOException { + // Schema is registry-fixed regardless of how many settings are configured. + JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); + verifySchema( + result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); + } + + @Test + public void testRestResolveIndexSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { + client().performRequest(new Request("PUT", "/rest_resolve_test")); + JSONObject result = + executeQuery( + "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); + verifyDataRows(result, rows("rest_resolve_test", "index")); + } + + // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { + // health filters rows server-side; a healthy cluster has no red indices, so count is 0. + JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); + verifyDataRows(result, rows(0)); + } + + @Test + public void testRestResolveIndexExpandWildcardsArg() throws IOException { + // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. + JSONObject result = + executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); + verifySchema(result, schema("name", "string"), schema("type", "string")); + } + + @Test + public void testRestRejectsDroppedLevelArg() throws IOException { + // level was dropped from the allow-list (no-op against the fixed health schema). + assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); + } + + @Test + public void testRestRejectsBadArgValue() throws IOException { + assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..b5c08cc8ad0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,31 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * fixed schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(2, schema.length()); + assertEquals("status", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); + assertEquals("int", schema.getJSONObject(1).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..2ccda31eea7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,6 +32,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..042a3aefbf7 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that verify the rest command is subject to the security plugin fine grained + * access control. The command dispatches standard transport actions under the caller identity, so + * the security ActionFilter authorizes each one by action name. A caller without the required + * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege + * can run them, and the resolve index endpoint requires the resolve index privilege because the + * command resolves all indices. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String ALPHA_INDEX = "rest_sec_alpha"; + private static final String BETA_INDEX = "rest_sec_beta"; + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesUsersAndIndices(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesUsersAndIndices() throws IOException { + createIndexIfAbsent(ALPHA_INDEX); + createIndexIfAbsent(BETA_INDEX); + + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] { + "cluster:admin/opensearch/ppl", + "cluster:monitor/health", + "cluster:monitor/state", + "cluster:monitor/nodes/stats", + "cluster:monitor/nodes/info" + }, + new String[] {"indices:admin/resolve/index"}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, + ALPHA_INDEX, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] {"indices:data/read/search*"}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunCatNodes() throws IOException { + JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); + verifyColumn(result, columnName("name")); + } + + @Test + public void monitorUserCanResolveIndex() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); + Set names = resolvedNames(result); + assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); + assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); + } + + @Test + public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { + assertDenied( + "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { + assertDenied( + "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); + } + + @Test + public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { + assertDenied( + "| rest '/_resolve/index' | fields name, type", + NO_MONITOR_USER, + "indices:admin/resolve/index"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } + + private void createIndexIfAbsent(String name) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity( + "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); + try { + client().performRequest(request); + } catch (ResponseException e) { + String body = TestUtils.getResponseBody(e.getResponse(), false); + if (!body.contains("resource_already_exists_exception")) { + throw e; + } + } + } + + private Set resolvedNames(JSONObject result) { + Set names = new HashSet<>(); + JSONArray datarows = result.getJSONArray("datarows"); + for (int i = 0; i < datarows.length(); i++) { + names.add(datarows.getJSONArray(i).getString(0)); + } + return names; + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index f3c3046f41e..c10bc474bae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -28,6 +28,7 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 68350c5a0fd..3b9c3619521 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,4 +114,105 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); + + /** + * Read-only cluster health snapshot for the {@code rest} command (backs {@code + * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's + * security thread-context; performs no privilege escalation and mutates nothing. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of health field name to value + */ + default Map clusterHealth(Map params) { + throw new UnsupportedOperationException("clusterHealth is not supported by this client"); + } + + /** + * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One + * map per index. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per index + */ + default List> catIndices(Map params) { + throw new UnsupportedOperationException("catIndices is not supported by this client"); + } + + /** + * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map + * per node with resource state. Runs under the caller's security thread-context; read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per node + */ + default List> catNodes(Map params) { + throw new UnsupportedOperationException("catNodes is not supported by this client"); + } + + /** + * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code + * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map describing the cluster manager node + */ + default List> catClusterManager(Map params) { + throw new UnsupportedOperationException("catClusterManager is not supported by this client"); + } + + /** + * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One + * map per installed plugin per node. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per plugin + */ + default List> catPlugins(Map params) { + throw new UnsupportedOperationException("catPlugins is not supported by this client"); + } + + /** + * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map + * per shard. Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per shard + */ + default List> catShards(Map params) { + throw new UnsupportedOperationException("catShards is not supported by this client"); + } + + /** + * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code + * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, + * cluster_manager_node). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return a single map of cluster-state field name to value + */ + default Map clusterState(Map params) { + throw new UnsupportedOperationException("clusterState is not supported by this client"); + } + + /** + * Read-only cluster-settings listing for the {@code rest} command (backs {@code + * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per setting + */ + default List> clusterSettings(Map params) { + throw new UnsupportedOperationException("clusterSettings is not supported by this client"); + } + + /** + * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). + * One map per resolved index, alias, or data stream (name, type). Read-only. + * + * @param params endpoint query args (already allow-list-validated) + * @return one map of column name to value per resolved name + */ + default List> resolveIndex(Map params) { + throw new UnsupportedOperationException("resolveIndex is not supported by this client"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..080a8627894 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,6 +18,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -25,6 +27,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -285,4 +288,274 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + return flattenHealth(response); + } + + @Override + public List> catIndices(Map params) { + ClusterHealthResponse response = + client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new java.util.LinkedHashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } + + @Override + public List> catNodes(Map params) { + org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = + new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); + statsRequest.all(); + org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = + client.admin().cluster().nodesStats(statsRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { + org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); + Map row = new java.util.LinkedHashMap<>(); + row.put("name", node.getName()); + row.put("ip", node.getHostAddress()); + row.put( + "node_role", + node.getRoles().stream() + .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) + .sorted() + .collect(java.util.stream.Collectors.joining(","))); + row.put( + "heap_percent", + ns.getJvm() == null || ns.getJvm().getMem() == null + ? null + : (int) ns.getJvm().getMem().getHeapUsedPercent()); + row.put( + "ram_percent", + ns.getOs() == null || ns.getOs().getMem() == null + ? null + : (int) ns.getOs().getMem().getUsedPercent()); + row.put( + "cpu", + ns.getProcess() == null || ns.getProcess().getCpu() == null + ? null + : (int) ns.getProcess().getCpu().getPercent()); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + List> rows = new java.util.ArrayList<>(); + if (cm != null) { + Map row = new java.util.LinkedHashMap<>(); + row.put("id", cm.getId()); + row.put("host", cm.getHostName()); + row.put("ip", cm.getHostAddress()); + row.put("node", cm.getName()); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = + new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); + infoRequest.all(); + org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = + client.admin().cluster().nodesInfo(infoRequest).actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { + org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = + info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); + if (plugins == null) { + continue; + } + for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", info.getNode().getName()); + row.put("component", pi.getName()); + row.put("version", pi.getVersion()); + rows.add(row); + } + } + return rows; + } + + @Override + public List> catShards(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.cluster.routing.ShardRouting sr : + response.getState().getRoutingTable().allShards()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("index", sr.getIndexName()); + row.put("shard", sr.id()); + row.put("prirep", sr.primary() ? "p" : "r"); + row.put("state", sr.state().name()); + org.opensearch.cluster.node.DiscoveryNode n = + sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); + row.put("node", n == null ? null : n.getName()); + rows.add(row); + } + return rows; + } + + @Override + public Map clusterState(Map params) { + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName().value()); + row.put("state_uuid", response.getState().stateUUID()); + row.put("version", response.getState().version()); + org.opensearch.cluster.node.DiscoveryNode cm = + response.getState().nodes().getClusterManagerNode(); + row.put("cluster_manager_node", cm == null ? null : cm.getName()); + return row; + } + + @Override + public List> clusterSettings(Map params) { + // The transport path has no SettingsFilter of its own; it is published from + // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings + // into memory when we cannot redact them, matching native GET /_cluster/settings. + org.opensearch.common.settings.SettingsFilter filter = + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); + if (filter == null) { + throw new IllegalStateException( + "cluster settings redaction filter is not initialized; refusing to return unredacted" + + " settings"); + } + org.opensearch.action.admin.cluster.state.ClusterStateResponse response = + client + .admin() + .cluster() + .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + org.opensearch.common.settings.Settings persistent = + filter.filter(response.getState().metadata().persistentSettings()); + org.opensearch.common.settings.Settings transientSettings = + filter.filter(response.getState().metadata().transientSettings()); + collectSettings(persistent, "persistent", rows); + collectSettings(transientSettings, "transient", rows); + return rows; + } + + private void collectSettings( + org.opensearch.common.settings.Settings settings, + String tier, + List> rows) { + if (settings == null) { + return; + } + for (String key : settings.keySet()) { + Map row = new java.util.LinkedHashMap<>(); + row.put("setting", key); + String value = settings.get(key); + if (value == null) { + // List-valued settings return null from get(); fall back to the joined list form. + java.util.List list = settings.getAsList(key); + value = list.isEmpty() ? null : String.join(",", list); + } + row.put("value", value); + row.put("tier", tier); + rows.add(row); + } + } + + @Override + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = + expandWildcards == null + ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}) + : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( + new String[] {"*"}, + org.opensearch.action.support.IndicesOptions.fromParameters( + expandWildcards, + null, + null, + null, + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request + .DEFAULT_INDICES_OPTIONS)); + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = + client + .execute( + org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) + .actionGet(); + List> rows = new java.util.ArrayList<>(); + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : + response.getIndices()) { + rows.add(resolveRow(idx.getName(), "index")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : + response.getAliases()) { + rows.add(resolveRow(alias.getName(), "alias")); + } + for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : + response.getDataStreams()) { + rows.add(resolveRow(ds.getName(), "data_stream")); + } + return rows; + } + + private Map resolveRow(String name, String type) { + Map row = new java.util.LinkedHashMap<>(); + row.put("name", name); + row.put("type", type); + return row; + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new java.util.LinkedHashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index f369c0003b8..e98c5bf95f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,15 +8,19 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -28,6 +32,7 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; +import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -272,4 +277,267 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public Map clusterHealth(Map params) { + try { + ClusterHealthRequest request = new ClusterHealthRequest(); + if (params != null && Boolean.parseBoolean(params.get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); + return flattenHealth(response); + } catch (IOException e) { + throw new IllegalStateException("Failed to get cluster health", e); + } + } + + @Override + public List> catIndices(Map params) { + try { + ClusterHealthResponse response = + client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + List> rows = new ArrayList<>(); + for (Map.Entry entry : response.getIndices().entrySet()) { + ClusterIndexHealth health = entry.getValue(); + Map row = new HashMap<>(); + row.put("index", entry.getKey()); + row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("pri", health.getNumberOfShards()); + row.put("rep", health.getNumberOfReplicas()); + row.put("active_shards", health.getActiveShards()); + rows.add(row); + } + String healthFilter = params == null ? null : params.get("health"); + if (healthFilter != null) { + rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); + } + return rows; + } catch (IOException e) { + throw new IllegalStateException("Failed to get cat indices", e); + } + } + + @Override + public List> catNodes(Map params) { + List> raw = + catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("ip", r.get("ip")); + row.put("node_role", r.get("node.role")); + row.put("heap_percent", asInt(r.get("heap.percent"))); + row.put("ram_percent", asInt(r.get("ram.percent"))); + row.put("cpu", asInt(r.get("cpu"))); + rows.add(row); + } + return rows; + } + + @Override + public List> catClusterManager(Map params) { + List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("id", r.get("id")); + row.put("host", r.get("host")); + row.put("ip", r.get("ip")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + @Override + public List> catPlugins(Map params) { + List> raw = catJson("/_cat/plugins", "name,component,version"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("name", r.get("name")); + row.put("component", r.get("component")); + row.put("version", r.get("version")); + rows.add(row); + } + return rows; + } + + @Override + public List> catShards(Map params) { + List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); + List> rows = new ArrayList<>(); + for (Map r : raw) { + Map row = new HashMap<>(); + row.put("index", r.get("index")); + row.put("shard", asInt(r.get("shard"))); + row.put("prirep", r.get("prirep")); + row.put("state", r.get("state")); + row.put("node", r.get("node")); + rows.add(row); + } + return rows; + } + + /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ + @SuppressWarnings("unchecked") + private List> catJson(String path, String columns) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + request.addParameter("format", "json"); + request.addParameter("h", columns); + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + List list = parser.list(); + List> rows = new ArrayList<>(); + for (Object o : list) { + rows.add((Map) o); + } + return rows; + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Integer asInt(Object value) { + if (value == null) { + return null; + } + try { + return (int) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public Map clusterState(Map params) { + Map state = + getJsonMap( + "/_cluster/state/master_node,version,metadata,nodes", + // nodes.*.name resolves the manager id to a name without over-fetching node IPs. + Map.of( + "filter_path", + "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); + Map row = new HashMap<>(); + row.put("cluster_name", state.get("cluster_name")); + row.put("state_uuid", state.get("state_uuid")); + row.put("version", asLong(state.get("version"))); + Object cmId = state.get("cluster_manager_node"); + String cmName = null; + Object nodes = state.get("nodes"); + if (cmId != null && nodes instanceof Map) { + Object n = ((Map) nodes).get(cmId.toString()); + if (n instanceof Map) { + Object name = ((Map) n).get("name"); + cmName = name == null ? null : name.toString(); + } + } + row.put("cluster_manager_node", cmName); + return row; + } + + @Override + @SuppressWarnings("unchecked") + public List> clusterSettings(Map params) { + Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); + List> rows = new ArrayList<>(); + for (String tier : new String[] {"persistent", "transient"}) { + Object section = body.get(tier); + if (section instanceof Map) { + for (Map.Entry e : ((Map) section).entrySet()) { + Map row = new HashMap<>(); + row.put("setting", e.getKey()); + row.put("value", e.getValue() == null ? null : e.getValue().toString()); + row.put("tier", tier); + rows.add(row); + } + } + } + return rows; + } + + /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ + @SuppressWarnings("unchecked") + private Map getJsonMap(String path, Map params) { + try { + org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); + if (params != null) { + params.forEach(request::addParameter); + } + org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); + try (org.opensearch.core.xcontent.XContentParser parser = + org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( + org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, + org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent())) { + return parser.map(); + } + } catch (IOException e) { + throw new IllegalStateException("Failed GET " + path, e); + } + } + + private static Long asLong(Object value) { + if (value == null) { + return null; + } + try { + return (long) Double.parseDouble(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + + @Override + @SuppressWarnings("unchecked") + public List> resolveIndex(Map params) { + String expandWildcards = params == null ? null : params.get("expand_wildcards"); + Map body = + getJsonMap( + "/_resolve/index/*", + expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); + List> rows = new ArrayList<>(); + addResolved(body.get("indices"), "index", rows); + addResolved(body.get("aliases"), "alias", rows); + addResolved(body.get("data_streams"), "data_stream", rows); + return rows; + } + + @SuppressWarnings("unchecked") + private void addResolved(Object section, String type, List> rows) { + if (section instanceof List) { + for (Object o : (List) section) { + if (o instanceof Map) { + Map row = new HashMap<>(); + row.put("name", ((Map) o).get("name")); + row.put("type", type); + rows.add(row); + } + } + } + } + + private Map flattenHealth(ClusterHealthResponse response) { + Map row = new HashMap<>(); + row.put("cluster_name", response.getClusterName()); + row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); + row.put("number_of_nodes", response.getNumberOfNodes()); + row.put("number_of_data_nodes", response.getNumberOfDataNodes()); + row.put("active_primary_shards", response.getActivePrimaryShards()); + row.put("active_shards", response.getActiveShards()); + row.put("relocating_shards", response.getRelocatingShards()); + row.put("initializing_shards", response.getInitializingShards()); + row.put("unassigned_shards", response.getUnassignedShards()); + row.put("timed_out", response.isTimedOut()); + return row; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index b596c7bc47a..ce2bdd4960b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,6 +71,17 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = + Setting.boolSetting( + Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); + + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of(), + Function.identity(), + Setting.Property.NodeScope); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -380,6 +391,16 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_REDACTION_ENABLED, + PPL_REST_REDACTION_ENABLED_SETTING); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -640,7 +661,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -711,6 +734,8 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_REDACTION_ENABLED_SETTING) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..7b911471242 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,9 +17,13 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -35,10 +41,29 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + if (isRestSource(name)) { + return restTable(name); + } else if (isSystemIndex(name)) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry.resolve(spec.getEndpoint()); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); + return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..96779726a7e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. + * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + private final boolean redact; + + public RestCatalogSource(OpenSearchClient client, RestSpec spec) { + this(client, spec, false); + } + + public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { + this.client = client; + this.spec = spec; + this.redact = redact; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); + RestEndpointRegistry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec, redact); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..e64a91ab54a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,428 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; +import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; +import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps + * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so + * the Calcite plan can fix its row type at plan time), and the query args it accepts. + * + *

    This is the single place the read-only allow-list is enforced. Endpoints outside the registry, + * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. + * Adding an endpoint is a reviewed change here, never arbitrary pass-through. + */ +public final class RestEndpointRegistry { + + private RestEndpointRegistry() {} + + /** Produces the raw rows for an endpoint via a read-only client call. */ + @FunctionalInterface + public interface RowFetcher { + List> fetch(OpenSearchClient client, RestSpec spec); + } + + /** A single allow-listed endpoint description. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final Set allowedArgs; + private final RowFetcher fetcher; + + Endpoint( + String path, + LinkedHashMap schema, + Set allowedArgs, + RowFetcher fetcher) { + this.path = path; + this.schema = schema; + this.allowedArgs = allowedArgs; + this.fetcher = fetcher; + } + + /** Dispatch the read-only call and shape the response into fixed-schema rows. */ + public List toRows(OpenSearchClient client, RestSpec spec) { + return toRows(client, spec, false); + } + + /** + * Shape the response into fixed-schema rows, masking network identifiers when redaction is + * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value + * column is zone-masked. Off by default. + */ + public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { + boolean redactCat = redact && path.startsWith("/_cat"); + boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); + List out = new ArrayList<>(); + for (Map raw : fetcher.fetch(client, spec)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + for (Map.Entry col : schema.entrySet()) { + ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); + tuple.put( + col.getKey(), + maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); + } + out.add(new ExprTupleValue(tuple)); + } + return out; + } + + private static ExprValue maskCell( + String column, + ExprType type, + ExprValue value, + boolean redactCat, + boolean redactSettingsValue) { + if (type != STRING || value.isNull()) { + return value; + } + if (redactCat) { + return stringValue(RestResponseRedactor.redact(value.stringValue())); + } + if (redactSettingsValue && "value".equals(column)) { + return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); + } + return value; + } + } + + private static final Map REGISTRY = buildRegistry(); + + private static Map buildRegistry() { + Map m = new LinkedHashMap<>(); + + // /_cluster/health — single-row cluster health snapshot (read-only monitor action). + LinkedHashMap healthSchema = new LinkedHashMap<>(); + healthSchema.put("cluster_name", STRING); + healthSchema.put("status", STRING); + healthSchema.put("number_of_nodes", INTEGER); + healthSchema.put("number_of_data_nodes", INTEGER); + healthSchema.put("active_primary_shards", INTEGER); + healthSchema.put("active_shards", INTEGER); + healthSchema.put("relocating_shards", INTEGER); + healthSchema.put("initializing_shards", INTEGER); + healthSchema.put("unassigned_shards", INTEGER); + healthSchema.put("timed_out", BOOLEAN); + m.put( + "/_cluster/health", + new Endpoint( + "/_cluster/health", + healthSchema, + Set.of("local"), + (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); + + // /_cat/indices — one row per index (read-only monitor action). + LinkedHashMap catSchema = new LinkedHashMap<>(); + catSchema.put("index", STRING); + catSchema.put("health", STRING); + catSchema.put("pri", INTEGER); + catSchema.put("rep", INTEGER); + catSchema.put("active_shards", INTEGER); + m.put( + "/_cat/indices", + new Endpoint( + "/_cat/indices", + catSchema, + Set.of("health"), + (client, spec) -> client.catIndices(spec.getArgs()))); + + // /_cat/nodes — one row per node with resource state (read-only monitor action). + LinkedHashMap nodesSchema = new LinkedHashMap<>(); + nodesSchema.put("name", STRING); + nodesSchema.put("ip", STRING); + nodesSchema.put("node_role", STRING); + nodesSchema.put("heap_percent", INTEGER); + nodesSchema.put("ram_percent", INTEGER); + nodesSchema.put("cpu", INTEGER); + m.put( + "/_cat/nodes", + new Endpoint( + "/_cat/nodes", + nodesSchema, + Set.of(), + (client, spec) -> client.catNodes(spec.getArgs()))); + + // /_cat/cluster_manager — single row identifying the elected cluster manager node. + LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); + clusterManagerSchema.put("id", STRING); + clusterManagerSchema.put("host", STRING); + clusterManagerSchema.put("ip", STRING); + clusterManagerSchema.put("node", STRING); + m.put( + "/_cat/cluster_manager", + new Endpoint( + "/_cat/cluster_manager", + clusterManagerSchema, + Set.of(), + (client, spec) -> client.catClusterManager(spec.getArgs()))); + + // /_cat/plugins — one row per installed plugin per node (read-only monitor action). + LinkedHashMap pluginsSchema = new LinkedHashMap<>(); + pluginsSchema.put("name", STRING); + pluginsSchema.put("component", STRING); + pluginsSchema.put("version", STRING); + m.put( + "/_cat/plugins", + new Endpoint( + "/_cat/plugins", + pluginsSchema, + Set.of(), + (client, spec) -> client.catPlugins(spec.getArgs()))); + + // /_cat/shards — one row per shard (read-only monitor action). + LinkedHashMap shardsSchema = new LinkedHashMap<>(); + shardsSchema.put("index", STRING); + shardsSchema.put("shard", INTEGER); + shardsSchema.put("prirep", STRING); + shardsSchema.put("state", STRING); + shardsSchema.put("node", STRING); + m.put( + "/_cat/shards", + new Endpoint( + "/_cat/shards", + shardsSchema, + Set.of(), + (client, spec) -> client.catShards(spec.getArgs()))); + + // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). + LinkedHashMap stateSchema = new LinkedHashMap<>(); + stateSchema.put("cluster_name", STRING); + stateSchema.put("state_uuid", STRING); + stateSchema.put("version", LONG); + stateSchema.put("cluster_manager_node", STRING); + m.put( + "/_cluster/state", + new Endpoint( + "/_cluster/state", + stateSchema, + Set.of(), + (client, spec) -> List.of(client.clusterState(spec.getArgs())))); + + // /_cluster/settings — one row per configured setting (persistent/transient tier). + LinkedHashMap settingsSchema = new LinkedHashMap<>(); + settingsSchema.put("setting", STRING); + settingsSchema.put("value", STRING); + settingsSchema.put("tier", STRING); + m.put( + "/_cluster/settings", + new Endpoint( + "/_cluster/settings", + settingsSchema, + Set.of(), + (client, spec) -> client.clusterSettings(spec.getArgs()))); + + // /_resolve/index — one row per resolved index/alias/data_stream name. + LinkedHashMap resolveSchema = new LinkedHashMap<>(); + resolveSchema.put("name", STRING); + resolveSchema.put("type", STRING); + m.put( + "/_resolve/index", + new Endpoint( + "/_resolve/index", + resolveSchema, + Set.of("expand_wildcards"), + (client, spec) -> client.resolveIndex(spec.getArgs()))); + + return m; + } + + /** + * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public static Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Supported read-only endpoints: " + + REGISTRY.keySet()); + } + Endpoint endpoint = REGISTRY.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " + + REGISTRY.keySet()); + } + return endpoint; + } + + /** Validate that every supplied query arg is accepted by the endpoint. */ + public static void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + for (String arg : spec.getArgs().keySet()) { + if (!endpoint.getAllowedArgs().contains(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + endpoint.getAllowedArgs()); + } + validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } + + // Allowed value domains for the get-args that are applied server-side. Keys are validated against + // the per-endpoint allow-list above; values are validated here so a user-supplied value is never + // passed unchecked into an admin transport request. + private static final Map> ARG_VALUE_DOMAINS = + Map.of( + "local", Set.of("true", "false"), + "health", Set.of("green", "yellow", "red")); + + private static final Set EXPAND_WILDCARDS_VALUES = + Set.of("open", "closed", "hidden", "none", "all"); + + /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ + private static void validateArgValue(String endpoint, String arg, String value) { + Set domain = ARG_VALUE_DOMAINS.get(arg); + if (domain != null) { + if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + } else if ("expand_wildcards".equals(arg)) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { + if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { + throw new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [expand_wildcards] has an unsupported value [" + + value + + "]. Allowed values: " + + EXPAND_WILDCARDS_VALUES); + } + } + } + } + + private static ExprValue coerce(String column, ExprType type, Object value) { + if (value == null) { + return ExprNullValue.of(); + } + try { + if (type == INTEGER) { + return integerValue(toNumber(value).intValue()); + } + if (type == LONG) { + return longValue(toNumber(value).longValue()); + } + if (type == DOUBLE) { + return doubleValue(toNumber(value).doubleValue()); + } + if (type == BOOLEAN) { + return booleanValue(toBoolean(value)); + } + } catch (IllegalArgumentException | ClassCastException e) { + // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint + // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, + // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely + // unexpected faults (NPE, etc.) are left to propagate. + throw new IllegalArgumentException( + "rest endpoint value for column [" + + column + + "] could not be coerced to " + + type + + ": [" + + value + + "]"); + } + return stringValue(String.valueOf(value)); + } + + /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ + private static Number toNumber(Object value) { + if (value instanceof Number n) { + return n; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new NumberFormatException("empty string"); + } + if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { + return Double.parseDouble(s); + } + return Long.parseLong(s); + } + + /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ + private static boolean toBoolean(Object value) { + if (value instanceof Boolean b) { + return b; + } + String s = String.valueOf(value).trim(); + if (s.isEmpty()) { + throw new IllegalArgumentException("empty string is not a boolean"); + } + if (s.equalsIgnoreCase("true")) { + return true; + } + if (s.equalsIgnoreCase("false")) { + return false; + } + throw new IllegalArgumentException("not a boolean: " + value); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..868dabdd64e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Dispatches an allow-listed, read-only management endpoint through the transport node client under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + private final boolean redact; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this(client, endpoint, spec, false); + } + + public RestRequest( + OpenSearchClient client, + RestEndpointRegistry.Endpoint endpoint, + RestSpec spec, + boolean redact) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + this.redact = redact; + } + + @Override + public List search() { + List rows = endpoint.toRows(client, spec, redact); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java new file mode 100644 index 00000000000..fd674dccde4 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import java.util.regex.Pattern; + +/** + * Masks network identifiers in rest command cell values. Enabled per deployment via {@code + * plugins.ppl.rest.redaction.enabled}; off by default. + */ +public final class RestResponseRedactor { + + private RestResponseRedactor() {} + + private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; + private static final Pattern IPV4 = + Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); + private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); + private static final Pattern EC2_HOST = + Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); + private static final Pattern IPV6 = + Pattern.compile( + "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" + + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", + Pattern.CASE_INSENSITIVE); + private static final Pattern AZ_NAME = + Pattern.compile( + "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", + Pattern.CASE_INSENSITIVE); + + private record Mask(Pattern pattern, String replacement) {} + + private static final List MASKS = + List.of( + new Mask(IPV4, "x.x.x.x"), + new Mask(INET, "inet[/x.x.x.x:y]"), + new Mask(EC2_HOST, ""), + new Mask(IPV6, "x.x.x.x"), + new Mask(AZ_NAME, "xx-xxxxx-xx")); + + /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ + public static String redact(String text) { + if (text == null || text.isEmpty()) { + return text; + } + String out = text; + for (Mask mask : MASKS) { + out = mask.pattern().matcher(out).replaceAll(mask.replacement()); + } + return out; + } + + /** Mask availability-zone names only. */ + public static String maskAvailabilityZone(String text) { + if (text == null || text.isEmpty()) { + return text; + } + return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java new file mode 100644 index 00000000000..d425c361e6a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import org.opensearch.common.settings.SettingsFilter; + +/** + * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' + * /_cluster/settings'} fetcher. + * + *

    The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with + * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the + * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads + * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the + * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction + * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. + * + *

    Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in + * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link + * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code + * getRestHandlers} fires lets the fetcher read the same instance without going back through the + * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. + */ +public final class RestSettingsFilterHolder { + + private static volatile SettingsFilter settingsFilter; + + private RestSettingsFilterHolder() {} + + public static void set(SettingsFilter instance) { + settingsFilter = instance; + } + + public static SettingsFilter get() { + return settingsFilter; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index b0c92dce8f9..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -26,17 +26,22 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + catalogTable, + schema); } @Override @@ -60,7 +65,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -68,10 +73,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..dff9b47265a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java new file mode 100644 index 00000000000..33be6cc8482 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.client; + +import static java.util.stream.Collectors.toSet; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using + * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. + */ +class OpenSearchNodeClientClusterSettingsFilterTest { + + @AfterEach + void clearHolder() { + RestSettingsFilterHolder.set(null); + } + + private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) + .thenReturn(stateResp); + when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); + when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); + return new OpenSearchNodeClient(nodeClient); + } + + @Test + void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("plugins.secret.token", "supersecret") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Publish a filter that redacts the secret key, exactly as the native endpoint would. + RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); + + List> rows = client.clusterSettings(Map.of()); + Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); + assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); + } + + @Test + void clusterSettingsRedactsByGlobPattern() { + Settings persistent = + Settings.builder() + .put("cluster.routing.allocation.enable", "all") + .put("s3.client.default.secret_key", "AKIAEXAMPLE") + .build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); + + Set keys = + client.clusterSettings(Map.of()).stream() + .map(r -> (String) r.get("setting")) + .collect(toSet()); + + assertTrue(keys.contains("cluster.routing.allocation.enable")); + assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); + } + + @Test + void clusterSettingsFailsClosedWhenNoFilterPublished() { + Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); + OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); + + // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw + // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter + // before any query, so this path is unreachable in cluster. + assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..0c570098924 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +77,15 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..102ec4da8f7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,15 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -20,8 +24,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -52,6 +57,68 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); + } + + @Test + public void getRestTableAllowedByWildcard() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + Table table = + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); + assertTrue(table instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cat/nodes")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java new file mode 100644 index 00000000000..8676373d2a6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, + * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) + * path. + */ +@ExtendWith(MockitoExtension.class) +class RestCatalogSourceTest { + + @Mock private OpenSearchClient client; + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + Map fieldTypes = source.getFieldTypes(); + assertThat(fieldTypes, hasEntry("status", STRING)); + assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); + } + + @Test + void isScannable() { + assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); + } + + @Test + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); + } + + @Test + void restRequestShapesResponseRows() { + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestCatalogSource source = new RestCatalogSource(client, healthSpec()); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void countTruncatesRows() { + Map idx1 = new LinkedHashMap<>(); + idx1.put("index", "a"); + Map idx2 = new LinkedHashMap<>(); + idx2.put("index", "b"); + when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); + + RestCatalogSource source = + new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..c5a978bb495 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,290 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +@ExtendWith(MockitoExtension.class) +class RestEndpointRegistryTest { + + @Mock private OpenSearchClient client; + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("status")); + assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint is simply absent from the registry and is refused here. + assertThrows( + IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); + assertThrows( + IllegalArgumentException.class, + () -> RestEndpointRegistry.resolve("/services/server/info")); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + RestEndpointRegistry.validate(spec); // no throw + } + + @Test + void catEndpointsRedactAddressesWhenRedactionEnabled() { + Map node = new LinkedHashMap<>(); + node.put("name", "ip-10-0-0-7"); + node.put("ip", "10.0.0.7"); + node.put("node_role", "dir"); + node.put("heap_percent", 44); + node.put("ram_percent", 95); + node.put("cpu", 2); + when(client.catNodes(any())).thenReturn(List.of(node)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); + RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("", redacted.get("name").stringValue()); + assertEquals(44, redacted.get("heap_percent").integerValue()); + + Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); + assertEquals("10.0.0.7", plain.get("ip").stringValue()); + assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); + } + + @Test + void catClusterManagerRedactsHostAndIp() { + Map row = new LinkedHashMap<>(); + row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); + row.put("host", "10.0.0.7"); + row.put("ip", "10.0.0.7"); + row.put("node", "71d03b567bb755839a73d437b2b066d4"); + when(client.catClusterManager(any())).thenReturn(List.of(row)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); + RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); + + Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("x.x.x.x", redacted.get("host").stringValue()); + assertEquals("x.x.x.x", redacted.get("ip").stringValue()); + assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); + assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); + } + + @Test + void nonCatEndpointNotRedactedEvenWhenEnabled() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "10.0.0.7"); + health.put("status", "green"); + health.put("number_of_nodes", 3); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); + } + + @Test + void clusterSettingsMasksAvailabilityZoneInValue() { + Map setting = new LinkedHashMap<>(); + setting.put("setting", "cluster.routing.allocation.awareness.attributes"); + setting.put("value", "zone:us-east-1a"); + setting.put("tier", "persistent"); + when(client.clusterSettings(any())).thenReturn(List.of(setting)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); + RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); + + Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); + assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); + assertEquals( + "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); + assertEquals("persistent", row.get("tier").stringValue()); + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsDroppedFlatSettingsArg() { + // flat_settings was dropped (redundant: settings are already flattened to dotted keys). + RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + } + + @Test + void validateAcceptsValidArgValues() { + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); + RestEndpointRegistry.validate( + new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException health = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); + assertTrue(health.getMessage().contains("unsupported value")); + + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + + assertThrows( + IllegalArgumentException.class, + () -> + RestEndpointRegistry.validate( + new RestSpec( + "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); + RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void coerceParsesNumericStringValues() { + // The cat JSON API returns numeric columns as strings; coerce must parse them. + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "3"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + } + + @Test + void coerceThrowsClearErrorOnUncoercibleValue() { + // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), + // not a raw ClassCastException / NumberFormatException (HTTP 500). + Map health = new LinkedHashMap<>(); + health.put("status", "green"); + health.put("number_of_nodes", "not-a-number"); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); + assertTrue(ex.getMessage().contains("number_of_nodes")); + assertTrue(ex.getMessage().contains("not-a-number")); + } + + @Test + void clusterHealthRowsAreShapedToFixedSchema() { + Map health = new LinkedHashMap<>(); + health.put("cluster_name", "test-cluster"); + health.put("status", "green"); + health.put("number_of_nodes", 1); + when(client.clusterHealth(any())).thenReturn(health); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); + List rows = + endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); + assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); + // a declared column the action did not return becomes null, never absent. + assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); + } + + @Test + void catIndicesRowsAreShapedToFixedSchema() { + Map idx = new LinkedHashMap<>(); + idx.put("index", "books"); + idx.put("health", "yellow"); + idx.put("pri", 1); + idx.put("rep", 1); + when(client.catIndices(any())).thenReturn(List.of(idx)); + + RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); + List rows = + endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); + + assertEquals(1, rows.size()); + assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); + assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java new file mode 100644 index 00000000000..3b038306da5 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class RestResponseRedactorTest { + + @Test + void masksIpv4() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); + assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); + assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); + } + + @Test + void doesNotMaskInvalidOrPartialIpv4() { + assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); + assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); + assertEquals("44", RestResponseRedactor.redact("44")); + } + + @Test + void masksEc2HostName() { + assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); + assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); + assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); + assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); + } + + @Test + void masksFullIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); + } + + @Test + void masksCompressedIpv6() { + assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); + assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); + } + + @Test + void masksInetAddress() { + assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); + } + + @Test + void masksAvailabilityZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); + // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). + assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); + assertEquals( + "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); + } + + @Test + void maskAvailabilityZoneMasksOnlyZones() { + assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); + assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); + assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); + } + + @Test + void leavesNonAddressesIntact() { + assertEquals( + "e4e136ea81e27370ff73cf753ba22d39", + RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); + assertEquals( + "data,ingest,remote_cluster_client", + RestResponseRedactor.redact("data,ingest,remote_cluster_client")); + assertEquals( + "x.x.x.x 44 95 imr - e4e136ea", + RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); + } + + @Test + void handlesNullAndEmpty() { + assertEquals(null, RestResponseRedactor.redact(null)); + assertEquals("", RestResponseRedactor.redact("")); + assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); + assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index e1278aa75e8..d9437fded5b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -198,6 +198,10 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); + // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can + // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. + org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); + return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index f8214096e41..5685d539541 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -107,13 +107,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..0cf87f0604e 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,6 +178,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 4f712042ed0..f9d67bd46ff 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 14655542062..354ffcd425d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -104,6 +105,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -208,6 +210,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -1796,4 +1808,6 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 88a9b9e1793..2f9168383b8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -110,6 +110,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -143,6 +144,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -254,6 +256,37 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
    + * Leading command that reads an allow-listed, read-only in-cluster management endpoint + * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved + * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves + * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE + * resolves to a system index. Allow-list/authorization enforcement happens at source-table + * construction in the storage engine (it owns the transport actions and per-endpoint schemas). + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index eb99a4b4381..7d849517951 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -96,6 +96,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -123,6 +124,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -166,6 +168,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..da9424c9c8f --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

    The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = + (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(2, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cat/indices", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d57ca8a69bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,4 +1939,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 9ea21684edb..d11348457c3 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,6 +48,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); From ee529142b82bfbb5721e6151c340f17171634ea5 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:30:32 -0700 Subject: [PATCH 35/41] Fix flaky TPC-H Q15 floating-point assertion (#5629) Signed-off-by: Kai Huang --- .../java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java index d83140e0dd5..0642b8f5651 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java @@ -314,7 +314,7 @@ public void testQ15() throws IOException { schema("total_revenue", "double")); verifyDataRows( actual, - rows(10, "Supplier#000000010", "Saygah3gYWMp72i PY", "34-852-489-8585", 797313.3838)); + closeTo(10, "Supplier#000000010", "Saygah3gYWMp72i PY", "34-852-489-8585", 797313.3838)); } @Test From 0821b290def72c2072c8ab626d1bdfa39fc861bb Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Thu, 16 Jul 2026 16:09:47 +0800 Subject: [PATCH 36/41] Support PPL foreach command (#5613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement PPL foreach command Signed-off-by: Songkan Tang * Complete PPL foreach collection modes Signed-off-by: Songkan Tang * Extract foreach planner constants Signed-off-by: Songkan Tang * Complete foreach collection type inference Signed-off-by: Songkan Tang * Refactor foreach command: reuse type system, extract planner, remove special cases - Extract ~400 lines of foreach planning from CalciteRelNodeVisitor into a dedicated ForeachPlanner class. - Replace string-based type plumbing (SqlTypeName names smuggled through ForeachBinding and reparsed at runtime) with RelDataType carried on the binding; foreach_pair_item now takes (pair, index) only and the call is built with the plan-time type directly. - Collapse ForeachBindingType {PAIR_ITEM, PAIR_ITER, PAIR_EXTRA, LAMBDA} into a single PAIR_SLOT; drop the unused LAMBDA variant. - Replace the AST arithmetic-scan heuristic for json_array element types with operand type inspection (json_array call) / plan-time JSON parse (string literal), using SqlTypeFamily. - Remove the reduce-arg0 special case in CalciteRexNodeVisitor: collection bindings are now staged on CalcitePlanContext and only activate inside cloned lambda contexts, so non-lambda reduce args resolve against the row naturally. - Foreach.Mode enum replaces stringly-typed mode comparisons; AstBuilder parses options/targets in a single pass without double-visiting expressions. - Delete dead code: FOREACH_TRANSFORM_* constants, FOREACH_PAIR_ITEM registry entry, validateHomogeneousJsonArrayArguments wrapper, duplicate collectionExpressionForMode branches. Signed-off-by: Songkan Tang * Narrow foreachTarget grammar to fix ANTLR error-recovery regression foreachTarget's logicalExpression alternative let ANTLR consider the foreach rule during error recovery of unrelated malformed queries, changing expected-token sets in syntax error messages (caught by UnifiedRelevanceSearchTest#testMatchMissingArguments: 'match(' with a missing field started reporting 26 candidate tokens instead of the expected ','). foreach only ever needs a function call (json_array), a string literal (JSON array text), or a field/wildcard as its target, so accept exactly those instead of the whole expression grammar. Signed-off-by: Songkan Tang * Assert full logical plans in foreach collection-mode unit tests Replace assertNotNull with verifyLogical so the tests pin the generated reduce/foreach_pair_collection/foreach_pair_item structure, placeholder slot indices, and json_array element-type inference. Signed-off-by: Songkan Tang * Restore usage-based element type inference for field-backed JSON arrays The refactor's jsonElementType defaulted opaque expressions (an index field holding JSON text - the primary Splunk use of json_array mode) to VARCHAR, which broke numeric aggregation over such fields: reduce resolved to [ARRAY, DOUBLE, DOUBLE] and failed. Splunk returns 60 for a field holding "[10,20,30]" summed via foreach; the original branch matched that by inferring element type from usage. Bring back the usage scan for the opaque case only: if the item placeholder is consumed by arithmetic the elements are DOUBLE, else VARCHAR. json_array() calls and string literals keep the plan-time content inspection. New coverage: - CalcitePPLForeachTest: plan assertions for field-backed json_array with numeric and string usage - ForeachFieldJsonIT: field holding "[10,20,30]" sums to 60 (Splunk parity), string-content field concats, and native OpenSearch array fields (long field holding [1,2,3]) documented as rejected - the mapping types them as scalar BIGINT at plan time Signed-off-by: Songkan Tang * Cover nested-field multivalue iteration in ForeachFieldJsonIT Nested-typed fields map to ARRAY at plan time so multivalue mode accepts and iterates them (verified: counting elements of a 2-element nested array returns 2). Pins the third field-backed collection shape alongside JSON-text fields (supported) and native scalar-mapped arrays (rejected). Signed-off-by: Songkan Tang * Pin cross-feed behavior of foreach collection modes in IT Splunk silently no-ops when a mode is fed the wrong collection shape (verified against Splunk 10.4.0). Our behavior intentionally differs and these tests document it: - json_array mode fed a real array iterates it (total=6) - more permissive than Splunk's no-op - multivalue mode fed a JSON-text field fails at plan time with SemanticCheckException - louder than Splunk's no-op Signed-off-by: Songkan Tang * Add user documentation for PPL foreach command Follows the structure of other command docs (timewrap, eval): syntax, parameters, placeholder table, notes on collection-mode accumulator semantics and type inference, and six runnable examples registered in docs/category.json for doctest. All example outputs verified against a live docTestCluster loaded with the doctest accounts dataset. Signed-off-by: Songkan Tang * Complete new-command checklist: v2 Analyzer guard and query anonymization - Analyzer.visitForeach throws the standard only-for-Calcite UnsupportedOperationException instead of leaving the v2 path undefined - PPLQueryDataAnonymizer.visitForeach renders the command with mode, masked option values, masked targets (collection targets can embed literals such as JSON array strings), and anonymized eval clauses; ForeachPlaceholder masks like a column reference UT: AnalyzerTest legacy-engine rejection; anonymizer coverage for multifield, multivalue-with-options, and json_array-with-literal-target (122/122 anonymizer tests pass) Signed-off-by: Songkan Tang * Use Locale.ROOT in testNoMv explain-plan case folding CI runs with randomized locales; under az-Cyrl the default-locale toLowerCase turns ARRAY_JOIN into array_joın (dotless i), so the contains("array_join") assertions fail. Reproduced locally with -Dtests.seed=FAD995E7580F1AF -Dtests.locale=az-Cyrl and verified fixed. Pre-existing bug in these tests (added by the timewrap PR), surfaced on this PR's CI run by locale randomization. Signed-off-by: Songkan Tang * Address foreach review feedback and Splunk semantics Signed-off-by: Songkan Tang * Cover foreach JSON numeric inference in IT Signed-off-by: Songkan Tang --------- Signed-off-by: Songkan Tang --- .../org/opensearch/sql/analysis/Analyzer.java | 6 + .../sql/ast/AbstractNodeVisitor.java | 10 + .../ast/expression/ForeachPlaceholder.java | 33 + .../org/opensearch/sql/ast/tree/Foreach.java | 77 +++ .../sql/calcite/CalcitePlanContext.java | 70 ++ .../sql/calcite/CalciteRelNodeVisitor.java | 11 +- .../sql/calcite/CalciteRexNodeVisitor.java | 101 +++ .../sql/calcite/ForeachPlanner.java | 630 ++++++++++++++++++ .../function/BuiltinFunctionName.java | 3 + .../ForeachPairCollectionFunctionImpl.java | 85 +++ .../ForeachPairItemFunctionImpl.java | 63 ++ .../ForeachStateFunctionImpl.java | 60 ++ .../function/CollectionUDF/LambdaUtils.java | 15 + .../function/PPLBuiltinOperators.java | 12 + .../expression/function/PPLFuncImpTable.java | 82 ++- .../jsonUDF/ForeachJsonArrayFunctionImpl.java | 110 +++ .../opensearch/sql/analysis/AnalyzerTest.java | 18 + .../ForeachFunctionImplTest.java | 49 ++ docs/category.json | 1 + docs/user/ppl/cmd/foreach.md | 191 ++++++ .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 20 +- .../remote/CalciteForeachCommandIT.java | 193 ++++++ .../calcite/remote/ForeachFieldJsonIT.java | 135 ++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 45 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 63 ++ .../sql/ppl/parser/AstExpressionBuilder.java | 12 + .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 33 + .../sql/ppl/antlr/PPLSyntaxParserTest.java | 51 ++ .../ppl/calcite/CalcitePPLForeachTest.java | 328 +++++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 26 + 32 files changed, 2527 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java create mode 100644 core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java create mode 100644 docs/user/ppl/cmd/foreach.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteForeachCommandIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLForeachTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index d7b8ceb1bff..8c1dbee006f 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -74,6 +74,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Join; @@ -571,6 +572,11 @@ public LogicalPlan visitGraphLookup(GraphLookup node, AnalysisContext context) { throw getOnlyForCalciteException("graphlookup"); } + @Override + public LogicalPlan visitForeach(Foreach node, AnalysisContext context) { + throw getOnlyForCalciteException("foreach"); + } + /** Build {@link ParseExpression} to context and skip to child nodes. */ @Override public LogicalPlan visitParse(Parse node, AnalysisContext context) { diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 6d8415fd7ea..a32354883bf 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -18,6 +18,7 @@ import org.opensearch.sql.ast.expression.Compare; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; @@ -62,6 +63,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Join; @@ -174,6 +176,10 @@ public T visitProject(Project node, C context) { return visitChildren(node, context); } + public T visitForeach(Foreach node, C context) { + return visitChildren(node, context); + } + public T visitAggregation(Aggregation node, C context) { return visitChildren(node, context); } @@ -254,6 +260,10 @@ public T visitField(Field node, C context) { return visitChildren(node, context); } + public T visitForeachPlaceholder(ForeachPlaceholder node, C context) { + return visitChildren(node, context); + } + public T visitQualifiedName(QualifiedName node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java b/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java new file mode 100644 index 00000000000..9362a70d22f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ForeachPlaceholder.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** Placeholder used by the PPL foreach command, such as {@code <>}. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class ForeachPlaceholder extends UnresolvedExpression { + private final String name; + + @Override + public List getChild() { + return ImmutableList.of(); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitForeachPlaceholder(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java b/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java new file mode 100644 index 00000000000..e11ecf541cd --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Foreach.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** AST node representing the PPL foreach command. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Foreach extends UnresolvedPlan { + private final Mode mode; + private final Map options; + private final List fieldPatterns; + private final UnresolvedExpression collectionExpression; + private final List evalClauses; + private UnresolvedPlan child; + + @Override + public Foreach attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return child == null ? ImmutableList.of() : ImmutableList.of(child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitForeach(this, context); + } + + /** Iteration mode of the foreach command. */ + public enum Mode { + MULTIFIELD, + MULTIVALUE, + JSON_ARRAY, + AUTO_COLLECTIONS; + + public static Mode of(String name) { + try { + return valueOf(name.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("foreach mode [" + name + "] is not supported"); + } + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + @Getter + @ToString + @EqualsAndHashCode + @RequiredArgsConstructor + public static class ForeachEvalClause { + private final String targetTemplate; + private final UnresolvedExpression expression; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 3f81cdbae4a..162a4895805 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -17,10 +17,12 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; +import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -78,6 +80,26 @@ public class CalcitePlanContext { @Getter public Map rexLambdaRefMap; + /** + * Foreach placeholder bindings active in this context, keyed by upper-cased placeholder name. + * Multifield mode activates bindings directly (placeholders resolve against the current row); + * collection modes stage bindings via {@link #stageForeachLambdaBindings} instead, so they only + * become active inside the lambda context cloned for the generated {@code reduce} call. + */ + @Getter private Map foreachBindings = new HashMap<>(); + + /** Bare identifiers enabled by explicit options such as {@code itemstr=ITEM}. */ + @Getter private Map foreachIdentifierBindings = new HashMap<>(); + + /** Bindings that become active in lambda contexts cloned from this one. */ + private Map stagedForeachLambdaBindings = new HashMap<>(); + + /** Bare identifier bindings staged for a generated foreach lambda. */ + private Map stagedForeachIdentifierBindings = new HashMap<>(); + + /** Expressions computed by earlier assignments in the same foreach eval iteration. */ + @Getter private Map foreachComputedBindings = new HashMap<>(); + /** * Maps AggregateFunction AST nodes to their output field index for HAVING/post-aggregate * resolution. @@ -125,6 +147,12 @@ private CalcitePlanContext(CalcitePlanContext parent) { this.rexLambdaRefMap = new HashMap<>(); // New map for lambda variables this.capturedVariables = new ArrayList<>(); // New list for captured variables this.inLambdaContext = true; // Mark that we're inside a lambda + // Active bindings carry over; staged bindings become active inside the lambda. + this.foreachBindings = new HashMap<>(parent.foreachBindings); + this.foreachBindings.putAll(parent.stagedForeachLambdaBindings); + this.foreachIdentifierBindings = new HashMap<>(parent.foreachIdentifierBindings); + this.foreachIdentifierBindings.putAll(parent.stagedForeachIdentifierBindings); + this.foreachComputedBindings = new HashMap<>(parent.foreachComputedBindings); } public RexNode resolveJoinCondition( @@ -203,6 +231,48 @@ public static void clearTimewrapSignals() { timewrapSeries.set(null); } + public void pushForeachBindings( + Map bindings, Map identifierBindings) { + foreachBindings = new HashMap<>(bindings); + foreachIdentifierBindings = new HashMap<>(identifierBindings); + } + + public void stageForeachLambdaBindings( + Map bindings, Map identifierBindings) { + stagedForeachLambdaBindings = new HashMap<>(bindings); + stagedForeachIdentifierBindings = new HashMap<>(identifierBindings); + } + + public void putForeachComputedBinding(String name, RexNode expression) { + foreachComputedBindings.put(name.toUpperCase(java.util.Locale.ROOT), expression); + } + + public void clearForeachBindings() { + foreachBindings.clear(); + foreachIdentifierBindings.clear(); + stagedForeachLambdaBindings.clear(); + stagedForeachIdentifierBindings.clear(); + foreachComputedBindings.clear(); + } + + /** + * A foreach placeholder binding. {@code FIELD} resolves to the named row field, {@code LITERAL} + * to a string literal, and {@code PAIR_SLOT} to slot {@code pairIndex} (typed {@code pairType}) + * of the named lambda pair variable. + */ + public record ForeachBinding( + String value, ForeachBindingType type, int pairIndex, @Nullable RelDataType pairType) { + public ForeachBinding(String value, ForeachBindingType type) { + this(value, type, -1, null); + } + } + + public enum ForeachBindingType { + FIELD, + LITERAL, + PAIR_SLOT + } + public void putRexLambdaRefMap(Map candidateMap) { this.rexLambdaRefMap.putAll(candidateMap); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 13ccdf15197..0df3c571c6b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -130,6 +130,7 @@ import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; +import org.opensearch.sql.ast.tree.Foreach; import org.opensearch.sql.ast.tree.GraphLookup; import org.opensearch.sql.ast.tree.GraphLookup.Direction; import org.opensearch.sql.ast.tree.Head; @@ -214,12 +215,14 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor newFields, List newNames, CalcitePlanContext context) { Set originalFieldNameSet = new HashSet<>(context.relBuilder.peek().getRowType().getFieldNames()); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 849b615f970..1bf1e217b51 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -19,6 +19,8 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; @@ -53,6 +55,7 @@ import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; import org.opensearch.sql.ast.expression.EqualTo; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; @@ -79,6 +82,8 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.ast.tree.Sort.SortOrder; import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBinding; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBindingType; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; @@ -92,10 +97,13 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.CoercionUtils; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.PPLFuncImpTable; @RequiredArgsConstructor public class CalciteRexNodeVisitor extends AbstractNodeVisitor { + private static final Pattern FOREACH_TEMPLATE = Pattern.compile("<<([A-Za-z0-9_]+)>>"); + private final CalciteRelNodeVisitor planVisitor; public RexNode analyze(UnresolvedExpression unresolved, CalcitePlanContext context) { @@ -134,6 +142,10 @@ public RexNode visitLiteral(Literal node, CalcitePlanContext context) { case NULL: return rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.NULL)); case STRING: + RexNode foreachTemplate = foreachTemplateLiteral(value.toString(), context); + if (foreachTemplate != null) { + return foreachTemplate; + } if (value.toString().length() == 1) { // To align Spark/PostgreSQL, Char(1) is useful, such as cast('1' to boolean) should // return true @@ -399,9 +411,98 @@ private boolean isBooleanLiteral(RexNode node) { /** Resolve qualified name. Note, the name should be case-sensitive. */ @Override public RexNode visitQualifiedName(QualifiedName node, CalcitePlanContext context) { + String name = node.toString(); + RexNode computed = context.getForeachComputedBindings().get(name.toUpperCase(Locale.ROOT)); + if (computed != null) { + return computed; + } + ForeachBinding binding = + context.getForeachIdentifierBindings().get(name.toUpperCase(Locale.ROOT)); + if (binding != null) { + return foreachBindingToRexNode(node.toString(), binding, context); + } return QualifiedNameResolver.resolve(node, context); } + @Override + public RexNode visitForeachPlaceholder(ForeachPlaceholder node, CalcitePlanContext context) { + ForeachBinding binding = foreachBinding(node.getName(), context); + if (binding == null) { + throw new SemanticCheckException("Unresolved foreach placeholder <<" + node.getName() + ">>"); + } + return foreachBindingToRexNode(node.getName(), binding, context); + } + + private ForeachBinding foreachBinding(String name, CalcitePlanContext context) { + return context.getForeachBindings().get(name.toUpperCase(Locale.ROOT)); + } + + private RexNode foreachTemplateLiteral(String value, CalcitePlanContext context) { + Matcher matcher = FOREACH_TEMPLATE.matcher(value); + List parts = new ArrayList<>(); + int start = 0; + boolean replaced = false; + while (matcher.find()) { + ForeachBinding binding = foreachBinding(matcher.group(1), context); + if (binding == null) { + continue; + } + if (matcher.start() > start) { + parts.add(context.rexBuilder.makeLiteral(value.substring(start, matcher.start()))); + } + if (binding.type() == ForeachBindingType.PAIR_SLOT) { + RelDataType varchar = + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + parts.add( + context.rexBuilder.makeCast( + varchar, foreachBindingToRexNode(matcher.group(1), binding, context), true, true)); + } else { + parts.add(context.rexBuilder.makeLiteral(binding.value())); + } + start = matcher.end(); + replaced = true; + } + if (!replaced) { + return null; + } + if (start < value.length()) { + parts.add(context.rexBuilder.makeLiteral(value.substring(start))); + } + if (parts.isEmpty()) { + return context.rexBuilder.makeLiteral(""); + } + RexNode result = parts.getFirst(); + for (int i = 1; i < parts.size(); i++) { + result = context.rexBuilder.makeCall(SqlStdOperatorTable.CONCAT, result, parts.get(i)); + } + return result; + } + + private RexNode foreachBindingToRexNode( + String name, ForeachBinding binding, CalcitePlanContext context) { + switch (binding.type()) { + case FIELD: + return context.relBuilder.field(binding.value()); + case PAIR_SLOT: + RexLambdaRef pair = context.getRexLambdaRefMap().get(binding.value()); + if (pair == null) { + throw new SemanticCheckException("Unresolved foreach lambda placeholder " + name); + } + // The slot's type is known at plan time, so assign it directly to the opaque extraction + // call. LambdaUtils' re-inference preserves it (a CAST would not survive the enumerable + // backend for complex types like arrays). + return context.rexBuilder.makeCall( + binding.pairType(), + PPLBuiltinOperators.FOREACH_PAIR_ITEM, + List.of( + pair, + context.rexBuilder.makeExactLiteral(BigDecimal.valueOf(binding.pairIndex())))); + case LITERAL: + default: + return context.rexBuilder.makeLiteral(binding.value()); + } + } + @Override public RexNode visitAlias(Alias node, CalcitePlanContext context) { RexNode expr = analyze(node.getDelegated(), context); diff --git a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java new file mode 100644 index 00000000000..d9ef71b2220 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java @@ -0,0 +1,630 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; + +import com.google.gson.JsonSyntaxException; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.ArraySqlType; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Field; +import org.opensearch.sql.ast.expression.ForeachPlaceholder; +import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.LambdaFunction; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.tree.Foreach; +import org.opensearch.sql.ast.tree.Foreach.ForeachEvalClause; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBinding; +import org.opensearch.sql.calcite.CalcitePlanContext.ForeachBindingType; +import org.opensearch.sql.calcite.utils.WildcardUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; +import org.opensearch.sql.expression.function.PPLFuncImpTable; + +/** + * Plans the PPL {@code foreach} command. + * + *

    Multifield mode expands each eval clause once per matching field, binding {@code <>} + * (and match placeholders) so the clause expression resolves against the current field. + * + *

    Collection modes (multivalue / json_array / auto_collections) rewrite each eval clause into a + * {@code reduce} over the collection: elements are packed into internal pairs {@code [item, iter, + * captured-field...]} so the lambda can reference the loop item, the loop index, and any row fields + * the clause mentions. Placeholder bindings are staged on the context and only become active inside + * the lambda; the reduce call's other arguments resolve against the row as usual. + */ +class ForeachPlanner { + + private static final String OPTION_FIELDSTR = "fieldstr"; + private static final String OPTION_MATCHSTR = "matchstr"; + private static final String OPTION_MATCHSEG = "matchseg"; + private static final String OPTION_ITEMSTR = "itemstr"; + private static final String OPTION_ITERSTR = "iterstr"; + private static final String PLACEHOLDER_FIELD = "FIELD"; + private static final String PLACEHOLDER_MATCHSTR = "MATCHSTR"; + private static final String PLACEHOLDER_MATCHSEG = "MATCHSEG"; + private static final String PLACEHOLDER_ITEM = "ITEM"; + private static final String PLACEHOLDER_ITER = "ITER"; + + /** Name of the lambda variable holding the internal pair inside generated reduce calls. */ + private static final String PAIR_VAR = "__foreach_pair"; + + private static final String STATE_VAR = "__foreach_state"; + + private static final Set ARITHMETIC_OPERATORS = Set.of("+", "-", "*", "/", "%"); + + private final CalciteRelNodeVisitor relVisitor; + private final CalciteRexNodeVisitor rexVisitor; + + ForeachPlanner(CalciteRelNodeVisitor relVisitor, CalciteRexNodeVisitor rexVisitor) { + this.relVisitor = relVisitor; + this.rexVisitor = rexVisitor; + } + + RelNode plan(Foreach node, CalcitePlanContext context) { + return node.getMode() == Foreach.Mode.MULTIFIELD + ? planMultifield(node, context) + : planCollection(node, context); + } + + // ---------------------------------------------------------------------- multifield + + private RelNode planMultifield(Foreach node, CalcitePlanContext context) { + List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); + Set matchingFields = new LinkedHashSet<>(); + for (String pattern : node.getFieldPatterns()) { + matchingFields.addAll(WildcardUtils.expandWildcardPattern(pattern, currentFields)); + } + + for (String fieldName : matchingFields) { + ForeachBindings bindings = + multifieldBindings(node.getFieldPatterns(), fieldName, node.getOptions()); + context.pushForeachBindings(bindings.values(), bindings.identifiers()); + try { + for (ForeachEvalClause clause : node.getEvalClauses()) { + RexNode expr = rexVisitor.analyze(clause.getExpression(), context); + String alias = substituteTemplate(clause.getTargetTemplate(), bindings.values()); + relVisitor.projectPlusOverriding( + List.of(context.relBuilder.alias(expr, alias)), List.of(alias), context); + } + } finally { + context.clearForeachBindings(); + } + } + return context.relBuilder.peek(); + } + + private ForeachBindings multifieldBindings( + List patterns, String fieldName, Map options) { + Map bindings = new LinkedHashMap<>(); + Map identifiers = new LinkedHashMap<>(); + bindOption( + bindings, + identifiers, + new ForeachBinding(fieldName, ForeachBindingType.FIELD), + PLACEHOLDER_FIELD, + OPTION_FIELDSTR, + options); + List orderedPatterns = + Stream.concat( + patterns.stream() + .filter(pattern -> !WildcardUtils.containsWildcard(pattern)) + .filter(pattern -> WildcardUtils.matchesWildcardPattern(pattern, fieldName)), + patterns.stream().filter(WildcardUtils::containsWildcard)) + .toList(); + for (String pattern : orderedPatterns) { + List segments = wildcardSegments(pattern, fieldName); + if (segments == null) { + continue; + } + bindOption( + bindings, + identifiers, + new ForeachBinding(String.join("", segments), ForeachBindingType.LITERAL), + PLACEHOLDER_MATCHSTR, + OPTION_MATCHSTR, + options); + for (int i = 0; i < segments.size(); i++) { + String defaultName = PLACEHOLDER_MATCHSEG + (i + 1); + bindOption( + bindings, + identifiers, + new ForeachBinding(segments.get(i), ForeachBindingType.LITERAL), + defaultName, + OPTION_MATCHSEG + (i + 1), + options); + } + break; + } + return new ForeachBindings(bindings, identifiers); + } + + /** + * Returns the substrings of {@code fieldName} captured by the wildcards in {@code pattern}, an + * empty list if the pattern matches without wildcards, or null if it does not match. + */ + private List wildcardSegments(String pattern, String fieldName) { + if (!WildcardUtils.matchesWildcardPattern(pattern, fieldName)) { + return null; + } + if (!WildcardUtils.containsWildcard(pattern)) { + return List.of(); + } + Matcher matcher = + Pattern.compile(WildcardUtils.convertWildcardPatternToRegex(pattern)).matcher(fieldName); + if (!matcher.matches()) { + return null; + } + List segments = new ArrayList<>(); + for (int i = 1; i <= matcher.groupCount(); i++) { + segments.add(matcher.group(i)); + } + return segments; + } + + // ---------------------------------------------------------------------- collection modes + + private RelNode planCollection(Foreach node, CalcitePlanContext context) { + Foreach.Mode mode = node.getMode(); + UnresolvedExpression collection = node.getCollectionExpression(); + if (collection == null && mode == Foreach.Mode.AUTO_COLLECTIONS) { + collection = firstArrayField(context); + } + if (collection == null) { + throw new SemanticCheckException("foreach " + mode + " mode requires a field"); + } + RexNode rawCollection = rexVisitor.analyze(collection, context); + boolean nativeArray = rawCollection.getType() instanceof ArraySqlType; + if ((mode == Foreach.Mode.MULTIVALUE && !nativeArray) + || (mode == Foreach.Mode.JSON_ARRAY && nativeArray)) { + return context.relBuilder.peek(); + } + collection = asArrayExpression(collection, nativeArray, context, node); + RexNode collectionRex = rexVisitor.analyze(collection, context); + if (!(collectionRex.getType() instanceof ArraySqlType arrayType)) { + return context.relBuilder.peek(); + } + RelDataType itemType = arrayType.getComponentType(); + RelDataType iterType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + List targetAliases = + node.getEvalClauses().stream().map(ForeachEvalClause::getTargetTemplate).toList(); + List initialTargets = + targetAliases.stream() + .map(alias -> rexVisitor.analyze(new Field(new QualifiedName(alias)), context)) + .toList(); + List capturedFields = + capturedFields(node.getEvalClauses(), context, node.getOptions(), targetAliases); + + // Pack [item, iter, captured...] so the reduce lambda can reach everything through one var. + List pairArgs = new ArrayList<>(); + pairArgs.add(collection); + capturedFields.stream() + .map(field -> new Field(new QualifiedName(field.getName()))) + .forEach(pairArgs::add); + Function pairedCollection = + new Function( + BuiltinFunctionName.FOREACH_PAIR_COLLECTION.getName().getFunctionName(), pairArgs); + RexNode pairedCollectionRex = rexVisitor.analyze(pairedCollection, context); + + List targetTypes = initialTargets.stream().map(RexNode::getType).toList(); + List probedExpressions = + analyzeCollectionEval( + node, + context, + pairedCollectionRex, + state(initialTargets, context), + itemType, + iterType, + capturedFields, + targetAliases, + targetTypes); + targetTypes = probedExpressions.stream().map(RexNode::getType).toList(); + List castInitialTargets = new ArrayList<>(); + for (int i = 0; i < initialTargets.size(); i++) { + castInitialTargets.add( + context.rexBuilder.makeCast(targetTypes.get(i), initialTargets.get(i), true, true)); + } + RexNode initialState = state(castInitialTargets, context); + + ForeachBindings bindings = + collectionBindings(node, itemType, iterType, capturedFields, targetAliases, targetTypes); + context.stageForeachLambdaBindings(bindings.values(), bindings.identifiers()); + try { + CalcitePlanContext lambdaContext = + prepareStateLambdaContext(context, pairedCollectionRex, initialState); + List updatedTargets = analyzeClauses(node.getEvalClauses(), lambdaContext); + RexNode updatedState = state(updatedTargets, lambdaContext); + RexNode lambda = + context.rexBuilder.makeLambdaCall( + updatedState, + List.of( + lambdaContext.getRexLambdaRefMap().get(STATE_VAR), + lambdaContext.getRexLambdaRefMap().get(PAIR_VAR))); + RexNode reducedState = + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, + BuiltinFunctionName.REDUCE, + pairedCollectionRex, + initialState, + lambda); + List outputs = new ArrayList<>(); + for (int i = 0; i < targetAliases.size(); i++) { + RexNode value = stateSlot(reducedState, i, targetTypes.get(i), context); + outputs.add(context.relBuilder.alias(value, targetAliases.get(i))); + } + relVisitor.projectPlusOverriding(outputs, targetAliases, context); + } finally { + context.clearForeachBindings(); + } + return context.relBuilder.peek(); + } + + private List analyzeCollectionEval( + Foreach node, + CalcitePlanContext context, + RexNode pairedCollection, + RexNode initialState, + RelDataType itemType, + RelDataType iterType, + List capturedFields, + List targetAliases, + List targetTypes) { + ForeachBindings bindings = + collectionBindings(node, itemType, iterType, capturedFields, targetAliases, targetTypes); + context.stageForeachLambdaBindings(bindings.values(), bindings.identifiers()); + try { + return analyzeClauses( + node.getEvalClauses(), + prepareStateLambdaContext(context, pairedCollection, initialState)); + } finally { + context.clearForeachBindings(); + } + } + + private CalcitePlanContext prepareStateLambdaContext( + CalcitePlanContext context, RexNode pairedCollection, RexNode initialState) { + LambdaFunction template = + new LambdaFunction( + new Literal(0, DataType.INTEGER), + List.of(new QualifiedName(STATE_VAR), new QualifiedName(PAIR_VAR))); + return rexVisitor.prepareLambdaContext( + context, + template, + List.of(pairedCollection, initialState), + BuiltinFunctionName.REDUCE.getName().getFunctionName(), + initialState.getType()); + } + + private List analyzeClauses( + List clauses, CalcitePlanContext lambdaContext) { + List expressions = new ArrayList<>(); + for (ForeachEvalClause clause : clauses) { + RexNode expression = rexVisitor.analyze(clause.getExpression(), lambdaContext); + expressions.add(expression); + lambdaContext.putForeachComputedBinding(clause.getTargetTemplate(), expression); + } + return expressions; + } + + private RexNode state(List values, CalcitePlanContext context) { + return PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, BuiltinFunctionName.FOREACH_STATE, values.toArray(RexNode[]::new)); + } + + private RexNode stateSlot(RexNode state, int slot, RelDataType type, CalcitePlanContext context) { + return context.rexBuilder.makeCall( + type, + PPLBuiltinOperators.FOREACH_PAIR_ITEM, + List.of(state, context.rexBuilder.makeExactLiteral(BigDecimal.valueOf(slot)))); + } + + private ForeachBindings collectionBindings( + Foreach node, + RelDataType itemType, + RelDataType iterType, + List capturedFields, + List targetAliases, + List targetTypes) { + Map bindings = new LinkedHashMap<>(); + Map identifiers = new LinkedHashMap<>(); + bindPairPlaceholder( + bindings, identifiers, 0, itemType, PLACEHOLDER_ITEM, OPTION_ITEMSTR, node.getOptions()); + bindPairPlaceholder( + bindings, identifiers, 1, iterType, PLACEHOLDER_ITER, OPTION_ITERSTR, node.getOptions()); + for (int i = 0; i < capturedFields.size(); i++) { + RelDataTypeField field = capturedFields.get(i); + bindIdentifier(identifiers, field.getName(), PAIR_VAR, i + 2, field.getType()); + } + for (int i = 0; i < targetAliases.size(); i++) { + bindIdentifier(identifiers, targetAliases.get(i), STATE_VAR, i, targetTypes.get(i)); + } + return new ForeachBindings(bindings, identifiers); + } + + private void bindPairPlaceholder( + Map bindings, + Map identifiers, + int slot, + RelDataType type, + String placeholder, + String option, + Map options) { + ForeachBinding binding = new ForeachBinding(PAIR_VAR, ForeachBindingType.PAIR_SLOT, slot, type); + bindings.put(placeholder, binding); + if (options.containsKey(option)) { + String customName = options.get(option).toUpperCase(Locale.ROOT); + bindings.put(customName, binding); + identifiers.put(customName, binding); + } + } + + private void bindIdentifier( + Map identifiers, + String name, + String variable, + int slot, + RelDataType type) { + String key = name.toUpperCase(Locale.ROOT); + identifiers.put(key, new ForeachBinding(variable, ForeachBindingType.PAIR_SLOT, slot, type)); + } + + private UnresolvedExpression firstArrayField(CalcitePlanContext context) { + return context.relBuilder.peek().getRowType().getFieldList().stream() + .filter(field -> field.getType() instanceof ArraySqlType) + .findFirst() + .map(field -> (UnresolvedExpression) new Field(new QualifiedName(field.getName()))) + .orElseThrow( + () -> + new SemanticCheckException( + "foreach auto_collections mode requires a multivalue field or JSON array")); + } + + /** + * Ensures the collection expression evaluates to a Calcite array. Non-array expressions (JSON + * array strings or {@code json_array()} calls) are wrapped in {@code foreach_json_array}, which + * parses the JSON and casts every element to one inferred type. + */ + private UnresolvedExpression asArrayExpression( + UnresolvedExpression collection, + boolean nativeArray, + CalcitePlanContext context, + Foreach node) { + if (nativeArray) { + return collection; + } + return new Function( + BuiltinFunctionName.FOREACH_JSON_ARRAY.getName().getFunctionName(), + List.of( + collection, + new Literal(jsonElementType(collection, context, node).name(), DataType.STRING))); + } + + /** + * Infers the element type a JSON array collection should be read as. JSON only distinguishes + * numbers and strings here, so the answer is DOUBLE (gson parses JSON numbers as doubles) or + * VARCHAR. + * + *

    For {@code json_array()} calls and string literals the content is visible at plan time; + * mixed content is rejected. For opaque expressions — typically a field holding JSON text, the + * primary Splunk use of json_array mode — content is unknowable, so infer from usage: an item + * placeholder consumed by arithmetic means numeric elements, anything else means strings. + */ + private SqlTypeName jsonElementType( + UnresolvedExpression collection, CalcitePlanContext context, Foreach node) { + if (collection instanceof Function function + && BuiltinFunctionName.JSON_ARRAY + .getName() + .getFunctionName() + .equalsIgnoreCase(function.getFuncName())) { + return elementTypeOf( + function.getFuncArgs().stream() + .map(arg -> rexVisitor.analyze(arg, context).getType()) + .map(RelDataType::getFamily) + .collect(Collectors.toSet())); + } + if (collection instanceof Literal literal && literal.getType() == DataType.STRING) { + try { + List values = gson.fromJson(String.valueOf(literal.getValue()), List.class); + if (values != null) { + return elementTypeOf( + values.stream() + .filter(Objects::nonNull) + .map(v -> v instanceof Number ? SqlTypeFamily.NUMERIC : SqlTypeFamily.CHARACTER) + .collect(Collectors.toSet())); + } + } catch (JsonSyntaxException ignored) { + // Malformed JSON literals return null at runtime; type choice is irrelevant. + } + return SqlTypeName.VARCHAR; + } + return itemRequiresNumericType(node, context) ? SqlTypeName.DOUBLE : SqlTypeName.VARCHAR; + } + + private boolean itemRequiresNumericType(Foreach node, CalcitePlanContext context) { + String itemName = node.getOptions().getOrDefault(OPTION_ITEMSTR, PLACEHOLDER_ITEM); + boolean customIdentifier = node.getOptions().containsKey(OPTION_ITEMSTR); + return node.getEvalClauses().stream() + .anyMatch( + clause -> + itemRequiresNumericType( + clause.getExpression(), itemName, customIdentifier, context)); + } + + private boolean itemRequiresNumericType( + Node node, String itemName, boolean customIdentifier, CalcitePlanContext context) { + if (node instanceof Compare compare) { + if ((containsItem(compare.getLeft(), itemName, customIdentifier) + && isNumericExpression(compare.getRight(), context)) + || (containsItem(compare.getRight(), itemName, customIdentifier) + && isNumericExpression(compare.getLeft(), context))) { + return true; + } + } + if (node instanceof Function function) { + for (int i = 0; i < function.getFuncArgs().size(); i++) { + if (containsItem(function.getFuncArgs().get(i), itemName, customIdentifier) + && (ARITHMETIC_OPERATORS.contains(function.getFuncName()) + || PPLFuncImpTable.INSTANCE.requiresNumericArgument(function.getFuncName(), i))) { + return true; + } + } + } + return node.getChild().stream() + .anyMatch(child -> itemRequiresNumericType(child, itemName, customIdentifier, context)); + } + + private boolean containsItem(Node node, String itemName, boolean customIdentifier) { + if (isItemReference(node, itemName, customIdentifier)) { + return true; + } + return node.getChild().stream() + .anyMatch(child -> containsItem(child, itemName, customIdentifier)); + } + + private boolean isItemReference(Node node, String itemName, boolean customIdentifier) { + String name; + if (node instanceof ForeachPlaceholder placeholder) { + name = placeholder.getName(); + } else if (node instanceof QualifiedName qualifiedName) { + if (!customIdentifier) { + return false; + } + name = qualifiedName.toString(); + } else { + return false; + } + return PLACEHOLDER_ITEM.equalsIgnoreCase(name) || itemName.equalsIgnoreCase(name); + } + + private boolean isNumericExpression(Node node, CalcitePlanContext context) { + if (node instanceof Literal literal) { + return Set.of( + DataType.SHORT, + DataType.INTEGER, + DataType.LONG, + DataType.FLOAT, + DataType.DOUBLE, + DataType.DECIMAL) + .contains(literal.getType()); + } + try { + return rexVisitor.analyze((UnresolvedExpression) node, context).getType().getFamily() + == SqlTypeFamily.NUMERIC; + } catch (RuntimeException e) { + return false; + } + } + + private SqlTypeName elementTypeOf(Set families) { + boolean numeric = families.contains(SqlTypeFamily.NUMERIC); + boolean character = families.contains(SqlTypeFamily.CHARACTER); + if (numeric && character) { + throw new SemanticCheckException( + "foreach json_array elements must be consistently strings or numbers"); + } + return numeric ? SqlTypeName.DOUBLE : SqlTypeName.VARCHAR; + } + + /** Row fields referenced by the eval clauses that must ride along inside the pairs. */ + private List capturedFields( + List evalClauses, + CalcitePlanContext context, + Map options, + List targetAliases) { + Set excluded = + Stream.concat(Stream.of(PAIR_VAR), targetAliases.stream()) + .map(name -> name.toUpperCase(Locale.ROOT)) + .collect(Collectors.toSet()); + if (options.containsKey(OPTION_ITEMSTR)) { + excluded.add(options.get(OPTION_ITEMSTR).toUpperCase(Locale.ROOT)); + } + if (options.containsKey(OPTION_ITERSTR)) { + excluded.add(options.get(OPTION_ITERSTR).toUpperCase(Locale.ROOT)); + } + Map rowFields = + context.relBuilder.peek().getRowType().getFieldList().stream() + .collect( + Collectors.toMap( + field -> field.getName().toUpperCase(Locale.ROOT), + field -> field, + (left, right) -> left, + LinkedHashMap::new)); + Map captured = new LinkedHashMap<>(); + evalClauses.forEach( + clause -> collectFieldReferences(clause.getExpression(), excluded, rowFields, captured)); + return new ArrayList<>(captured.values()); + } + + private void collectFieldReferences( + Node node, + Set excluded, + Map rowFields, + Map captured) { + if (node instanceof QualifiedName qualifiedName) { + String key = qualifiedName.toString().toUpperCase(Locale.ROOT); + RelDataTypeField field = rowFields.get(key); + if (field != null && !excluded.contains(key)) { + captured.putIfAbsent(key, field); + } + } + node.getChild().forEach(child -> collectFieldReferences(child, excluded, rowFields, captured)); + } + + // ---------------------------------------------------------------------- shared + + private void bindOption( + Map bindings, + Map identifiers, + ForeachBinding binding, + String defaultName, + String option, + Map options) { + bindings.put(defaultName.toUpperCase(Locale.ROOT), binding); + if (options.containsKey(option)) { + String customName = options.get(option).toUpperCase(Locale.ROOT); + bindings.put(customName, binding); + identifiers.put(customName, binding); + } + } + + private String substituteTemplate(String template, Map bindings) { + String result = template; + for (Map.Entry entry : bindings.entrySet()) { + result = + result.replaceAll( + "(?i)<<" + Pattern.quote(entry.getKey()) + ">>", + Matcher.quoteReplacement(entry.getValue().value())); + } + return result; + } + + private record ForeachBindings( + Map values, Map identifiers) {} +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 5ce02e6e0d3..e30d723ccfc 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -69,6 +69,9 @@ public enum BuiltinFunctionName { /** Collection functions */ ARRAY(FunctionName.of("array")), + FOREACH_JSON_ARRAY(FunctionName.of("foreach_json_array"), true), + FOREACH_PAIR_COLLECTION(FunctionName.of("foreach_pair_collection"), true), + FOREACH_STATE(FunctionName.of("foreach_state"), true), ARRAY_LENGTH(FunctionName.of("array_length")), ARRAY_SLICE(FunctionName.of("array_slice"), true), ARRAY_COMPACT(FunctionName.of("array_compact")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java new file mode 100644 index 00000000000..918a4715291 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairCollectionFunctionImpl.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Builds the internal foreach pair array: [item, iter, captured-field...]. */ +public class ForeachPairCollectionFunctionImpl extends ImplementorUDF { + public ForeachPairCollectionFunctionImpl() { + super(new ForeachPairCollectionImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataType pair = + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), pair, true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachPairCollectionImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachPairCollectionFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length == 0 || args[0] == null) { + return null; + } + List source = toList(args[0]); + List pairs = new ArrayList<>(); + for (int i = 0; i < source.size(); i++) { + Object[] pair = new Object[args.length + 1]; + pair[0] = source.get(i); + pair[1] = i; + for (int j = 1; j < args.length; j++) { + pair[j + 1] = args[j]; + } + pairs.add(pair); + } + return pairs; + } + + private static List toList(Object value) { + if (value instanceof List list) { + return list; + } + if (value instanceof Object[] array) { + return Arrays.asList(array); + } + throw new IllegalArgumentException("foreach pair collection requires an array input"); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java new file mode 100644 index 00000000000..075453f38a6 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachPairItemFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * Extracts one slot of an internal foreach pair: {@code foreach_pair_item(pair, index)}. Returns + * OTHER by default; the foreach planner assigns every call its inferred slot type explicitly. + */ +public class ForeachPairItemFunctionImpl extends ImplementorUDF { + public ForeachPairItemFunctionImpl() { + super(new ForeachPairItemImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachPairItemImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachPairItemFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length < 2 || args[0] == null || args[1] == null) { + return null; + } + int index = ((Number) args[1]).intValue(); + List pair = args[0] instanceof Object[] array ? Arrays.asList(array) : (List) args[0]; + return index < 0 || index >= pair.size() ? null : pair.get(index); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java new file mode 100644 index 00000000000..f71294f05b2 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachStateFunctionImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Packs the typed accumulator slots used by a collection-mode foreach eval. */ +public class ForeachStateFunctionImpl extends ImplementorUDF { + public ForeachStateFunctionImpl() { + super(new ForeachStateImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataType slot = + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(SqlTypeName.OTHER), true); + return SqlTypeUtil.createArrayType(opBinding.getTypeFactory(), slot, true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachStateImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachStateFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + return Arrays.asList(args); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java index 314ac3ad945..e8f6d75d7ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CollectionUDF/LambdaUtils.java @@ -16,6 +16,8 @@ import org.apache.calcite.rex.RexLambda; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlCastFunction; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; @@ -68,6 +70,11 @@ public static RelDataType inferReturnTypeFromLambda( public static RexCall reInferReturnTypeForRexCallInsideLambda( RexCall rexCall, Map argTypes, RelDataTypeFactory typeFactory) { + // A CAST's target type lives on the call itself and cannot be re-derived from its operands; + // re-inferring it trips SqlCastFunction's assertion. Keep the call as-is. + if (rexCall.getKind() == SqlKind.CAST || rexCall.getOperator() instanceof SqlCastFunction) { + return rexCall; + } List filledOperands = new ArrayList<>(); List rexCallOperands = rexCall.getOperands(); for (RexNode rexNode : rexCallOperands) { @@ -89,6 +96,14 @@ public static RexCall reInferReturnTypeForRexCallInsideLambda( .getOperator() .inferReturnType( new RexCallBinding(typeFactory, rexCall.getOperator(), filledOperands, List.of())); + // An opaque helper call may carry a more precise type assigned when the call was built (e.g. + // foreach pair slot extraction); re-inference must not erase it. + if ((returnType.getSqlTypeName() == SqlTypeName.ANY + || returnType.getSqlTypeName() == SqlTypeName.OTHER) + && rexCall.getType().getSqlTypeName() != SqlTypeName.ANY + && rexCall.getType().getSqlTypeName() != SqlTypeName.OTHER) { + returnType = rexCall.getType(); + } return rexCall.clone(returnType, filledOperands); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index a24015de992..d64f04bb9ad 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -47,6 +47,9 @@ import org.opensearch.sql.expression.function.CollectionUDF.ExistsFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.FilterFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ForallFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachPairCollectionFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachPairItemFunctionImpl; +import org.opensearch.sql.expression.function.CollectionUDF.ForeachStateFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVAppendFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVFindFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.MVZipFunctionImpl; @@ -54,6 +57,7 @@ import org.opensearch.sql.expression.function.CollectionUDF.MapRemoveFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.ReduceFunctionImpl; import org.opensearch.sql.expression.function.CollectionUDF.TransformFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.ForeachJsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; @@ -405,6 +409,14 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("forall"); public static final SqlOperator EXISTS = new ExistsFunctionImpl().toUDF("exists"); public static final SqlOperator ARRAY = new ArrayFunctionImpl().toUDF("array"); + public static final SqlOperator FOREACH_JSON_ARRAY = + new ForeachJsonArrayFunctionImpl().toUDF("foreach_json_array"); + public static final SqlOperator FOREACH_PAIR_COLLECTION = + new ForeachPairCollectionFunctionImpl().toUDF("foreach_pair_collection"); + public static final SqlOperator FOREACH_PAIR_ITEM = + new ForeachPairItemFunctionImpl().toUDF("foreach_pair_item"); + public static final SqlOperator FOREACH_STATE = + new ForeachStateFunctionImpl().toUDF("foreach_state"); public static final SqlOperator MAP_APPEND = new MapAppendFunctionImpl().toUDF("map_append"); public static final SqlOperator MAP_REMOVE = new MapRemoveFunctionImpl().toUDF("MAP_REMOVE"); public static final SqlOperator MVAPPEND = new MVAppendFunctionImpl().toUDF("mvappend"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index b9350d18d84..151c4a96655 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -75,6 +75,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.FIRST; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FORALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_JSON_ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_PAIR_COLLECTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FOREACH_STATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_UNIXTIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.GET_FORMAT; @@ -273,6 +276,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -315,6 +319,8 @@ import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -428,6 +434,63 @@ private PPLFuncImpTable(Builder builder, AggBuilder aggBuilder) { this.aggExternalFunctionRegistry = new ConcurrentHashMap<>(); } + /** Returns whether every known signature requires a numeric value at the given argument. */ + public boolean requiresNumericArgument(String functionName, int argumentIndex) { + Optional builtin = BuiltinFunctionName.of(functionName); + if (builtin.isEmpty()) { + return false; + } + List> implementations = + new ArrayList<>(functionRegistry.getOrDefault(builtin.get(), List.of())); + implementations.addAll(externalFunctionRegistry.getOrDefault(builtin.get(), List.of())); + boolean foundArgument = false; + for (Pair implementation : implementations) { + PPLTypeChecker checker = implementation.getKey().typeChecker(); + if (checker == null) { + return false; + } + try { + List> signatures = + checker.getParameterTypes().stream() + .filter(parameters -> argumentIndex < parameters.size()) + .toList(); + if (signatures.isEmpty()) { + return false; + } + foundArgument = true; + List acceptedTypes = + signatures.stream().map(parameters -> parameters.get(argumentIndex)).toList(); + if (acceptedTypes.stream().allMatch(ExprCoreType.numberTypes()::contains)) { + continue; + } + if (acceptedTypes.stream().anyMatch(type -> type != ExprCoreType.UNKNOWN) + || !requiresNumericByValidation(checker, signatures, argumentIndex)) { + return false; + } + } catch (RuntimeException e) { + return false; + } + } + return foundArgument; + } + + private boolean requiresNumericByValidation( + PPLTypeChecker checker, List> signatures, int argumentIndex) { + RelDataType numericType = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); + RelDataType stringType = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); + return signatures.stream() + .anyMatch( + signature -> { + List numericArguments = + new ArrayList<>(Collections.nCopies(signature.size(), numericType)); + if (!checker.checkOperandTypes(numericArguments)) { + return false; + } + numericArguments.set(argumentIndex, stringType); + return !checker.checkOperandTypes(numericArguments); + }); + } + /** * Register an operator from external services dynamically. * @@ -956,11 +1019,17 @@ void populate() { wrapSqlOperandTypeChecker( SqlLibraryOperators.REGEXP_REPLACE_3.getOperandTypeChecker(), REPLACE.name(), false)); registerOperator(UPPER, SqlStdOperatorTable.UPPER); - registerOperator(ABS, SqlStdOperatorTable.ABS); - registerOperator(ACOS, SqlStdOperatorTable.ACOS); - registerOperator(ASIN, SqlStdOperatorTable.ASIN); - registerOperator(ATAN, SqlStdOperatorTable.ATAN); - registerOperator(ATAN2, SqlStdOperatorTable.ATAN2); + registerOperator(ABS, SqlStdOperatorTable.ABS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ACOS, SqlStdOperatorTable.ACOS, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ASIN, SqlStdOperatorTable.ASIN, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ATAN, SqlStdOperatorTable.ATAN, PPLTypeChecker.family(SqlTypeFamily.NUMERIC)); + registerOperator( + ATAN2, + SqlStdOperatorTable.ATAN2, + PPLTypeChecker.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.NUMERIC)); // TODO, workaround to support sequence CompositeOperandTypeChecker. registerOperator( CEIL, @@ -1235,6 +1304,9 @@ void populate() { registerOperator(FILTER, PPLBuiltinOperators.FILTER); registerOperator(TRANSFORM, PPLBuiltinOperators.TRANSFORM); registerOperator(REDUCE, PPLBuiltinOperators.REDUCE); + registerOperator(FOREACH_JSON_ARRAY, PPLBuiltinOperators.FOREACH_JSON_ARRAY); + registerOperator(FOREACH_PAIR_COLLECTION, PPLBuiltinOperators.FOREACH_PAIR_COLLECTION); + registerOperator(FOREACH_STATE, PPLBuiltinOperators.FOREACH_STATE); // Register Json function register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java new file mode 100644 index 00000000000..28dff66a8ae --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonSyntaxException; +import java.math.BigDecimal; +import java.util.List; +import java.util.stream.StreamSupport; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** Converts a JSON array string to a Calcite enumerable array for foreach collection modes. */ +public class ForeachJsonArrayFunctionImpl extends ImplementorUDF { + public ForeachJsonArrayFunctionImpl() { + super(new ForeachJsonArrayImplementor(), NullPolicy.NONE); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + SqlTypeName elementTypeName = + SqlTypeName.valueOf(opBinding.getOperandLiteralValue(1, String.class)); + return SqlTypeUtil.createArrayType( + opBinding.getTypeFactory(), + opBinding + .getTypeFactory() + .createTypeWithNullability( + opBinding.getTypeFactory().createSqlType(elementTypeName), true), + true); + }; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class ForeachJsonArrayImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + return Expressions.call( + Types.lookupMethod(ForeachJsonArrayFunctionImpl.class, "eval", Object[].class), + translatedOperands); + } + } + + public static Object eval(Object... args) { + if (args.length != 2 || args[0] == null) { + return null; + } + SqlTypeName elementType = SqlTypeName.valueOf(String.valueOf(args[1])); + try { + if (args[0] instanceof List values) { + return values.stream().map(value -> cast(value, elementType)).toList(); + } + JsonArray values = gson.fromJson(String.valueOf(args[0]), JsonArray.class); + if (values == null) { + return List.of(); + } + return StreamSupport.stream(values.spliterator(), false) + .map(value -> cast(value, elementType)) + .toList(); + } catch (JsonSyntaxException e) { + return List.of(); + } + } + + private static Object cast(Object value, SqlTypeName elementType) { + if (value instanceof JsonElement element) { + if (element.isJsonNull()) { + return elementType == SqlTypeName.VARCHAR ? "null" : null; + } + if (element.isJsonArray() || element.isJsonObject()) { + return element.toString(); + } + value = + element.getAsJsonPrimitive().isNumber() ? element.getAsNumber() : element.getAsString(); + } + if (value == null) { + return null; + } + return switch (elementType) { + case DOUBLE -> ((Number) value).doubleValue(); + case VARCHAR -> + value instanceof List || value instanceof java.util.Map + ? gson.toJson(value) + : String.valueOf(value); + case DECIMAL -> BigDecimal.valueOf(((Number) value).doubleValue()); + default -> value; + }; + } +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index fee5bfcaa66..d073a6802aa 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -1943,6 +1943,24 @@ public void regex_command_throws_unsupported_exception_with_legacy_engine() { "Regex is supported only when plugins.calcite.enabled=true", exception.getMessage()); } + @Test + public void foreach_command_throws_unsupported_exception_with_legacy_engine() { + UnsupportedOperationException exception = + assertThrows( + UnsupportedOperationException.class, + () -> + analyze( + new org.opensearch.sql.ast.tree.Foreach( + org.opensearch.sql.ast.tree.Foreach.Mode.MULTIFIELD, + ImmutableMap.of(), + ImmutableList.of("integer_value"), + null, + ImmutableList.of()) + .attach(relation("schema")))); + assertEquals( + "foreach is supported only when plugins.calcite.enabled=true", exception.getMessage()); + } + @Test public void rex_command_throws_unsupported_operation_exception_in_legacy_engine() { UnsupportedOperationException exception = diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java new file mode 100644 index 00000000000..e3c3724e0aa --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.CollectionUDF; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.jsonUDF.ForeachJsonArrayFunctionImpl; + +public class ForeachFunctionImplTest { + + @Test + public void testPairCollectionPreservesNullElements() { + List pairs = + (List) + ForeachPairCollectionFunctionImpl.eval( + new Object[] {new Object[] {1, null, 3}, "captured"}); + + assertEquals(3, pairs.size()); + assertArrayEquals(new Object[] {1, 0, "captured"}, (Object[]) pairs.get(0)); + assertArrayEquals(new Object[] {null, 1, "captured"}, (Object[]) pairs.get(1)); + assertArrayEquals(new Object[] {3, 2, "captured"}, (Object[]) pairs.get(2)); + } + + @Test + public void testJsonArrayPreservesTopLevelNestedValuesAndNull() { + Object values = ForeachJsonArrayFunctionImpl.eval("[[1,2],{\"a\":1},null]", "VARCHAR"); + + assertEquals(List.of("[1,2]", "{\"a\":1}", "null"), values); + } + + @Test + public void testMalformedJsonArrayIsEmpty() { + assertEquals(List.of(), ForeachJsonArrayFunctionImpl.eval("not-json", "VARCHAR")); + } + + @Test + public void testStatePreservesHeterogeneousAndNullSlots() { + assertEquals( + Arrays.asList(3, null, "seen"), + ForeachStateFunctionImpl.eval(new Object[] {3, null, "seen"})); + } +} diff --git a/docs/category.json b/docs/category.json index 04f2bbae22e..a8665b82eac 100644 --- a/docs/category.json +++ b/docs/category.json @@ -20,6 +20,7 @@ "user/ppl/cmd/fieldformat.md", "user/ppl/cmd/fields.md", "user/ppl/cmd/fillnull.md", + "user/ppl/cmd/foreach.md", "user/ppl/cmd/grok.md", "user/ppl/cmd/head.md", "user/ppl/cmd/join.md", diff --git a/docs/user/ppl/cmd/foreach.md b/docs/user/ppl/cmd/foreach.md new file mode 100644 index 00000000000..45dcb939f8a --- /dev/null +++ b/docs/user/ppl/cmd/foreach.md @@ -0,0 +1,191 @@ +# foreach + +The `foreach` command runs a templated `eval` expression for each field in a field list, each element of a multivalue (array) field, or each element of a JSON array. It eliminates repetitive `eval` statements when the same computation applies to many fields or to every element of a collection. + +## Syntax + +The `foreach` command has the following syntax: + +```syntax +foreach [mode=] [

    This does the same rewrite as Calcite's {@code ConvertToChecked} but preserves each call's + * originally inferred type (via {@code makeCall(type, op, operands)}) and touches only the three + * arithmetic operators, so it does not re-derive the types of unrelated calls (e.g. {@code + * CEIL}/{@code DIVIDE}) the way {@code ConvertToChecked} does. + */ + private static RelNode withCheckedArithmetic(RelNode calcitePlan, CalcitePlanContext context) { + RexShuttle checkedShuttle = + new RexShuttle() { + @Override + public RexNode visitCall(RexCall call) { + RexNode visited = super.visitCall(call); + if (!(visited instanceof RexCall rexCall)) { + return visited; + } + SqlOperator checked = + switch (rexCall.getOperator().getKind()) { + case PLUS -> SqlStdOperatorTable.CHECKED_PLUS; + case MINUS -> SqlStdOperatorTable.CHECKED_MINUS; + case TIMES -> SqlStdOperatorTable.CHECKED_MULTIPLY; + default -> null; + }; + // Only integer/long arithmetic can overflow silently and has a checked + // implementation (Math.addExact etc.). Float/double/decimal have no checked variant + // (SqlFunctions.checkedMultiply(double,double) does not exist) and follow IEEE 754, so + // leave them untouched. + if (checked == null || !isCheckableIntegerArithmetic(rexCall)) { + return visited; + } + return context.rexBuilder.makeCall(rexCall.getType(), checked, rexCall.getOperands()); + } + }; + return calcitePlan.accept( + new RelHomogeneousShuttle() { + @Override + public RelNode visit(RelNode other) { + RelNode visited = super.visitChildren(other); + return visited.accept(checkedShuttle); + } + }); + } + + /** Returns whether the result and every operand are BIGINT. */ + private static boolean isCheckableIntegerArithmetic(RexCall call) { + if (!isCheckableLongType(call.getType())) { + return false; + } + return call.getOperands().stream().allMatch(op -> isCheckableLongType(op.getType())); + } + + private static boolean isCheckableLongType(org.apache.calcite.rel.type.RelDataType type) { + return type.getSqlTypeName() == org.apache.calcite.sql.type.SqlTypeName.BIGINT; + } + + /** + * Walk the cause chain to find an {@link ArithmeticException} raised by checked arithmetic. Row- + * level overflow surfaces wrapped (SQLException -> RuntimeException -> ErrorReport), so a + * top-level {@code catch (ArithmeticException)} is insufficient. + */ + private static ArithmeticException findArithmeticOverflow(@Nullable Throwable t) { + for (Throwable cause = t; + cause != null && cause != cause.getCause(); + cause = cause.getCause()) { + if (cause instanceof ArithmeticException arithmeticException) { + return arithmeticException; + } + } + return null; + } + // TODO https://github.com/opensearch-project/sql/issues/3457 // Calcite is not available for SQL query now. Maybe release in 3.1.0? private boolean shouldUseCalcite(QueryType queryType) { diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index e42d867705c..427a0334b58 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -11,6 +11,10 @@ Arithmetic expressions are formed by combining numeric literals and binary arith 4. `/`: Division. When [`plugins.ppl.syntax.legacy.preferred`](../admin/settings.md) is `true` (default), integer operands follow the legacy truncating result. When the setting is `false`, the operands are promoted to floating-point, preserving the fractional part. Division by zero returns `NULL`. 5. `%`: Modulo. This operator can only be used with integers and returns the remainder of the division. +### Overflow behavior + +Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + ### Precedence You can use parentheses to control the precedence of arithmetic operators. Otherwise, operators with higher precedence are performed first. diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index f349523ec56..b6adbc58332 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index c42ecef2132..efd325acf52 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+(CAST($10):BIGINT, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index ffd55ffb1fb..101238a2a7b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -6,4 +6,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index 64e868f9f8c..90ec92f0b2f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age=[$t0], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index 8f60e23e491..5186752243b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):BIGINT], expr#3=[+($t2, $t1)], age2=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+(CAST($0):BIGINT, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index 7ad040f826d..9ac70f8b5c7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[-($t14, $t13)], proj#0..12=[{exprs}], age2=[$t14], age3=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEzHsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9Cn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+(CAST($10):BIGINT, $7), CAST($10):BIGINT) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQE3HsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICIrIiwKICAgICAgICAia2luZCI6ICJDSEVDS0VEX1BMVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQ0FTVCIsCiAgICAgICAgImtpbmQiOiAiQ0FTVCIsCiAgICAgICAgInN5bnRheCI6ICJTUEVDSUFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index e1328084f77..1009e374b1f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject($f1=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBQ3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBS3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml index fbec63d8c6f..88bae9d4d59 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCwnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCynsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIkNIRUNLRURfTUlOVVMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml index de78240cbea..9be9d4ea119 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"VARCHAR\\\",\\n \\\"nullable\\\": true,\\n \\\"precision\\\": -1\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address","671 Bristol Street"]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"{\\n \\\"op\\\": {\\n \\\"name\\\": \\\"=\\\",\\n \\\"kind\\\": \\\"EQUALS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"op\\\": {\\n \\\"name\\\": \\\"-\\\",\\n \\\"kind\\\": \\\"CHECKED_MINUS\\\",\\n \\\"syntax\\\": \\\"BINARY\\\"\\n },\\n \\\"operands\\\": [\\n {\\n \\\"dynamicParam\\\": 0,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 1,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ],\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n },\\n {\\n \\\"dynamicParam\\\": 2,\\n \\\"type\\\": {\\n \\\"type\\\": \\\"BIGINT\\\",\\n \\\"nullable\\\": true\\n }\\n }\\n ]\\n}\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age","address"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index f5834404f19..f1b1e2b4524 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[CAST($t10):BIGINT], expr#14=[+($t13, $t7)], expr#15=[1:BIGINT], expr#16=[+($t7, $t15)], proj#0..12=[{exprs}], age2=[$t14], balance2=[$t16]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCN3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDQVNUIiwKICAgICAgICAia2luZCI6ICJDQVNUIiwKICAgICAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+(CAST($10):BIGINT, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCP3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIkNIRUNLRURfUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNBU1QiLAogICAgICAgICJraW5kIjogIkNBU1QiLAogICAgICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml new file mode 100644 index 00000000000..930e61fdf75 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164.yml @@ -0,0 +1,170 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# Integer/long arithmetic (+, -, *) must not silently wrap on overflow. +# +# byte/short/int overflow is prevented by operand widening (short/byte -> INT, any int/long -> +# BIGINT) in PPLFuncImpTable, so those produce the correct wider value (HTTP 200). long (BIGINT) +# arithmetic has no wider integer type to widen into, so overflow is detected via checked +# arithmetic (Math.addExact / multiplyExact) and surfaced as a 4xx client error instead of wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_overflow_5164 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + int_field: + type: integer + long_field: + type: long + small_int: + type: integer + short_field: + type: short + - do: + bulk: + index: test_overflow_5164 + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"int_field": 2147483647, "long_field": 9223372036854775807, "small_int": 10, "short_field": 30000}' + - '{"index": {"_id": "2"}}' + - '{"int_field": 100, "long_field": 200, "small_int": 5, "short_field": 3}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_overflow_5164 + ignore_unavailable: true + +--- +"Normal integer addition does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where small_int = 5 | eval sum = int_field + small_int | fields int_field, small_int, sum + - match: { total: 1 } + - match: { datarows: [[100, 5, 105]] } + +--- +"Integer addition overflow widens instead of wrapping": + - skip: + features: + - headers + # int + int is widened to BIGINT before the operation; 2147483647 + 1 = 2147483648 fits exactly. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field + 1 | fields widened + - match: { total: 1 } + - match: { datarows: [[2147483648]] } + +--- +"Integer multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # 2147483647 * 2 = 4294967294 exceeds INT range but is exact once widened to BIGINT. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where int_field = 2147483647 | eval widened = int_field * 2 | fields widened + - match: { total: 1 } + - match: { datarows: [[4294967294]] } + +--- +"Short multiplication overflow widens instead of wrapping": + - skip: + features: + - headers + # short * short historically wrapped into the 16-bit SMALLINT range (bug bash #8: 30000 * 30000 + # wrapped). Widening to INT keeps the exact value 900000000. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where short_field = 30000 | eval widened = short_field * short_field | fields widened + - match: { total: 1 } + - match: { datarows: [[900000000]] } + +--- +"Long addition overflow throws error": + - skip: + features: + - headers + # long + long has no wider integer type; overflow is detected and surfaced as a 4xx. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field + 1 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long multiplication overflow throws error": + - skip: + features: + - headers + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field * 2 | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Long subtraction overflow throws error": + - skip: + features: + - headers + # long_field is i64::MAX; subtracting -1 overflows past the top of the BIGINT range. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 9223372036854775807 | eval overflow = long_field - (-1) | fields overflow + - match: { "$body": "/overflow/" } + +--- +"Normal long arithmetic does not error": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_overflow_5164 | where long_field = 200 | eval product = long_field * 2 | fields product + - match: { total: 1 } + - match: { datarows: [[400]] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java index d77dee3e297..8a0b4daa57a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -232,6 +232,7 @@ public Object toJson(RexNode node) { map.put("operands", list); switch (node.getKind()) { case MINUS: + case CHECKED_MINUS: case CAST: case SAFE_CAST: map.put("type", toJson(node.getType())); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index dab778923b6..aad2623a6f7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -55,7 +55,7 @@ public static Optional> getOrderEquivalentInputInfo(RexNo case MINUS_PREFIX: return getOrderEquivalentInputInfo(((RexCall) expr).getOperands().get(0)) .map(inputInfo -> Pair.of(inputInfo.getLeft(), !inputInfo.getRight())); - case PLUS, MINUS: + case PLUS, MINUS, CHECKED_PLUS, CHECKED_MINUS: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1); @@ -68,12 +68,14 @@ public static Optional> getOrderEquivalentInputInfo(RexNo } RexNode variable = operand0Lit ? operand1 : operand0; - boolean flipped = (expr.getKind() == SqlKind.MINUS) && operand0Lit; + boolean isMinus = + expr.getKind() == SqlKind.MINUS || expr.getKind() == SqlKind.CHECKED_MINUS; + boolean flipped = isMinus && operand0Lit; return getOrderEquivalentInputInfo(variable) .map(inputInfo -> Pair.of(inputInfo.getLeft(), flipped != inputInfo.getRight())); } - case TIMES: + case TIMES, CHECKED_TIMES: { RexNode operand0 = ((RexCall) expr).getOperands().get(0); RexNode operand1 = ((RexCall) expr).getOperands().get(1); From 9a35e76fea098b08788d1130f20e7e4e7672d107 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 16:54:27 -0700 Subject: [PATCH 38/41] feat(ci): add PPL lint rule validation check (eventstats PoC) Add a cross-repository CI check that keeps the OpenSearch-Dashboards PPL lint rule 'unsupported-window-function-in-eventstats' and the SQL backend in agreement. Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an OSD checkout and asserts the rule's diagnostic counts. Backend half: a Gradle integration test sends the same queries to the live /_plugins/_ppl endpoint of the SQL plugin built from the checkout. Both halves consume one shared contract file. - integ-test/.../ppl-lint/unsupported-window-function-in-eventstats.spec.json - scripts/ppl-lint/run-frontend-contract.mjs - integ-test/.../calcite/remote/PplLintRuleValidationIT.java - .github/workflows/ppl-lint-rule-validation.yml - scripts/ppl-lint-rule-validation.sh Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 148 ++++++++++++++++ .../remote/PplLintRuleValidationIT.java | 130 ++++++++++++++ ...ed-window-function-in-eventstats.spec.json | 29 ++++ scripts/ppl-lint-rule-validation.sh | 105 +++++++++++ scripts/ppl-lint/run-frontend-contract.mjs | 164 ++++++++++++++++++ 5 files changed, 576 insertions(+) create mode 100644 .github/workflows/ppl-lint-rule-validation.yml create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java create mode 100644 integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json create mode 100755 scripts/ppl-lint-rule-validation.sh create mode 100644 scripts/ppl-lint/run-frontend-contract.mjs diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml new file mode 100644 index 00000000000..61ee0be6fb8 --- /dev/null +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -0,0 +1,148 @@ +name: PPL lint rule validation + +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule +# `unsupported-window-function-in-eventstats` and the SQL backend must agree. +# +# Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an +# OSD checkout and asserts the rule's diagnostic counts. +# Backend half: a Gradle integration test sends the same queries to the live +# `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. +# +# The OSD detector is loaded from `main` by default (PR and nightly runs), so a +# removed or changed detector is detected. `workflow_dispatch` can target a +# specific OSD ref to reproduce a run or test an unmerged OSD branch. + +on: + pull_request: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + inputs: + osd_ref: + description: OSD commit or branch to test instead of main + required: false + type: string + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + ppl-lint-rule-validation: + name: PPL lint rule validation + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve OSD ref + id: osd-ref + env: + REQUESTED_REF: ${{ inputs.osd_ref }} + run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: opensearch-project/OpenSearch-Dashboards + ref: ${{ steps.osd-ref.outputs.ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-sha + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding + # it, so an OSD toolchain bump does not silently drift this job. + - name: Read OSD Node version + id: osd-node + run: echo "version=$(cat .ci/OpenSearch-Dashboards/.nvmrc)" >> "$GITHUB_OUTPUT" + + - name: Set up Node ${{ steps.osd-node.outputs.version }} + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version: ${{ steps.osd-node.outputs.version }} + + - name: Read OSD Yarn version + id: osd-yarn + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + echo "version=$yarn_version" >> "$GITHUB_OUTPUT" + + - name: Pin Yarn ${{ steps.osd-yarn.outputs.version }} + run: npm install -g "yarn@${{ steps.osd-yarn.outputs.version }}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: yarn osd bootstrap + + # The Gradle test cluster runs the version from build.gradle's + # `opensearch.version` default (e.g. 3.8.0-SNAPSHOT). Export the release + # portion (3.8.0) as PPL_SQL_VERSION so the frontend applies the same + # version filtering the backend does, without maintaining a second string. + - name: Resolve OpenSearch version + id: os-version + run: | + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + version="${raw%%-*}" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" + + # The frontend contract only runs Node (no OpenSearch cluster), so it runs + # as the default container user. Only the Gradle integTest below must run + # as a non-root user, because OpenSearch refuses to start as root. + - name: Run frontend contract (OSD analyzer) + working-directory: .ci/OpenSearch-Dashboards + env: + PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json + PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }} + run: | + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + | tee "$GITHUB_WORKSPACE/frontend-contract.log" + + - name: Run backend integration test (live /_plugins/_ppl) + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + + - name: Upload failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-rule-validation-artifacts + path: | + frontend-contract.log + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java new file mode 100644 index 00000000000..ac92f979595 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Backend half of the PPL lint rule validation contract. + * + *

    This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from + * the current checkout and asserts, per contract case, that: + * + *

      + *
    • a rejected query returns the contracted HTTP status and structured error body ({@code + * status}, {@code error.type}, {@code error.reason}); and + *
    • a valid control query returns HTTP 200 with data. + *
    + * + *

    The contract file is shared verbatim with the SQL-owned OSD frontend adapter ({@code + * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer + * diagnostic and the SQL backend behavior. The rejection-body parsing mirrors the existing {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT} pattern; the live fixture and + * Calcite setup follow {@link org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + */ +public class PplLintRuleValidationIT extends PPLIntegTestCase { + + private static final String CONTRACT_RESOURCE = + "src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json"; + + @Override + public void init() throws Exception { + super.init(); + // eventstats via the Calcite path. Disallow fallback so an unsupported window function is + // rejected rather than silently degrading to the V2 engine. + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.ACCOUNT); + } + + @Test + public void testValidatesUnsupportedWindowFunctionContract() throws IOException { + JSONObject contract = loadContract(); + String index = contract.getString("index"); + JSONArray cases = contract.getJSONArray("cases"); + + for (int i = 0; i < cases.length(); i++) { + JSONObject testCase = cases.getJSONObject(i); + String caseId = testCase.getString("id"); + String query = testCase.getString("query").replace("{{index}}", index); + JSONObject backendExpected = testCase.getJSONObject("backendExpected"); + int expectedStatus = backendExpected.getInt("httpStatus"); + + if (expectedStatus == 200) { + verifyAcceptedCase(caseId, query); + } else { + verifyRejectedCase(caseId, query, expectedStatus, backendExpected.getJSONObject("body")); + } + } + } + + /** A valid control query must return HTTP 200. executeQuery already asserts the 200 status. */ + private void verifyAcceptedCase(String caseId, String query) throws IOException { + JSONObject response = executeQuery(query); + assertTrue( + "case \"" + + caseId + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + } + + /** + * A rejected query must throw a {@link ResponseException} whose response carries the contracted + * HTTP status and structured error fields. executeQuery internally asserts 200, so a non-200 + * response surfaces as a ResponseException before it can return. + */ + private void verifyRejectedCase( + String caseId, String query, int expectedStatus, JSONObject expectedBody) { + ResponseException exception = assertThrows(ResponseException.class, () -> executeQuery(query)); + + int actualStatus = exception.getResponse().getStatusLine().getStatusCode(); + assertEquals( + "case \"" + caseId + "\": unexpected HTTP status for query: " + query, + expectedStatus, + actualStatus); + + JSONObject body; + try { + body = new JSONObject(getResponseBody(exception.getResponse(), true)); + } catch (IOException e) { + throw new RuntimeException( + "case \"" + caseId + "\": failed to read rejection response body for query: " + query, e); + } + + assertEquals( + "case \"" + caseId + "\": unexpected top-level status field for query: " + query, + expectedBody.getInt("status"), + body.getInt("status")); + + JSONObject expectedError = expectedBody.getJSONObject("error"); + JSONObject actualError = body.getJSONObject("error"); + + assertEquals( + "case \"" + caseId + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + assertEquals( + "case \"" + caseId + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } + + private JSONObject loadContract() throws IOException { + String path = TestUtils.getResourceFilePath(CONTRACT_RESOURCE); + return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); + } +} diff --git a/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..02fbab79a3c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,29 @@ +{ + "ruleId": "unsupported-window-function-in-eventstats", + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "eventstats-rank", + "query": "source={{index}} | eventstats rank() as rank_value", + "frontendDiagnosticCount": 1, + "backendExpected": { + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + }, + { + "id": "eventstats-avg-control", + "query": "source={{index}} | eventstats avg(age) as avg_age", + "frontendDiagnosticCount": 0, + "backendExpected": { + "httpStatus": 200 + } + } + ] +} diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh new file mode 100755 index 00000000000..1552353d052 --- /dev/null +++ b/scripts/ppl-lint-rule-validation.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Local developer entry point for the PPL lint rule validation contract. +# +# Runs both halves of the cross-repository check from a SQL checkout: +# 1. Frontend: loads the compiled OpenSearch-Dashboards (OSD) PPL analyzer and +# asserts the rule's diagnostic counts against the shared contract. +# 2. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout. +# +# Usage: +# # OSD main frontend check plus SQL backend IT (fetches OSD into .ci/) +# ./scripts/ppl-lint-rule-validation.sh +# +# # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) +# OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh +# +# # Reproduce a CI run against a specific OSD revision +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# +# # Skip one half +# SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_FRONTEND=1 ./scripts/ppl-lint-rule-validation.sh + +set -euo pipefail + +SQL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SQL_ROOT" + +OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" +OSD_REF="${OSD_REF:-main}" +DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" +CONTRACT_FILE="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json" +FRONTEND_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + +log() { echo "[ppl-lint-rule-validation] $*"; } + +resolve_opensearch_version() { + local raw + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | + sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + echo "${raw%%-*}" +} + +run_frontend() { + local osd_checkout="$1" + + if [[ ! -d "$osd_checkout/node_modules" ]]; then + log "Bootstrapping OSD at $osd_checkout (this can take a while)..." + (cd "$osd_checkout" && yarn osd bootstrap) + else + log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." + fi + + local os_version + os_version="$(resolve_opensearch_version)" + log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version)..." + ( + cd "$osd_checkout" + PPL_LINT_CONTRACT_FILE="$CONTRACT_FILE" \ + PPL_SQL_VERSION="$os_version" \ + node -r ./src/setup_node_env "$FRONTEND_SCRIPT" + ) +} + +if [[ "${SKIP_FRONTEND:-0}" != "1" ]]; then + if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then + OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" + log "Using existing OSD checkout: $OSD_CHECKOUT" + else + OSD_CHECKOUT="$DEFAULT_OSD_CHECKOUT" + if [[ ! -d "$OSD_CHECKOUT/.git" ]]; then + log "Cloning OSD ($OSD_REF) into $OSD_CHECKOUT ..." + mkdir -p "$(dirname "$OSD_CHECKOUT")" + git clone --depth 1 --branch "$OSD_REF" "$OSD_REPO_URL" "$OSD_CHECKOUT" 2>/dev/null || + git clone "$OSD_REPO_URL" "$OSD_CHECKOUT" + fi + log "Checking out OSD ref: $OSD_REF" + git -C "$OSD_CHECKOUT" fetch --depth 1 origin "$OSD_REF" 2>/dev/null || true + git -C "$OSD_CHECKOUT" checkout "$OSD_REF" 2>/dev/null || + git -C "$OSD_CHECKOUT" checkout FETCH_HEAD + fi + + OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" + log "OSD revision under test: $OSD_SHA" + + run_frontend "$OSD_CHECKOUT" + log "Frontend contract passed." +else + log "SKIP_FRONTEND=1 — skipping the OSD frontend contract." +fi + +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + log "Running backend integration test: $IT_CLASS" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" + log "Backend integration test passed." +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test." +fi + +log "Done." diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs new file mode 100644 index 00000000000..603e3feab6c --- /dev/null +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -0,0 +1,164 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * SQL-owned frontend contract adapter for the PPL lint rule validation CI. + * + * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, + * for example: + * + * cd .ci/OpenSearch-Dashboards + * PPL_LINT_CONTRACT_FILE= \ + * PPL_SQL_VERSION= \ + * node -r ./src/setup_node_env \ + * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + * + * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation + * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles + * `packages/osd-monaco/src/**` TypeScript on `require()` regardless of where the + * entry script lives. That is what lets this SQL-owned `.mjs` load the compiled + * OSD analyzer without OSD's own Jest. + * + * The analyzer, catalog and detector registry are NOT re-exported from the + * `@osd/monaco` package barrel, so they are loaded via their deep module paths. + * Because this is an ES module, `require` is obtained with `createRequire`, and + * the modules are resolved against the OSD checkout root (process.cwd()) rather + * than the location of this script (which lives in the SQL repo, not OSD). + */ + +import assert from 'assert'; +import fs from 'fs'; +import path from 'path'; +import { createRequire } from 'module'; + +const RULE_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; +const CATALOG_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/src/ppl/lint/detector_registry'; + +function fail(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-frontend-contract] FAIL: ${message}`); + process.exit(1); +} + +function loadContract() { + const contractFile = process.env.PPL_LINT_CONTRACT_FILE; + if (!contractFile) { + fail('PPL_LINT_CONTRACT_FILE is not set.'); + } + if (!fs.existsSync(contractFile)) { + fail(`Contract file not found: ${contractFile}`); + } + try { + return JSON.parse(fs.readFileSync(contractFile, 'utf8')); + } catch (error) { + fail(`Could not parse contract file ${contractFile}: ${error.message}`); + return undefined; // unreachable + } +} + +function loadOsdAnalyzer() { + // Resolve the OSD compiled analyzer from the checkout root. In CI this script + // runs after `cd .ci/OpenSearch-Dashboards`, so process.cwd() is that root. + const osdRoot = process.cwd(); + const require = createRequire(path.join(osdRoot, 'noop.js')); + + const resolveOsd = (relativeModule) => { + const absolute = path.join(osdRoot, relativeModule); + if (!fs.existsSync(`${absolute}.ts`) && !fs.existsSync(`${absolute}.js`)) { + fail( + `Expected OSD module not found under the checkout root: ${relativeModule}\n` + + `Resolved OSD root: ${osdRoot}\n` + + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` + ); + } + return require(absolute); + }; + + const { PPLLanguageAnalyzer } = resolveOsd(RULE_MODULE); + const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + const { getDetector } = resolveOsd(DETECTOR_REGISTRY_MODULE); + + if (typeof PPLLanguageAnalyzer !== 'function') { + fail(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); + } + return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot }; +} + +function assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector) { + const entry = getBundledCatalog().find((candidate) => candidate.id === ruleId); + if (!entry) { + fail(`Rule "${ruleId}" is not present in the OSD bundled catalog.`); + } + if (!entry.enabled) { + fail(`Rule "${ruleId}" is present but disabled in the OSD bundled catalog.`); + } + if (entry.severity !== 'error') { + fail(`Rule "${ruleId}" severity is "${entry.severity}", expected "error".`); + } + if (typeof getDetector(entry.detector) !== 'function') { + fail(`Rule "${ruleId}" has no registered detector "${entry.detector}".`); + } + return entry; +} + +function main() { + const contract = loadContract(); + const ruleId = contract.ruleId; + const index = contract.index; + const sqlVersion = process.env.PPL_SQL_VERSION; + + const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = loadOsdAnalyzer(); + + const entry = assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] OSD root: ${osdRoot}\n` + + `[ppl-lint-frontend-contract] rule "${ruleId}" enabled=${entry.enabled} severity=${entry.severity} detector="${entry.detector}"\n` + + `[ppl-lint-frontend-contract] PPL_SQL_VERSION=${sqlVersion || '(unset)'}\n` + + `[ppl-lint-frontend-contract] running ${contract.cases.length} case(s) against index "${index}"` + ); + + const analyzer = new PPLLanguageAnalyzer(); + const failures = []; + + for (const testCase of contract.cases) { + const query = testCase.query.split('{{index}}').join(index); + const result = analyzer.lint(query, { + dataSourceVersion: sqlVersion, + isCalcite: true, + }); + const matches = result.diagnostics.filter((diagnostic) => diagnostic.ruleId === ruleId); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] ${testCase.id}: expected ${testCase.frontendDiagnosticCount}, ` + + `got ${matches.length} — ${query}` + ); + + try { + assert.strictEqual( + matches.length, + testCase.frontendDiagnosticCount, + `case "${testCase.id}": expected ${testCase.frontendDiagnosticCount} "${ruleId}" ` + + `diagnostic(s) but received ${matches.length} for query: ${query}` + ); + } catch (error) { + failures.push(error.message); + } + } + + if (failures.length > 0) { + fail(`${failures.length} case(s) failed:\n- ${failures.join('\n- ')}`); + } + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] PASS: all ${contract.cases.length} case(s) matched for "${ruleId}".` + ); +} + +main(); From 738672e1d09254e4ca53c9668e5fecb58ee599e9 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 17:41:10 -0700 Subject: [PATCH 39/41] style(ci): use $(...) instead of legacy backticks in ppl-lint workflow Addresses shellcheck SC2006/SC2046 on the chown/su lines so actionlint runs clean. Behavior is unchanged. Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-rule-validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 61ee0be6fb8..4d545ea677b 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -133,8 +133,8 @@ jobs: - name: Run backend integration test (live /_plugins/_ppl) run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" - name: Upload failure artifacts if: ${{ failure() }} From 74fcfaa6a5e6ece5c87752f02b767a428a436936 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 20:59:33 -0700 Subject: [PATCH 40/41] ci: re-trigger PPL lint rule validation after Actions recovery Signed-off-by: Hanyu Wei From 9aca44c40e296278a8fda1afffe6a2bc04c6018c Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 22:01:59 -0700 Subject: [PATCH 41/41] fix(ci): split ppl-lint validation into frontend/backend jobs The OpenSearch CI container is Amazon Linux 2 (glibc 2.26), but OSD requires Node 22 whose prebuilt binary needs glibc >= 2.27. Running the Node frontend contract inside that container failed with 'GLIBC_2.27 not found'. Split into two required jobs: 'frontend' runs the OSD analyzer contract on a bare ubuntu-latest runner (modern glibc, actions/setup-node works), and 'backend' keeps the Gradle integration test in the CI container where the OpenSearch test cluster needs it. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 4d545ea677b..9e9d5e11dc8 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -3,10 +3,17 @@ name: PPL lint rule validation # Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule # `unsupported-window-function-in-eventstats` and the SQL backend must agree. # -# Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an -# OSD checkout and asserts the rule's diagnostic counts. -# Backend half: a Gradle integration test sends the same queries to the live -# `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. +# Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD +# analyzer from an OSD checkout and asserts the rule's diagnostic counts. This +# runs on a bare ubuntu-latest runner because OSD requires a modern Node whose +# prebuilt binary needs a newer glibc than the OpenSearch CI container (Amazon +# Linux 2) provides. +# Backend half (backend job): a Gradle integration test sends the same queries +# to the live `/_plugins/_ppl` endpoint of the SQL plugin built from this +# checkout. This runs inside the OpenSearch CI container because the Gradle +# test cluster needs it. +# +# Both jobs are required; a failure on either side fails the SQL PR check. # # The OSD detector is loaded from `main` by default (PR and nightly runs), so a # removed or changed detector is detected. `workflow_dispatch` can target a @@ -24,23 +31,10 @@ on: type: string jobs: - Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main - with: - product: opensearch - - ppl-lint-rule-validation: - name: PPL lint rule validation - needs: Get-CI-Image-Tag + frontend: + name: Frontend contract (OSD analyzer) runs-on: ubuntu-latest - container: - image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} - steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -58,49 +52,33 @@ jobs: path: .ci/OpenSearch-Dashboards - name: Record OSD revision - id: osd-sha run: | sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) - echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" - - name: Set up JDK 21 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 - with: - distribution: 'temurin' - java-version: 21 - # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding # it, so an OSD toolchain bump does not silently drift this job. - - name: Read OSD Node version - id: osd-node - run: echo "version=$(cat .ci/OpenSearch-Dashboards/.nvmrc)" >> "$GITHUB_OUTPUT" - - - name: Set up Node ${{ steps.osd-node.outputs.version }} + - name: Set up Node from OSD .nvmrc uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: - node-version: ${{ steps.osd-node.outputs.version }} + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - - name: Read OSD Yarn version - id: osd-yarn + - name: Pin Yarn from OSD engines working-directory: .ci/OpenSearch-Dashboards run: | yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') - echo "version=$yarn_version" >> "$GITHUB_OUTPUT" - - - name: Pin Yarn ${{ steps.osd-yarn.outputs.version }} - run: npm install -g "yarn@${{ steps.osd-yarn.outputs.version }}" + npm install -g "yarn@${yarn_version}" - name: Cache OSD Yarn dependencies uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 with: path: | ~/.cache/yarn - key: ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} restore-keys: | - ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}- + ${{ runner.os }}-osd-yarn- - name: Bootstrap OpenSearch-Dashboards working-directory: .ci/OpenSearch-Dashboards @@ -118,10 +96,7 @@ jobs: echo "version=$version" >> "$GITHUB_OUTPUT" echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" - # The frontend contract only runs Node (no OpenSearch cluster), so it runs - # as the default container user. Only the Gradle integTest below must run - # as a non-root user, because OpenSearch refuses to start as root. - - name: Run frontend contract (OSD analyzer) + - name: Run frontend contract working-directory: .ci/OpenSearch-Dashboards env: PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json @@ -131,7 +106,42 @@ jobs: "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ | tee "$GITHUB_WORKSPACE/frontend-contract.log" - - name: Run backend integration test (live /_plugins/_ppl) + - name: Upload frontend log + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-frontend-contract-log + path: frontend-contract.log + + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend: + name: Backend integration test (live /_plugins/_ppl) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. + - name: Run backend integration test run: | chown -R 1000:1000 "$(pwd)" su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" @@ -141,8 +151,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true with: - name: ppl-lint-rule-validation-artifacts + name: ppl-lint-backend-artifacts path: | - frontend-contract.log integ-test/build/reports/** integ-test/build/testclusters/*/logs/*