From bc169b486512523cd4b60344b377d3ed6ffe017b Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Fri, 17 Jul 2026 13:12:25 -0700 Subject: [PATCH 01/78] Revert "[Feature] Add PPL `rest` command" (#5635) This reverts the changes introduced by PR #5599 (merge commit 454ac4e7dd87d1f4120d5eed4bf0b0a8893d5838). --- .../sql/common/setting/Settings.java | 2 - .../opensearch/sql/ast/tree/RestRelation.java | 25 - .../sql/utils/SystemIndexUtils.java | 118 ----- docs/category.json | 1 - docs/user/ppl/cmd/rest.md | 94 ---- docs/user/ppl/index.md | 1 - doctest/build.gradle | 5 - integ-test/build.gradle | 10 - .../sql/calcite/CalciteNoPushdownIT.java | 1 - .../sql/calcite/remote/CalciteExplainIT.java | 9 - .../sql/calcite/remote/CalcitePPLRestIT.java | 210 --------- .../sql/plugin/AnalyticsEngineCompatIT.java | 31 -- .../sql/ppl/NewAddedCommandsIT.java | 13 - .../sql/security/RestCommandSecurityIT.java | 154 ------- .../opensearch/sql/sql/VectorSearchIT.java | 1 - .../opensearch/client/OpenSearchClient.java | 101 ----- .../client/OpenSearchNodeClient.java | 273 ----------- .../client/OpenSearchRestClient.java | 268 ----------- .../rules/EnumerableCatalogScanRule.java | 63 --- .../rules/EnumerableSystemIndexScanRule.java | 50 ++ .../planner/rules/OpenSearchIndexRules.java | 6 +- .../setting/OpenSearchSettings.java | 27 +- .../storage/OpenSearchStorageEngine.java | 31 +- .../storage/rest/RestCatalogSource.java | 63 --- .../storage/rest/RestEndpointRegistry.java | 428 ------------------ .../opensearch/storage/rest/RestRequest.java | 57 --- .../storage/rest/RestResponseRedactor.java | 64 --- .../rest/RestSettingsFilterHolder.java | 40 -- ...va => AbstractCalciteSystemIndexScan.java} | 12 +- ... => CalciteEnumerableSystemIndexScan.java} | 23 +- ...ava => CalciteLogicalSystemIndexScan.java} | 20 +- .../system/CalciteScannableCatalogScan.java | 29 -- .../storage/system/CatalogSource.java | 39 -- .../system/OpenSearchCatalogTable.java | 67 --- .../storage/system/OpenSearchSystemIndex.java | 105 +++++ ...a => OpenSearchSystemIndexEnumerator.java} | 9 +- .../system/SystemIndexCatalogSource.java | 70 --- ...chNodeClientClusterSettingsFilterTest.java | 99 ---- .../setting/OpenSearchSettingsTest.java | 12 - .../storage/OpenSearchStorageEngineTest.java | 71 +-- .../storage/rest/RestCatalogSourceTest.java | 123 ----- .../rest/RestEndpointRegistryTest.java | 290 ------------ .../rest/RestResponseRedactorTest.java | 98 ---- ...st.java => OpenSearchSystemIndexTest.java} | 25 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 4 - .../plugin/rest/RestUnifiedQueryAction.java | 9 +- .../rest/RestUnifiedQueryActionTest.java | 6 - ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 - ppl/src/main/antlr/OpenSearchPPLParser.g4 | 14 - .../opensearch/sql/ppl/parser/AstBuilder.java | 33 -- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 19 - .../sql/ppl/calcite/CalcitePPLRestTest.java | 67 --- .../sql/ppl/parser/AstBuilderTest.java | 28 -- .../ppl/utils/PPLQueryDataAnonymizerTest.java | 12 - 54 files changed, 206 insertions(+), 3226 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java delete mode 100644 docs/user/ppl/cmd/rest.md delete mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{AbstractCalciteCatalogScan.java => AbstractCalciteSystemIndexScan.java} (68%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteEnumerableCatalogScan.java => CalciteEnumerableSystemIndexScan.java} (80%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteLogicalCatalogScan.java => CalciteLogicalSystemIndexScan.java} (59%) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchCatalogEnumerator.java => OpenSearchSystemIndexEnumerator.java} (90%) delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java delete mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchCatalogTableTest.java => OpenSearchSystemIndexTest.java} (76%) delete mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 5473aa8812e..bf3e65d8741 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,8 +36,6 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), - PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), - PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java deleted file mode 100644 index d0a0e34d3c9..00000000000 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ast.tree; - -import java.util.Collections; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.opensearch.sql.ast.expression.UnresolvedExpression; - -/** - * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, - * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that - * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source - * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. - */ -@ToString -@EqualsAndHashCode(callSuper = false) -public class RestRelation extends Relation { - public RestRelation(UnresolvedExpression tableName) { - super(Collections.singletonList(tableName)); - } -} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 7589cf522f6..c9fa35d7068 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,9 +5,6 @@ package org.opensearch.sql.utils; -import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; -import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -37,121 +34,6 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } - /** - * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} - * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a - * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution - * the same way the uppercase system-mapping names do. - */ - private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; - - private static final String REST_SOURCE_PREFIX = "REST"; - - /** True if the resolved table name is a {@code rest} source token. */ - public static boolean isRestSource(String indexName) { - return indexName.endsWith(REST_SOURCE_SUFFIX); - } - - /** - * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link - * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. - * The endpoint/args have already been allow-list-validated before this is called. - */ - public static String restTable(RestSpec spec) { - StringBuilder sb = new StringBuilder(); - sb.append("endpoint=").append(spec.getEndpoint()); - if (spec.getCount() != null) { - sb.append('\n').append("count=").append(spec.getCount()); - } - if (spec.getTimeout() != null) { - sb.append('\n').append("timeout=").append(spec.getTimeout()); - } - if (spec.getArgs() != null) { - for (Map.Entry e : spec.getArgs().entrySet()) { - sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); - } - } - return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; - } - - /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ - public static RestSpec decodeRestSpec(String indexName) { - // Validate the token shape before slicing it: callers gate on isRestSource today, but a - // public decoder must not assume its precondition, otherwise a malformed token would throw an - // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. - if (!isRestSource(indexName)) { - throw new IllegalArgumentException("not a valid rest source token: " + indexName); - } - String body = - indexName.substring( - REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); - String decoded = fromHex(body); - String endpoint = null; - Integer count = null; - String timeout = null; - LinkedHashMap args = new LinkedHashMap<>(); - for (String line : decoded.split("\n")) { - if (line.isEmpty()) { - continue; - } - int eq = line.indexOf('='); - if (eq < 0) { - continue; - } - String k = line.substring(0, eq); - String v = line.substring(eq + 1); - if (k.equals("endpoint")) { - endpoint = v; - } else if (k.equals("count")) { - count = Integer.parseInt(v); - } else if (k.equals("timeout")) { - timeout = v; - } else if (k.startsWith("arg.")) { - args.put(k.substring("arg.".length()), v); - } - } - if (endpoint == null) { - throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); - } - return new RestSpec(endpoint, args, count, timeout); - } - - private static String toHex(String s) { - StringBuilder h = new StringBuilder(); - for (byte b : s.getBytes(StandardCharsets.UTF_8)) { - h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); - } - return h.toString(); - } - - private static String fromHex(String h) { - if (h.length() % 2 != 0) { - throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); - } - byte[] bytes = new byte[h.length() / 2]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = - (byte) - ((Character.digit(h.charAt(2 * i), 16) << 4) - + Character.digit(h.charAt(2 * i + 1), 16)); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - /** - * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional - * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) - * share it without a cross-module dependency. - */ - @Getter - @RequiredArgsConstructor - public static class RestSpec { - private final String endpoint; - private final Map args; - private final Integer count; - private final String timeout; - } - /** * Compose system mapping table. * diff --git a/docs/category.json b/docs/category.json index a8665b82eac..7296b089650 100644 --- a/docs/category.json +++ b/docs/category.json @@ -35,7 +35,6 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", - "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md deleted file mode 100644 index a9761aa87e8..00000000000 --- a/docs/user/ppl/cmd/rest.md +++ /dev/null @@ -1,94 +0,0 @@ -# rest - -The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. - -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. - -## Enabling the command - -The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): - -```yaml -plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] -``` - -Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. - -The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. - -## Syntax - -The `rest` command has the following syntax: - -```syntax -rest [count=] [timeout=] [= ...] -``` - -## Parameters - -The `rest` command supports the following parameters. - -| Parameter | Required/Optional | Description | -| --- | --- | --- | -| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | -| `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | -| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | - -## Allow-list - -`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. - -| Endpoint | Output columns | Accepted args | -| --- | --- | --- | -| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | -| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | -| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | -| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | -| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | -| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | -| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | -| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | - -## Example 1: Counting the nodes in the cluster - -The following query reads cluster health and projects a column that is deterministic on a single-node cluster: - -```ppl -| rest '/_cluster/health' | fields number_of_nodes -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+-----------------+ -| number_of_nodes | -|-----------------| -| 1 | -+-----------------+ -``` - -`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. - -## Example 2: Composing downstream commands over a cat endpoint - -The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: - -```ppl -| rest '/_cat/cluster_manager' | stats count() as managers -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+----------+ -| managers | -|----------| -| 1 | -+----------+ -``` - -For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 939684c0ecc..37947113800 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,7 +74,6 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/doctest/build.gradle b/doctest/build.gradle index 1ac658457dc..cce64170f56 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,11 +205,6 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' - // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into - // the registered endpoints so the rest.md examples run against an enabled command. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c18fa6e37f6..1435d1d499d 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,11 +387,6 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" - // The rest command is disabled by default (empty allow-list). Opt this cluster into the - // registered endpoints so the rest integration tests exercise the enabled path. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -410,9 +405,6 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" - // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -427,8 +419,6 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" - // Composite-default cluster: PPL queries route to the analytics engine unless excluded. - setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 801d56fd49d..907f91fa98e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -58,7 +58,6 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, - CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 4ab1d40a5af..b78a71e534c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -66,15 +66,6 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. - @Test - public void explainRestCommand() throws IOException { - String result = explainQueryToString("| rest '/_cluster/health' | fields status"); - Assert.assertTrue( - "Expected a rest scan node in the explain output, got: " + result, - result.contains("CatalogScan")); - } - @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java deleted file mode 100644 index 70ee4ef0de6..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.calcite.remote; - -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; -import org.opensearch.client.ResponseException; -import org.opensearch.sql.ppl.PPLIntegTestCase; - -/** - * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also - * verifies that a non-allow-listed / mutating endpoint is refused. - */ -public class CalcitePPLRestIT extends PPLIntegTestCase { - - @Override - public void init() throws Exception { - super.init(); - enableCalcite(); - } - - @Test - public void testRestClusterHealthSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); - verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); - } - - @Test - public void testRestClusterHealthDataRows() throws IOException { - // Single-node test cluster: exactly one node, status is green or yellow. - JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestRejectsNonAllowListedEndpoint() throws IOException { - assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); - } - - @Test - public void testRestRejectsEmptyEndpoint() throws IOException { - assertRestBadRequest("| rest ''", "non-empty path"); - } - - @Test - public void testRestRejectsDisallowedArg() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); - } - - @Test - public void testRestRejectsNegativeCount() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); - } - - @Test - public void testRestRejectsTimeoutArg() throws IOException { - assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); - } - - /** - * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) - * with the given substring in the response body. Covers allow-list and bad-argument rejection. - */ - private void assertRestBadRequest(String query, String expectedSubstring) { - ResponseException e = - org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); - org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); - org.junit.Assert.assertTrue( - "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), - e.getMessage().contains(expectedSubstring)); - } - - @Test - public void testRestCatIndicesSchema() throws IOException { - // Schema is fixed by the registry, independent of how many indices exist. - JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); - verifySchema(result, schema("index", "string"), schema("health", "string")); - } - - @Test - public void testRestCatIndicesReturnsCreatedIndex() throws IOException { - // Create a known index, then confirm rest surfaces it and downstream where/fields compose. - client().performRequest(new Request("PUT", "/rest_cat_test")); - JSONObject result = - executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); - verifyDataRows(result, rows("rest_cat_test")); - } - - @Test - public void testRestCatNodesSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); - verifySchema(result, schema("name", "string"), schema("cpu", "int")); - } - - @Test - public void testRestCatNodesSingleNode() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatClusterManagerSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); - verifySchema(result, schema("node", "string"), schema("id", "string")); - } - - @Test - public void testRestCatClusterManagerSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatPluginsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); - verifySchema(result, schema("component", "string"), schema("version", "string")); - } - - @Test - public void testRestCatShardsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); - verifySchema( - result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); - } - - @Test - public void testRestClusterStateSchema() throws IOException { - // Assert the string columns; version is the LONG epoch column. - JSONObject result = - executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); - verifySchema( - result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); - } - - @Test - public void testRestClusterStateSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestClusterSettingsSchema() throws IOException { - // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); - verifySchema( - result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); - } - - @Test - public void testRestResolveIndexSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { - client().performRequest(new Request("PUT", "/rest_resolve_test")); - JSONObject result = - executeQuery( - "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); - verifyDataRows(result, rows("rest_resolve_test", "index")); - } - - // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- - - @Test - public void testRestClusterHealthLocalArg() throws IOException { - // local=true reads health from the local node; on a single-node cluster the row is unchanged. - JSONObject result = - executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { - // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); - verifyDataRows(result, rows(0)); - } - - @Test - public void testRestResolveIndexExpandWildcardsArg() throws IOException { - // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. - JSONObject result = - executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestRejectsDroppedLevelArg() throws IOException { - // level was dropped from the allow-list (no-op against the fixed health schema). - assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); - } - - @Test - public void testRestRejectsBadArgValue() throws IOException { - assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index b5c08cc8ad0..f6ec903c395 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,13 +5,9 @@ package org.opensearch.sql.plugin; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; -import org.json.JSONArray; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -60,31 +56,4 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } - - /** - * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it - * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its - * fixed schema and correct data unchanged when the analytics-engine plugin is present. - */ - @Test - public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { - Request request = new Request("POST", "/_plugins/_ppl"); - request.setJsonEntity( - "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); - Response response = client().performRequest(request); - assertEquals(200, response.getStatusLine().getStatusCode()); - - JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); - - JSONArray schema = result.getJSONArray("schema"); - assertEquals(2, schema.length()); - assertEquals("status", schema.getJSONObject(0).getString("name")); - assertEquals("string", schema.getJSONObject(0).getString("type")); - assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); - assertEquals("int", schema.getJSONObject(1).getString("type")); - - JSONArray datarows = result.getJSONArray("datarows"); - assertEquals(1, datarows.length()); - assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 2ccda31eea7..837865a3585 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,19 +32,6 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - @Test - public void testRest() throws IOException { - JSONObject result; - try { - result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - } - if (isCalciteEnabled()) { - assertFalse(result.getJSONArray("datarows").isEmpty()); - } - } - @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java deleted file mode 100644 index 042a3aefbf7..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.security; - -import static org.opensearch.sql.util.MatcherUtils.columnName; -import static org.opensearch.sql.util.MatcherUtils.verifyColumn; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; -import org.opensearch.client.Request; -import org.opensearch.client.ResponseException; -import org.opensearch.sql.legacy.TestUtils; - -/** - * Integration tests that verify the rest command is subject to the security plugin fine grained - * access control. The command dispatches standard transport actions under the caller identity, so - * the security ActionFilter authorizes each one by action name. A caller without the required - * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege - * can run them, and the resolve index endpoint requires the resolve index privilege because the - * command resolves all indices. - */ -public class RestCommandSecurityIT extends SecurityTestBase { - - private static final String ALPHA_INDEX = "rest_sec_alpha"; - private static final String BETA_INDEX = "rest_sec_beta"; - - private static final String MONITOR_USER = "rest_monitor_user"; - private static final String MONITOR_ROLE = "rest_monitor_role"; - - private static final String NO_MONITOR_USER = "rest_no_monitor_user"; - private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; - - @Override - protected void init() throws Exception { - super.init(); - setupRolesUsersAndIndices(); - enableCalcite(); - // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported - // command error. Disable fallback so the denial reason surfaces to the caller. - disallowCalciteFallback(); - } - - private void setupRolesUsersAndIndices() throws IOException { - createIndexIfAbsent(ALPHA_INDEX); - createIndexIfAbsent(BETA_INDEX); - - createRoleWithPermissions( - MONITOR_ROLE, - "*", - new String[] { - "cluster:admin/opensearch/ppl", - "cluster:monitor/health", - "cluster:monitor/state", - "cluster:monitor/nodes/stats", - "cluster:monitor/nodes/info" - }, - new String[] {"indices:admin/resolve/index"}); - createUser(MONITOR_USER, MONITOR_ROLE); - - createRoleWithPermissions( - NO_MONITOR_ROLE, - ALPHA_INDEX, - new String[] {"cluster:admin/opensearch/ppl"}, - new String[] {"indices:data/read/search*"}); - createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); - } - - @Test - public void monitorUserCanRunCatNodes() throws IOException { - JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); - verifyColumn(result, columnName("name")); - } - - @Test - public void monitorUserCanResolveIndex() throws IOException { - JSONObject result = - executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); - Set names = resolvedNames(result); - assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); - assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); - } - - @Test - public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { - assertDenied( - "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); - } - - @Test - public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { - assertDenied( - "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); - } - - @Test - public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { - assertDenied( - "| rest '/_resolve/index' | fields name, type", - NO_MONITOR_USER, - "indices:admin/resolve/index"); - } - - /** - * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on - * the Calcite only rest path surfaces as a client or server error whose body carries the security - * denial reason, so this checks the denial signal rather than a fixed status code. - */ - private void assertDenied(String query, String user, String deniedAction) throws IOException { - try { - executeQueryAsUser(query, user); - fail("Expected a permission denial for user without privilege: " + user); - } catch (ResponseException e) { - int status = e.getResponse().getStatusLine().getStatusCode(); - String body = TestUtils.getResponseBody(e.getResponse(), false); - assertTrue("Expected an error status, got " + status, status >= 400); - assertTrue( - "Response should indicate a permission denial. Status " + status + ", body: " + body, - body.contains("no permissions") - || body.contains("Forbidden") - || body.contains("security_exception") - || body.contains(deniedAction)); - } - } - - private void createIndexIfAbsent(String name) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity( - "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); - try { - client().performRequest(request); - } catch (ResponseException e) { - String body = TestUtils.getResponseBody(e.getResponse(), false); - if (!body.contains("resource_already_exists_exception")) { - throw e; - } - } - } - - private Set resolvedNames(JSONObject result) { - Set names = new HashSet<>(); - JSONArray datarows = result.getJSONArray("datarows"); - for (int i = 0; i < datarows.length(); i++) { - names.add(datarows.getJSONArray(i).getString(0)); - } - return names; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index c10bc474bae..f3c3046f41e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -28,7 +28,6 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { - super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 3b9c3619521..68350c5a0fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,105 +114,4 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); - - /** - * Read-only cluster health snapshot for the {@code rest} command (backs {@code - * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's - * security thread-context; performs no privilege escalation and mutates nothing. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of health field name to value - */ - default Map clusterHealth(Map params) { - throw new UnsupportedOperationException("clusterHealth is not supported by this client"); - } - - /** - * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One - * map per index. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per index - */ - default List> catIndices(Map params) { - throw new UnsupportedOperationException("catIndices is not supported by this client"); - } - - /** - * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map - * per node with resource state. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per node - */ - default List> catNodes(Map params) { - throw new UnsupportedOperationException("catNodes is not supported by this client"); - } - - /** - * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code - * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map describing the cluster manager node - */ - default List> catClusterManager(Map params) { - throw new UnsupportedOperationException("catClusterManager is not supported by this client"); - } - - /** - * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One - * map per installed plugin per node. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per plugin - */ - default List> catPlugins(Map params) { - throw new UnsupportedOperationException("catPlugins is not supported by this client"); - } - - /** - * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map - * per shard. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per shard - */ - default List> catShards(Map params) { - throw new UnsupportedOperationException("catShards is not supported by this client"); - } - - /** - * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code - * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, - * cluster_manager_node). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of cluster-state field name to value - */ - default Map clusterState(Map params) { - throw new UnsupportedOperationException("clusterState is not supported by this client"); - } - - /** - * Read-only cluster-settings listing for the {@code rest} command (backs {@code - * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per setting - */ - default List> clusterSettings(Map params) { - throw new UnsupportedOperationException("clusterSettings is not supported by this client"); - } - - /** - * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). - * One map per resolved index, alias, or data stream (name, type). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per resolved name - */ - default List> resolveIndex(Map params) { - throw new UnsupportedOperationException("resolveIndex is not supported by this client"); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 080a8627894..b491f38ef80 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,8 +18,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -27,7 +25,6 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -288,274 +285,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); - return flattenHealth(response); - } - - @Override - public List> catIndices(Map params) { - ClusterHealthResponse response = - client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new java.util.LinkedHashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } - - @Override - public List> catNodes(Map params) { - org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = - new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); - statsRequest.all(); - org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = - client.admin().cluster().nodesStats(statsRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { - org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); - Map row = new java.util.LinkedHashMap<>(); - row.put("name", node.getName()); - row.put("ip", node.getHostAddress()); - row.put( - "node_role", - node.getRoles().stream() - .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) - .sorted() - .collect(java.util.stream.Collectors.joining(","))); - row.put( - "heap_percent", - ns.getJvm() == null || ns.getJvm().getMem() == null - ? null - : (int) ns.getJvm().getMem().getHeapUsedPercent()); - row.put( - "ram_percent", - ns.getOs() == null || ns.getOs().getMem() == null - ? null - : (int) ns.getOs().getMem().getUsedPercent()); - row.put( - "cpu", - ns.getProcess() == null || ns.getProcess().getCpu() == null - ? null - : (int) ns.getProcess().getCpu().getPercent()); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - List> rows = new java.util.ArrayList<>(); - if (cm != null) { - Map row = new java.util.LinkedHashMap<>(); - row.put("id", cm.getId()); - row.put("host", cm.getHostName()); - row.put("ip", cm.getHostAddress()); - row.put("node", cm.getName()); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = - new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); - infoRequest.all(); - org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = - client.admin().cluster().nodesInfo(infoRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { - org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = - info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); - if (plugins == null) { - continue; - } - for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", info.getNode().getName()); - row.put("component", pi.getName()); - row.put("version", pi.getVersion()); - rows.add(row); - } - } - return rows; - } - - @Override - public List> catShards(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.cluster.routing.ShardRouting sr : - response.getState().getRoutingTable().allShards()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("index", sr.getIndexName()); - row.put("shard", sr.id()); - row.put("prirep", sr.primary() ? "p" : "r"); - row.put("state", sr.state().name()); - org.opensearch.cluster.node.DiscoveryNode n = - sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); - row.put("node", n == null ? null : n.getName()); - rows.add(row); - } - return rows; - } - - @Override - public Map clusterState(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName().value()); - row.put("state_uuid", response.getState().stateUUID()); - row.put("version", response.getState().version()); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - row.put("cluster_manager_node", cm == null ? null : cm.getName()); - return row; - } - - @Override - public List> clusterSettings(Map params) { - // The transport path has no SettingsFilter of its own; it is published from - // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings - // into memory when we cannot redact them, matching native GET /_cluster/settings. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - org.opensearch.common.settings.Settings persistent = - filter.filter(response.getState().metadata().persistentSettings()); - org.opensearch.common.settings.Settings transientSettings = - filter.filter(response.getState().metadata().transientSettings()); - collectSettings(persistent, "persistent", rows); - collectSettings(transientSettings, "transient", rows); - return rows; - } - - private void collectSettings( - org.opensearch.common.settings.Settings settings, - String tier, - List> rows) { - if (settings == null) { - return; - } - for (String key : settings.keySet()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("setting", key); - String value = settings.get(key); - if (value == null) { - // List-valued settings return null from get(); fall back to the joined list form. - java.util.List list = settings.getAsList(key); - value = list.isEmpty() ? null : String.join(",", list); - } - row.put("value", value); - row.put("tier", tier); - rows.add(row); - } - } - - @Override - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = - expandWildcards == null - ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}) - : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}, - org.opensearch.action.support.IndicesOptions.fromParameters( - expandWildcards, - null, - null, - null, - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request - .DEFAULT_INDICES_OPTIONS)); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = - client - .execute( - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : - response.getIndices()) { - rows.add(resolveRow(idx.getName(), "index")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : - response.getAliases()) { - rows.add(resolveRow(alias.getName(), "alias")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : - response.getDataStreams()) { - rows.add(resolveRow(ds.getName(), "data_stream")); - } - return rows; - } - - private Map resolveRow(String name, String type) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", name); - row.put("type", type); - return row; - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index e98c5bf95f4..f369c0003b8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,19 +8,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -32,7 +28,6 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -277,267 +272,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - try { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); - return flattenHealth(response); - } catch (IOException e) { - throw new IllegalStateException("Failed to get cluster health", e); - } - } - - @Override - public List> catIndices(Map params) { - try { - ClusterHealthResponse response = - client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - List> rows = new ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new HashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } catch (IOException e) { - throw new IllegalStateException("Failed to get cat indices", e); - } - } - - @Override - public List> catNodes(Map params) { - List> raw = - catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("ip", r.get("ip")); - row.put("node_role", r.get("node.role")); - row.put("heap_percent", asInt(r.get("heap.percent"))); - row.put("ram_percent", asInt(r.get("ram.percent"))); - row.put("cpu", asInt(r.get("cpu"))); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("id", r.get("id")); - row.put("host", r.get("host")); - row.put("ip", r.get("ip")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - List> raw = catJson("/_cat/plugins", "name,component,version"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("component", r.get("component")); - row.put("version", r.get("version")); - rows.add(row); - } - return rows; - } - - @Override - public List> catShards(Map params) { - List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("index", r.get("index")); - row.put("shard", asInt(r.get("shard"))); - row.put("prirep", r.get("prirep")); - row.put("state", r.get("state")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ - @SuppressWarnings("unchecked") - private List> catJson(String path, String columns) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - request.addParameter("format", "json"); - request.addParameter("h", columns); - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - List list = parser.list(); - List> rows = new ArrayList<>(); - for (Object o : list) { - rows.add((Map) o); - } - return rows; - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Integer asInt(Object value) { - if (value == null) { - return null; - } - try { - return (int) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public Map clusterState(Map params) { - Map state = - getJsonMap( - "/_cluster/state/master_node,version,metadata,nodes", - // nodes.*.name resolves the manager id to a name without over-fetching node IPs. - Map.of( - "filter_path", - "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); - Map row = new HashMap<>(); - row.put("cluster_name", state.get("cluster_name")); - row.put("state_uuid", state.get("state_uuid")); - row.put("version", asLong(state.get("version"))); - Object cmId = state.get("cluster_manager_node"); - String cmName = null; - Object nodes = state.get("nodes"); - if (cmId != null && nodes instanceof Map) { - Object n = ((Map) nodes).get(cmId.toString()); - if (n instanceof Map) { - Object name = ((Map) n).get("name"); - cmName = name == null ? null : name.toString(); - } - } - row.put("cluster_manager_node", cmName); - return row; - } - - @Override - @SuppressWarnings("unchecked") - public List> clusterSettings(Map params) { - Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); - List> rows = new ArrayList<>(); - for (String tier : new String[] {"persistent", "transient"}) { - Object section = body.get(tier); - if (section instanceof Map) { - for (Map.Entry e : ((Map) section).entrySet()) { - Map row = new HashMap<>(); - row.put("setting", e.getKey()); - row.put("value", e.getValue() == null ? null : e.getValue().toString()); - row.put("tier", tier); - rows.add(row); - } - } - } - return rows; - } - - /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ - @SuppressWarnings("unchecked") - private Map getJsonMap(String path, Map params) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - if (params != null) { - params.forEach(request::addParameter); - } - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - return parser.map(); - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Long asLong(Object value) { - if (value == null) { - return null; - } - try { - return (long) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - Map body = - getJsonMap( - "/_resolve/index/*", - expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); - List> rows = new ArrayList<>(); - addResolved(body.get("indices"), "index", rows); - addResolved(body.get("aliases"), "alias", rows); - addResolved(body.get("data_streams"), "data_stream", rows); - return rows; - } - - @SuppressWarnings("unchecked") - private void addResolved(Object section, String type, List> rows) { - if (section instanceof List) { - for (Object o : (List) section) { - if (o instanceof Map) { - Map row = new HashMap<>(); - row.put("name", ((Map) o).get("name")); - row.put("type", type); - rows.add(row); - } - } - } - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new HashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java deleted file mode 100644 index 9905ca564ab..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; -import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; - -/** - * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link - * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain - * {@link CalciteEnumerableCatalogScan}. - */ -public class EnumerableCatalogScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalCatalogScan.class, - s -> s.getCatalogTable() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableCatalogScanRule") - .withRuleFactory(EnumerableCatalogScanRule::new); - - protected EnumerableCatalogScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalCatalogScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; - if (scan.getCatalogTable().getSource().isScannable()) { - return new CalciteScannableCatalogScan( - scan.getCluster(), - scan.getHints(), - scan.getTable(), - scan.getCatalogTable(), - scan.getSchema()); - } - return new CalciteEnumerableCatalogScan( - scan.getCluster(), - scan.getHints(), - scan.getTable(), - scan.getCatalogTable(), - scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java new file mode 100644 index 00000000000..616d1178873 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; + +/** + * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link + * CalciteEnumerableSystemIndexScan}. + */ +public class EnumerableSystemIndexScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalSystemIndexScan.class, + s -> s.getSysIndex() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableSystemIndexScanRule") + .withRuleFactory(EnumerableSystemIndexScanRule::new); + + /** Creates an EnumerableProjectRule. */ + protected EnumerableSystemIndexScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalSystemIndexScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; + return new CalciteEnumerableSystemIndexScan( + scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index c200ffa8909..3c8508cc455 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule CATALOG_SCAN_RULE = - EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = + EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - CATALOG_SCAN_RULE, + SYSTEM_INDEX_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index ce2bdd4960b..b596c7bc47a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,17 +71,6 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = - Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); - - public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = - Setting.listSetting( - Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of(), - Function.identity(), - Setting.Property.NodeScope); - public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -391,16 +380,6 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_ALLOWED_ENDPOINTS, - PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -661,9 +640,7 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - if (clusterSettings.get(setting) != null) { - latestSettings.put(key, clusterSettings.get(setting)); - } + latestSettings.put(key, clusterSettings.get(setting)); } /** @@ -734,8 +711,6 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) - .add(PPL_REST_REDACTION_ENABLED_SETTING) - .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 7b911471242..1b7de315fb6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,8 +5,6 @@ package org.opensearch.sql.opensearch.storage; -import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; -import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -17,13 +15,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; -import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; -import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; -import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; +import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -41,29 +35,10 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isRestSource(name)) { - return restTable(name); - } else if (isSystemIndex(name)) { - return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + if (isSystemIndex(name)) { + return new OpenSearchSystemIndex(client, settings, name); } else { return new OpenSearchIndex(client, settings, name); } } - - private Table restTable(String name) { - RestSpec spec = decodeRestSpec(name); - RestEndpointRegistry.resolve(spec.getEndpoint()); - List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); - if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { - throw new IllegalArgumentException( - allowed == null || allowed.isEmpty() - ? "the rest command is disabled on this cluster" - : "rest endpoint [" - + spec.getEndpoint() - + "] is not enabled on this cluster. Enabled endpoints: " - + allowed); - } - boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); - return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java deleted file mode 100644 index 96779726a7e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Map; -import lombok.Getter; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.opensearch.storage.system.CatalogSource; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management - * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. - * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. - */ -@Getter -public class RestCatalogSource implements CatalogSource { - - private final OpenSearchClient client; - private final RestSpec spec; - private final RestEndpointRegistry.Endpoint endpoint; - private final boolean redact; - - public RestCatalogSource(OpenSearchClient client, RestSpec spec) { - this(client, spec, false); - } - - public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { - this.client = client; - this.spec = spec; - this.redact = redact; - // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); - } - - @Override - public Map getFieldTypes() { - return endpoint.getSchema(); - } - - @Override - public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec, redact); - } - - @Override - public boolean isScannable() { - return true; - } - - @Override - public PhysicalPlan implementV2(LogicalPlan plan) { - throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java deleted file mode 100644 index e64a91ab54a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; -import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; -import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.model.ExprValueUtils.longValue; -import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.LONG; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import lombok.Getter; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps - * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so - * the Calcite plan can fix its row type at plan time), and the query args it accepts. - * - *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, - * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. - * Adding an endpoint is a reviewed change here, never arbitrary pass-through. - */ -public final class RestEndpointRegistry { - - private RestEndpointRegistry() {} - - /** Produces the raw rows for an endpoint via a read-only client call. */ - @FunctionalInterface - public interface RowFetcher { - List> fetch(OpenSearchClient client, RestSpec spec); - } - - /** A single allow-listed endpoint description. */ - @Getter - public static final class Endpoint { - private final String path; - private final LinkedHashMap schema; - private final Set allowedArgs; - private final RowFetcher fetcher; - - Endpoint( - String path, - LinkedHashMap schema, - Set allowedArgs, - RowFetcher fetcher) { - this.path = path; - this.schema = schema; - this.allowedArgs = allowedArgs; - this.fetcher = fetcher; - } - - /** Dispatch the read-only call and shape the response into fixed-schema rows. */ - public List toRows(OpenSearchClient client, RestSpec spec) { - return toRows(client, spec, false); - } - - /** - * Shape the response into fixed-schema rows, masking network identifiers when redaction is - * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value - * column is zone-masked. Off by default. - */ - public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { - boolean redactCat = redact && path.startsWith("/_cat"); - boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); - List out = new ArrayList<>(); - for (Map raw : fetcher.fetch(client, spec)) { - LinkedHashMap tuple = new LinkedHashMap<>(); - for (Map.Entry col : schema.entrySet()) { - ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); - tuple.put( - col.getKey(), - maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); - } - out.add(new ExprTupleValue(tuple)); - } - return out; - } - - private static ExprValue maskCell( - String column, - ExprType type, - ExprValue value, - boolean redactCat, - boolean redactSettingsValue) { - if (type != STRING || value.isNull()) { - return value; - } - if (redactCat) { - return stringValue(RestResponseRedactor.redact(value.stringValue())); - } - if (redactSettingsValue && "value".equals(column)) { - return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); - } - return value; - } - } - - private static final Map REGISTRY = buildRegistry(); - - private static Map buildRegistry() { - Map m = new LinkedHashMap<>(); - - // /_cluster/health — single-row cluster health snapshot (read-only monitor action). - LinkedHashMap healthSchema = new LinkedHashMap<>(); - healthSchema.put("cluster_name", STRING); - healthSchema.put("status", STRING); - healthSchema.put("number_of_nodes", INTEGER); - healthSchema.put("number_of_data_nodes", INTEGER); - healthSchema.put("active_primary_shards", INTEGER); - healthSchema.put("active_shards", INTEGER); - healthSchema.put("relocating_shards", INTEGER); - healthSchema.put("initializing_shards", INTEGER); - healthSchema.put("unassigned_shards", INTEGER); - healthSchema.put("timed_out", BOOLEAN); - m.put( - "/_cluster/health", - new Endpoint( - "/_cluster/health", - healthSchema, - Set.of("local"), - (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); - - // /_cat/indices — one row per index (read-only monitor action). - LinkedHashMap catSchema = new LinkedHashMap<>(); - catSchema.put("index", STRING); - catSchema.put("health", STRING); - catSchema.put("pri", INTEGER); - catSchema.put("rep", INTEGER); - catSchema.put("active_shards", INTEGER); - m.put( - "/_cat/indices", - new Endpoint( - "/_cat/indices", - catSchema, - Set.of("health"), - (client, spec) -> client.catIndices(spec.getArgs()))); - - // /_cat/nodes — one row per node with resource state (read-only monitor action). - LinkedHashMap nodesSchema = new LinkedHashMap<>(); - nodesSchema.put("name", STRING); - nodesSchema.put("ip", STRING); - nodesSchema.put("node_role", STRING); - nodesSchema.put("heap_percent", INTEGER); - nodesSchema.put("ram_percent", INTEGER); - nodesSchema.put("cpu", INTEGER); - m.put( - "/_cat/nodes", - new Endpoint( - "/_cat/nodes", - nodesSchema, - Set.of(), - (client, spec) -> client.catNodes(spec.getArgs()))); - - // /_cat/cluster_manager — single row identifying the elected cluster manager node. - LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); - clusterManagerSchema.put("id", STRING); - clusterManagerSchema.put("host", STRING); - clusterManagerSchema.put("ip", STRING); - clusterManagerSchema.put("node", STRING); - m.put( - "/_cat/cluster_manager", - new Endpoint( - "/_cat/cluster_manager", - clusterManagerSchema, - Set.of(), - (client, spec) -> client.catClusterManager(spec.getArgs()))); - - // /_cat/plugins — one row per installed plugin per node (read-only monitor action). - LinkedHashMap pluginsSchema = new LinkedHashMap<>(); - pluginsSchema.put("name", STRING); - pluginsSchema.put("component", STRING); - pluginsSchema.put("version", STRING); - m.put( - "/_cat/plugins", - new Endpoint( - "/_cat/plugins", - pluginsSchema, - Set.of(), - (client, spec) -> client.catPlugins(spec.getArgs()))); - - // /_cat/shards — one row per shard (read-only monitor action). - LinkedHashMap shardsSchema = new LinkedHashMap<>(); - shardsSchema.put("index", STRING); - shardsSchema.put("shard", INTEGER); - shardsSchema.put("prirep", STRING); - shardsSchema.put("state", STRING); - shardsSchema.put("node", STRING); - m.put( - "/_cat/shards", - new Endpoint( - "/_cat/shards", - shardsSchema, - Set.of(), - (client, spec) -> client.catShards(spec.getArgs()))); - - // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). - LinkedHashMap stateSchema = new LinkedHashMap<>(); - stateSchema.put("cluster_name", STRING); - stateSchema.put("state_uuid", STRING); - stateSchema.put("version", LONG); - stateSchema.put("cluster_manager_node", STRING); - m.put( - "/_cluster/state", - new Endpoint( - "/_cluster/state", - stateSchema, - Set.of(), - (client, spec) -> List.of(client.clusterState(spec.getArgs())))); - - // /_cluster/settings — one row per configured setting (persistent/transient tier). - LinkedHashMap settingsSchema = new LinkedHashMap<>(); - settingsSchema.put("setting", STRING); - settingsSchema.put("value", STRING); - settingsSchema.put("tier", STRING); - m.put( - "/_cluster/settings", - new Endpoint( - "/_cluster/settings", - settingsSchema, - Set.of(), - (client, spec) -> client.clusterSettings(spec.getArgs()))); - - // /_resolve/index — one row per resolved index/alias/data_stream name. - LinkedHashMap resolveSchema = new LinkedHashMap<>(); - resolveSchema.put("name", STRING); - resolveSchema.put("type", STRING); - m.put( - "/_resolve/index", - new Endpoint( - "/_resolve/index", - resolveSchema, - Set.of("expand_wildcards"), - (client, spec) -> client.resolveIndex(spec.getArgs()))); - - return m; - } - - /** - * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, - * {@code /services/*}, plugin admin endpoints) is refused here. - */ - public static Endpoint resolve(String path) { - if (path == null || path.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint must be a non-empty path. Supported read-only endpoints: " - + REGISTRY.keySet()); - } - Endpoint endpoint = REGISTRY.get(path); - if (endpoint == null) { - throw new IllegalArgumentException( - "rest endpoint [" - + path - + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " - + REGISTRY.keySet()); - } - return endpoint; - } - - /** Validate that every supplied query arg is accepted by the endpoint. */ - public static void validate(RestSpec spec) { - Endpoint endpoint = resolve(spec.getEndpoint()); - if (spec.getCount() != null && spec.getCount() < 0) { - throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] count must be a non-negative integer, got [" - + spec.getCount() - + "]"); - } - if (spec.getTimeout() != null) { - // The timeout token is reserved in the grammar for forward compatibility, but a single - // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs - // cluster-manager vs client socket timeouts differ per action). Reject it with a clear - // client error rather than silently ignoring it. - throw new IllegalArgumentException( - "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); - } - if (spec.getArgs() != null) { - for (String arg : spec.getArgs().keySet()) { - if (!endpoint.getAllowedArgs().contains(arg)) { - throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] does not accept arg [" - + arg - + "]. Allowed args: " - + endpoint.getAllowedArgs()); - } - validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); - } - } - } - - // Allowed value domains for the get-args that are applied server-side. Keys are validated against - // the per-endpoint allow-list above; values are validated here so a user-supplied value is never - // passed unchecked into an admin transport request. - private static final Map> ARG_VALUE_DOMAINS = - Map.of( - "local", Set.of("true", "false"), - "health", Set.of("green", "yellow", "red")); - - private static final Set EXPAND_WILDCARDS_VALUES = - Set.of("open", "closed", "hidden", "none", "all"); - - /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ - private static void validateArgValue(String endpoint, String arg, String value) { - Set domain = ARG_VALUE_DOMAINS.get(arg); - if (domain != null) { - if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [" - + arg - + "] has an unsupported value [" - + value - + "]. Allowed values: " - + domain); - } - } else if ("expand_wildcards".equals(arg)) { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { - if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - } - } - } - - private static ExprValue coerce(String column, ExprType type, Object value) { - if (value == null) { - return ExprNullValue.of(); - } - try { - if (type == INTEGER) { - return integerValue(toNumber(value).intValue()); - } - if (type == LONG) { - return longValue(toNumber(value).longValue()); - } - if (type == DOUBLE) { - return doubleValue(toNumber(value).doubleValue()); - } - if (type == BOOLEAN) { - return booleanValue(toBoolean(value)); - } - } catch (IllegalArgumentException | ClassCastException e) { - // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint - // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, - // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely - // unexpected faults (NPE, etc.) are left to propagate. - throw new IllegalArgumentException( - "rest endpoint value for column [" - + column - + "] could not be coerced to " - + type - + ": [" - + value - + "]"); - } - return stringValue(String.valueOf(value)); - } - - /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ - private static Number toNumber(Object value) { - if (value instanceof Number n) { - return n; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new NumberFormatException("empty string"); - } - if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { - return Double.parseDouble(s); - } - return Long.parseLong(s); - } - - /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ - private static boolean toBoolean(Object value) { - if (value instanceof Boolean b) { - return b; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new IllegalArgumentException("empty string is not a boolean"); - } - if (s.equalsIgnoreCase("true")) { - return true; - } - if (s.equalsIgnoreCase("false")) { - return false; - } - throw new IllegalArgumentException("not a boolean: " + value); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java deleted file mode 100644 index 868dabdd64e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * Dispatches an allow-listed, read-only management endpoint through the transport node client under - * the caller's security thread-context and returns the response shaped to the endpoint's fixed - * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link - * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to - * the system-index scan family. - */ -public class RestRequest implements OpenSearchSystemRequest { - - private final OpenSearchClient client; - private final RestEndpointRegistry.Endpoint endpoint; - private final RestSpec spec; - private final boolean redact; - - public RestRequest( - OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { - this(client, endpoint, spec, false); - } - - public RestRequest( - OpenSearchClient client, - RestEndpointRegistry.Endpoint endpoint, - RestSpec spec, - boolean redact) { - this.client = client; - this.endpoint = endpoint; - this.spec = spec; - this.redact = redact; - } - - @Override - public List search() { - List rows = endpoint.toRows(client, spec, redact); - if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { - return rows.subList(0, spec.getCount()); - } - return rows; - } - - @Override - public String toString() { - return "RestRequest{endpoint=" + endpoint.getPath() + "}"; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java deleted file mode 100644 index fd674dccde4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import java.util.regex.Pattern; - -/** - * Masks network identifiers in rest command cell values. Enabled per deployment via {@code - * plugins.ppl.rest.redaction.enabled}; off by default. - */ -public final class RestResponseRedactor { - - private RestResponseRedactor() {} - - private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; - private static final Pattern IPV4 = - Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); - private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); - private static final Pattern EC2_HOST = - Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); - private static final Pattern IPV6 = - Pattern.compile( - "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" - + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", - Pattern.CASE_INSENSITIVE); - private static final Pattern AZ_NAME = - Pattern.compile( - "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", - Pattern.CASE_INSENSITIVE); - - private record Mask(Pattern pattern, String replacement) {} - - private static final List MASKS = - List.of( - new Mask(IPV4, "x.x.x.x"), - new Mask(INET, "inet[/x.x.x.x:y]"), - new Mask(EC2_HOST, ""), - new Mask(IPV6, "x.x.x.x"), - new Mask(AZ_NAME, "xx-xxxxx-xx")); - - /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ - public static String redact(String text) { - if (text == null || text.isEmpty()) { - return text; - } - String out = text; - for (Mask mask : MASKS) { - out = mask.pattern().matcher(out).replaceAll(mask.replacement()); - } - return out; - } - - /** Mask availability-zone names only. */ - public static String maskAvailabilityZone(String text) { - if (text == null || text.isEmpty()) { - return text; - } - return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java deleted file mode 100644 index d425c361e6a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import org.opensearch.common.settings.SettingsFilter; - -/** - * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' - * /_cluster/settings'} fetcher. - * - *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with - * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the - * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads - * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the - * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction - * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. - * - *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in - * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link - * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code - * getRestHandlers} fires lets the fetcher read the same instance without going back through the - * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. - */ -public final class RestSettingsFilterHolder { - - private static volatile SettingsFilter settingsFilter; - - private RestSettingsFilterHolder() {} - - public static void set(SettingsFilter instance) { - settingsFilter = instance; - } - - public static SettingsFilter get() { - return settingsFilter; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java index 19d585ea684..fa543ab266b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ @Getter -public abstract class AbstractCalciteCatalogScan extends TableScan { - public final OpenSearchCatalogTable catalogTable; +public abstract class AbstractCalciteSystemIndexScan extends TableScan { + public final OpenSearchSystemIndex sysIndex; protected final RelDataType schema; - protected AbstractCalciteCatalogScan( + protected AbstractCalciteSystemIndexScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { super(cluster, traitSet, hints, table); - this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); + this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java index 58f86874c73..b0c92dce8f9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java @@ -26,22 +26,17 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ -public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan +/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ +public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan implements EnumerableRel { - public CalciteEnumerableCatalogScan( + public CalciteEnumerableSystemIndexScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { super( - cluster, - cluster.traitSetOf(EnumerableConvention.INSTANCE), - hints, - table, - catalogTable, - schema); + cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); } @Override @@ -65,7 +60,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -73,10 +68,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchCatalogEnumerator( + return new OpenSearchSystemIndexEnumerator( getFieldPath(), - catalogTable.getSource().createRequest(), - catalogTable.createOpenSearchResourceMonitor()); + sysIndex.getSystemIndexBundle().getRight(), + sysIndex.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java index 4278539c2b0..012ceec8c13 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; -/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ -public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { +/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ +public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { - public CalciteLogicalCatalogScan( - RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { + public CalciteLogicalSystemIndexScan( + RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - catalogTable, + sysIndex, table.getRowType()); } - protected CalciteLogicalCatalogScan( + protected CalciteLogicalSystemIndexScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { - super(cluster, traitSet, hints, table, catalogTable, schema); + super(cluster, traitSet, hints, table, sysIndex, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java deleted file mode 100644 index 1d021cda354..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.List; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.calcite.plan.Scannable; - -/** - * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, - * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via - * {@link CatalogSource#isScannable()}. - */ -public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { - public CalciteScannableCatalogScan( - RelOptCluster cluster, - List hints, - RelOptTable table, - OpenSearchCatalogTable catalogTable, - RelDataType schema) { - super(cluster, hints, table, catalogTable, schema); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java deleted file mode 100644 index 85dabe06b9a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.Map; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; - -/** - * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a - * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} - * rather than defining a separate table and Calcite scan hierarchy. - */ -public interface CatalogSource { - - /** Fixed schema mapping each column name to its type. */ - Map getFieldTypes(); - - /** - * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. - */ - OpenSearchSystemRequest createRequest(); - - /** - * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} - * marker for the {@code collect} short-circuit. Defaults to {@code false}. - */ - default boolean isScannable() { - return false; - } - - /** The V2 physical plan path, or throws when the source is Calcite only. */ - PhysicalPlan implementV2(LogicalPlan plan); -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java deleted file mode 100644 index 9bcb3a3ff0c..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.Map; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; - -/** - * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint - * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a - * pluggable {@link CatalogSource}. - */ -@Getter -public class OpenSearchCatalogTable extends AbstractOpenSearchTable { - - private final CatalogSource source; - private final Settings settings; - - public OpenSearchCatalogTable(CatalogSource source, Settings settings) { - this.source = source; - this.settings = settings; - } - - @Override - public boolean exists() { - return true; - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch catalog table is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return source.getFieldTypes(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalCatalogScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return source.implementV2(plan); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java new file mode 100644 index 00000000000..6bae4d21aba --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** OpenSearch System Index Table Implementation. */ +@Getter +public class OpenSearchSystemIndex extends AbstractOpenSearchTable { + /** System Index Name. */ + private final Pair systemIndexBundle; + + @Getter private final Settings settings; + + public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { + this.systemIndexBundle = buildIndexBundle(client, indexName); + this.settings = settings; + } + + @Override + public boolean exists() { + return true; // TODO: implement for system index later + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch system index is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return systemIndexBundle.getLeft().getMapping(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); + } + + @VisibleForTesting + @RequiredArgsConstructor + public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); + } + } + + /** + * Constructor of ElasticsearchSystemIndexName. + * + * @param indexName index name; + */ + private Pair buildIndexBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java index dff9b47265a..1bb8f9d5293 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java @@ -16,11 +16,8 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** - * Resource-monitored iteration over the rows produced by a read-only catalog {@link - * OpenSearchSystemRequest}. - */ -public class OpenSearchCatalogEnumerator implements Enumerator { +/** Supports a simple iteration over a collection for OpenSearch system index */ +public class OpenSearchSystemIndexEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -38,7 +35,7 @@ public class OpenSearchCatalogEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchCatalogEnumerator( + public OpenSearchSystemIndexEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java deleted file mode 100644 index 3541e745e9d..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import java.util.Map; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** - * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and - * index field mappings. - */ -public class SystemIndexCatalogSource implements CatalogSource { - - private final Pair bundle; - - public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { - this.bundle = buildBundle(client, indexName); - } - - @Override - public Map getFieldTypes() { - return bundle.getLeft().getMapping(); - } - - @Override - public OpenSearchSystemRequest createRequest() { - return bundle.getRight(); - } - - @Override - public PhysicalPlan implementV2(LogicalPlan plan) { - return plan.accept( - new DefaultImplementor() { - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(bundle.getRight()); - } - }, - null); - } - - private static Pair buildBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java deleted file mode 100644 index 33be6cc8482..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.client; - -import static java.util.stream.Collectors.toSet; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.opensearch.action.admin.cluster.state.ClusterStateRequest; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; -import org.opensearch.transport.client.node.NodeClient; - -/** - * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using - * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. - */ -class OpenSearchNodeClientClusterSettingsFilterTest { - - @AfterEach - void clearHolder() { - RestSettingsFilterHolder.set(null); - } - - private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { - NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); - ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); - when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) - .thenReturn(stateResp); - when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); - when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); - return new OpenSearchNodeClient(nodeClient); - } - - @Test - void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("plugins.secret.token", "supersecret") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Publish a filter that redacts the secret key, exactly as the native endpoint would. - RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); - - List> rows = client.clusterSettings(Map.of()); - Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); - assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); - } - - @Test - void clusterSettingsRedactsByGlobPattern() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("s3.client.default.secret_key", "AKIAEXAMPLE") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); - - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable")); - assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); - } - - @Test - void clusterSettingsFailsClosedWhenNoFilterPublished() { - Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw - // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter - // before any query, so this path is unreachable in cluster. - assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 0c570098924..5024d416086 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,15 +9,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -77,15 +74,6 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } - @Test - void restSettingsAreNonDynamic() { - assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); - assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); - List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); - assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); - assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); - } - @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index 102ec4da8f7..fa04395e065 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,15 +7,11 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; -import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -24,9 +20,8 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.Table; -import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -57,68 +52,6 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll( - () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); - } - - @Test - public void getRestTableAllowedByWildcard() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("*")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - Table table = - engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); - assertTrue(table instanceof OpenSearchCatalogTable); - } - - @Test - public void getRestTableAllowedBySubset() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - assertTrue( - engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) - instanceof OpenSearchCatalogTable); - } - - @Test - public void getRestTableRejectedWhenEndpointNotInSubset() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); - IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> - engine.getTable( - new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); - assertTrue(e.getMessage().contains("is not enabled on this cluster")); - } - - @Test - public void getRestTableDisabledWhenListEmpty() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> - engine.getTable( - new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); - assertTrue(e.getMessage().contains("disabled on this cluster")); + assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java deleted file mode 100644 index 8676373d2a6..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasEntry; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, - * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) - * path. - */ -@ExtendWith(MockitoExtension.class) -class RestCatalogSourceTest { - - @Mock private OpenSearchClient client; - - private RestSpec healthSpec() { - return new RestSpec("/_cluster/health", Map.of(), null, null); - } - - @Test - void getFieldTypesReturnsFixedEndpointSchema() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - Map fieldTypes = source.getFieldTypes(); - assertThat(fieldTypes, hasEntry("status", STRING)); - assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); - } - - @Test - void isScannable() { - assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); - } - - @Test - void implementV2IsUnsupported() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); - } - - @Test - void constructorRejectsNonAllowListedEndpoint() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); - } - - @Test - void constructorRejectsDisallowedArg() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource( - client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); - } - - @Test - void constructorRejectsNegativeCount() { - assertThrows( - IllegalArgumentException.class, - () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); - } - - @Test - void constructorRejectsTimeoutArg() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); - } - - @Test - void restRequestShapesResponseRows() { - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - } - - @Test - void countTruncatesRows() { - Map idx1 = new LinkedHashMap<>(); - idx1.put("index", "a"); - Map idx2 = new LinkedHashMap<>(); - idx2.put("index", "b"); - when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - - RestCatalogSource source = - new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java deleted file mode 100644 index c5a978bb495..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -@ExtendWith(MockitoExtension.class) -class RestEndpointRegistryTest { - - @Mock private OpenSearchClient client; - - @Test - void resolveAllowListedEndpoint() { - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - assertEquals("/_cluster/health", endpoint.getPath()); - assertEquals(STRING, endpoint.getSchema().get("status")); - assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); - } - - @Test - void resolveRejectsNonAllowListedEndpoint() { - // A mutating endpoint is simply absent from the registry and is refused here. - assertThrows( - IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); - assertThrows( - IllegalArgumentException.class, - () -> RestEndpointRegistry.resolve("/services/server/info")); - } - - @Test - void validateRejectsUnknownArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsAllowedArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); - RestEndpointRegistry.validate(spec); // no throw - } - - @Test - void catEndpointsRedactAddressesWhenRedactionEnabled() { - Map node = new LinkedHashMap<>(); - node.put("name", "ip-10-0-0-7"); - node.put("ip", "10.0.0.7"); - node.put("node_role", "dir"); - node.put("heap_percent", 44); - node.put("ram_percent", 95); - node.put("cpu", 2); - when(client.catNodes(any())).thenReturn(List.of(node)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); - RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("", redacted.get("name").stringValue()); - assertEquals(44, redacted.get("heap_percent").integerValue()); - - Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); - assertEquals("10.0.0.7", plain.get("ip").stringValue()); - assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); - } - - @Test - void catClusterManagerRedactsHostAndIp() { - Map row = new LinkedHashMap<>(); - row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); - row.put("host", "10.0.0.7"); - row.put("ip", "10.0.0.7"); - row.put("node", "71d03b567bb755839a73d437b2b066d4"); - when(client.catClusterManager(any())).thenReturn(List.of(row)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); - RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("host").stringValue()); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); - assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); - } - - @Test - void nonCatEndpointNotRedactedEvenWhenEnabled() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "10.0.0.7"); - health.put("status", "green"); - health.put("number_of_nodes", 3); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); - } - - @Test - void clusterSettingsMasksAvailabilityZoneInValue() { - Map setting = new LinkedHashMap<>(); - setting.put("setting", "cluster.routing.allocation.awareness.attributes"); - setting.put("value", "zone:us-east-1a"); - setting.put("tier", "persistent"); - when(client.clusterSettings(any())).thenReturn(List.of(setting)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); - RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); - assertEquals( - "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); - assertEquals("persistent", row.get("tier").stringValue()); - } - - @Test - void validateRejectsDroppedLevelArg() { - // level was dropped (no-op against the fixed cluster-level health schema); now unknown. - RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("does not accept arg")); - } - - @Test - void validateRejectsDroppedFlatSettingsArg() { - // flat_settings was dropped (redundant: settings are already flattened to dotted keys). - RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsValidArgValues() { - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); - } - - @Test - void validateRejectsBadArgValue() { - IllegalArgumentException health = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); - assertTrue(health.getMessage().contains("unsupported value")); - - IllegalArgumentException local = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); - assertTrue(local.getMessage().contains("unsupported value")); - - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec( - "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); - } - - @Test - void resolveRejectsBlankEndpoint() { - IllegalArgumentException emptyEx = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); - assertTrue(emptyEx.getMessage().contains("non-empty path")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); - } - - @Test - void validateRejectsNegativeCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("non-negative")); - } - - @Test - void validateAcceptsZeroCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); - RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit - } - - @Test - void validateRejectsTimeoutArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("timeout")); - } - - @Test - void coerceParsesNumericStringValues() { - // The cat JSON API returns numeric columns as strings; coerce must parse them. - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "3"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - - assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - } - - @Test - void coerceThrowsClearErrorOnUncoercibleValue() { - // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), - // not a raw ClassCastException / NumberFormatException (HTTP 500). - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "not-a-number"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - IllegalArgumentException ex = - assertThrows( - IllegalArgumentException.class, - () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); - assertTrue(ex.getMessage().contains("number_of_nodes")); - assertTrue(ex.getMessage().contains("not-a-number")); - } - - @Test - void clusterHealthRowsAreShapedToFixedSchema() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "test-cluster"); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - - assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - // a declared column the action did not return becomes null, never absent. - assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); - } - - @Test - void catIndicesRowsAreShapedToFixedSchema() { - Map idx = new LinkedHashMap<>(); - idx.put("index", "books"); - idx.put("health", "yellow"); - idx.put("pri", 1); - idx.put("rep", 1); - when(client.catIndices(any())).thenReturn(List.of(idx)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); - List rows = - endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); - - assertEquals(1, rows.size()); - assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); - assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java deleted file mode 100644 index 3b038306da5..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class RestResponseRedactorTest { - - @Test - void masksIpv4() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); - assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); - assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); - } - - @Test - void doesNotMaskInvalidOrPartialIpv4() { - assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); - assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); - assertEquals("44", RestResponseRedactor.redact("44")); - } - - @Test - void masksEc2HostName() { - assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); - assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); - assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); - assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); - } - - @Test - void masksFullIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); - } - - @Test - void masksCompressedIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); - } - - @Test - void masksInetAddress() { - assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); - } - - @Test - void masksAvailabilityZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); - // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); - assertEquals( - "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); - } - - @Test - void maskAvailabilityZoneMasksOnlyZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); - assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); - assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); - } - - @Test - void leavesNonAddressesIntact() { - assertEquals( - "e4e136ea81e27370ff73cf753ba22d39", - RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); - assertEquals( - "data,ingest,remote_cluster_client", - RestResponseRedactor.redact("data,ingest,remote_cluster_client")); - assertEquals( - "x.x.x.x 44 95 imr - e4e136ea", - RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); - } - - @Test - void handlesNullAndEmpty() { - assertEquals(null, RestResponseRedactor.redact(null)); - assertEquals("", RestResponseRedactor.redact("")); - assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); - assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java index df81225afbe..0b0aa1ec521 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java @@ -32,12 +32,8 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; -/** - * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: - * schema delegation, the predefined-table contract, and the V2 physical path. - */ @ExtendWith(MockitoExtension.class) -class OpenSearchCatalogTableTest { +class OpenSearchSystemIndexTest { @Mock private OpenSearchClient client; @@ -45,37 +41,36 @@ class OpenSearchCatalogTableTest { @Mock private Settings settings; - private OpenSearchCatalogTable systemTable(String name) { - return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); - } - @Test void testGetFieldTypesOfMetaTable() { - final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); + OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - final Map fieldTypes = - systemTable(mappingTable("test_index")).getFieldTypes(); + OpenSearchSystemIndex systemIndex = + new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); + final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - assertTrue(systemTable(TABLE_INFO).exists()); + Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + assertTrue(systemIndex.exists()); } @Test void testCreateTable() { - Table systemIndex = systemTable(TABLE_INFO); + Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); + OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index d9437fded5b..e1278aa75e8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -198,10 +198,6 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); - // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can - // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); - return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 5685d539541..f8214096e41 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -107,14 +107,13 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine serves neither the system catalog nor the rest command's reserved - // in-cluster source; both fall back to the default (Calcite) pipeline. + // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean defaultPipeline = + boolean systemCatalog = extractIndexName(query, queryType, context) - .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) + .map(RestUnifiedQueryAction::isSystemCatalog) .orElse(false); - return !defaultPipeline; + return !systemCatalog; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 0cf87f0604e..111597bb587 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,12 +178,6 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } - @Test - public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { - enableClusterComposite(); - assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); - } - @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index fab30b66c3f..91d06b1fcd1 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,8 +13,6 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; -REST: 'REST'; -TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index fda8d66d135..e05892c5a9d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,7 +45,6 @@ subSearch // commands pplCommands : describeCommand - | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -106,7 +105,6 @@ commands commandName : SEARCH | DESCRIBE - | REST | SHOW | WHERE | FIELDS @@ -212,16 +210,6 @@ describeCommand : DESCRIBE tableSourceClause ; - -restCommand - : REST stringLiteral (restArgument)* - ; - -restArgument - : COUNT EQUAL integerLiteral - | TIMEOUT EQUAL stringLiteral - | ident EQUAL literalValue - ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -1853,6 +1841,4 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE - // rest command token, also usable as a free-text search term / identifier - | TIMEOUT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e7d42793fe8..6d5ca92d167 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -113,7 +113,6 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; -import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -147,7 +146,6 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; -import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -259,37 +257,6 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } - /** - * Rest command.
- * Leading command that reads an allow-listed, read-only in-cluster management endpoint - * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved - * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves - * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE - * resolves to a system index. Allow-list/authorization enforcement happens at source-table - * construction in the storage engine (it owns the transport actions and per-endpoint schemas). - */ - @Override - public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { - String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); - LinkedHashMap args = new LinkedHashMap<>(); - Integer count = null; - String timeout = null; - for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { - if (arg.COUNT() != null) { - count = Integer.parseInt(arg.integerLiteral().getText()); - } else if (arg.TIMEOUT() != null) { - timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); - } else { - args.put( - StringUtils.unquoteIdentifier(arg.ident().getText()), - StringUtils.unquoteText(arg.literalValue().getText())); - } - } - String token = - SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); - return new RestRelation(new QualifiedName(token)); - } - /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 540869d1642..8adb5b79ce3 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -97,7 +97,6 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; -import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -125,7 +124,6 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; -import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -169,23 +167,6 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { - if (node instanceof RestRelation) { - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); - StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); - if (spec.getCount() != null) { - sb.append(" count=").append(MASK_LITERAL); - } - if (spec.getTimeout() != null) { - sb.append(" timeout=").append(MASK_LITERAL); - } - if (spec.getArgs() != null) { - for (String key : spec.getArgs().keySet()) { - sb.append(' ').append(key).append('=').append(MASK_LITERAL); - } - } - return sb.toString(); - } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java deleted file mode 100644 index da9424c9c8f..00000000000 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl.calcite; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import org.junit.Test; -import org.opensearch.sql.ast.Node; -import org.opensearch.sql.ast.tree.Project; -import org.opensearch.sql.ast.tree.RestRelation; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; -import org.opensearch.sql.ppl.parser.AstBuilder; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** - * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. - * - *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> - * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code - * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the - * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan - * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and - * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the - * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a - * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides - * {@code visitRelation} exactly like {@code DESCRIBE}. - */ -public class CalcitePPLRestTest { - - private final PPLSyntaxParser parser = new PPLSyntaxParser(); - private final Settings settings = mock(Settings.class); - - private Node parse(String ppl) { - return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); - } - - @Test - public void restHealthProjectsDeclaredColumns() { - Project project = - (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); - RestRelation rest = (RestRelation) project.getChild().get(0); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - // downstream fields compose on top of the rest row source. - assertEquals(2, project.getProjectList().size()); - } - - @Test - public void restReservedNameRoundTrips() { - RestRelation rest = - (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); - String reserved = rest.getTableQualifiedName().toString(); - assertTrue(SystemIndexUtils.isRestSource(reserved)); - SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); - assertEquals("/_cat/indices", spec.getEndpoint()); - assertEquals(Integer.valueOf(10), spec.getCount()); - assertEquals("5s", spec.getTimeout()); - assertEquals("green", spec.getArgs().get("health")); - } -} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d57ca8a69bb..9d70487c741 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,32 +1939,4 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } - - // rest command tests - - @Test - public void testRestCommand() { - org.opensearch.sql.ast.tree.Project project = - (org.opensearch.sql.ast.tree.Project) - plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); - org.opensearch.sql.ast.tree.RestRelation rest = - (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - assertTrue(spec.getArgs().isEmpty()); - } - - @Test - public void testRestCommandWithArgs() { - org.opensearch.sql.ast.tree.RestRelation rest = - (org.opensearch.sql.ast.tree.RestRelation) - plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - assertEquals(Integer.valueOf(5), spec.getCount()); - assertEquals("30s", spec.getTimeout()); - assertEquals("indices", spec.getArgs().get("level")); - } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 1822b690feb..286f601b52c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,18 +48,6 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } - @Test - public void testRestCommand() { - assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); - } - - @Test - public void testRestCommandMasksArgValues() { - assertEquals( - "rest /_cluster/health count=*** timeout=*** level=***", - anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); - } - @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); From e5a0bc307089d573b9ddc5876f103281cfa739f3 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Mon, 20 Jul 2026 07:29:49 -0700 Subject: [PATCH 02/78] [Feature] Add PPL `makeresults` command (#5622) * [Feature] Add PPL makeresults command Add the makeresults leading command on the Calcite (v3) path. It generates in-memory rows with no index scan: - count=N produces N rows, each with a single _time timestamp set to query time - format=csv|json data=... parses an inline literal into typed rows, with column types synthesized following OpenSearch dynamic-mapping semantics and surfaced through the same type path an index scan uses (JSON int->long, decimal->float, string->keyword; typed CSV name:type via cast's vocabulary; bare CSV->string) Grammar lives in the ppl/ copies only. Adds the MakeResults AST node, AstBuilder wiring with MakeResultsDataParser, CalciteRelNodeVisitor.visitMakeResults building LogicalValues + Project, a V2 Analyzer reject stub, and anonymizer rendering. Includes unit tests, AstBuilder and anonymizer tests, integration tests, and the user doc. Signed-off-by: Louis Chu * Address Songkan's comments Signed-off-by: Louis Chu * Address PR-Agent edge cases: empty-first-row guard, skip whitespace CSV lines Signed-off-by: Louis Chu * makeresults: cap inline data by cells (rows x columns) and per-value width Replace the flat data<=5000 rows guard with rows*cols<=5000 cells (the Janino 64KB per-method codegen cliff scales with rows x columns, so a flat row cap was unsafe for multi-column data). Replace the 29999 total-data char cap with a per-value guard (single cell value <= 60000 chars, under the JVM 65535-byte constant-pool limit). Trim over-verbose comments. Signed-off-by: Louis Chu * makeresults: add ragged-row and at-limit boundary tests; fix index version to 3.8 Add four boundary tests to CalcitePPLMakeResultsTest: CSV row with more columns rejected, CSV row with fewer columns padded to null, count at the 5000 cap allowed, and a cell value at the 60000-char limit allowed. Correct the PPL command index to list makeresults as 3.8 (since 3.8). Signed-off-by: Louis Chu --------- Signed-off-by: Louis Chu --- .../org/opensearch/sql/analysis/Analyzer.java | 6 + .../sql/ast/AbstractNodeVisitor.java | 5 + .../opensearch/sql/ast/tree/MakeResults.java | 47 +++ .../org/opensearch/sql/ast/tree/Values.java | 37 +- .../sql/calcite/CalciteRelNodeVisitor.java | 151 ++++++- docs/user/ppl/cmd/makeresults.md | 88 ++++ docs/user/ppl/index.md | 1 + .../standalone/CalcitePPLMakeResultsIT.java | 130 ++++++ .../sql/ppl/NewAddedCommandsIT.java | 21 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 4 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 15 + .../opensearch/sql/ppl/parser/AstBuilder.java | 42 ++ .../sql/ppl/utils/MakeResultsDataParser.java | 384 ++++++++++++++++++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 5 + .../calcite/CalcitePPLMakeResultsTest.java | 254 ++++++++++++ .../sql/ppl/parser/AstBuilderTest.java | 7 + .../ppl/utils/PPLQueryDataAnonymizerTest.java | 5 + 17 files changed, 1193 insertions(+), 9 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java create mode 100644 docs/user/ppl/cmd/makeresults.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java create mode 100644 ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 8c1dbee006f..701d1545b76 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -82,6 +82,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -562,6 +563,11 @@ public LogicalPlan visitNoMv(NoMv node, AnalysisContext context) { throw getOnlyForCalciteException("nomv"); } + @Override + public LogicalPlan visitMakeResults(MakeResults node, AnalysisContext context) { + throw getOnlyForCalciteException("makeresults"); + } + @Override public LogicalPlan visitMvExpand(MvExpand node, AnalysisContext context) { throw getOnlyForCalciteException("mvexpand"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index a32354883bf..acb6e105661 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -71,6 +71,7 @@ import org.opensearch.sql.ast.tree.Limit; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -344,6 +345,10 @@ public T visitValues(Values node, C context) { return visitChildren(node, context); } + public T visitMakeResults(MakeResults node, C context) { + return visitChildren(node, context); + } + public T visitAlias(Alias node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java b/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java new file mode 100644 index 00000000000..a8b9a388f92 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/MakeResults.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +/** + * AST node for the {@code makeresults} leading command (count path). Generates {@code count} + * in-memory rows, each carrying a single {@code @timestamp} column set to query time. + * + *

The {@code format=csv|json data="..."} form is parsed into a shared {@link Values} node + * instead (see {@code MakeResultsDataParser}), so inline literal rows flow through the common + * {@code visitValues} builder. + */ +@ToString +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class MakeResults extends UnresolvedPlan { + + private final int count; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + throw new UnsupportedOperationException("MakeResults node is supposed to have no child node"); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitMakeResults(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Values.java b/core/src/main/java/org/opensearch/sql/ast/tree/Values.java index 65d7e8d7cb2..55bea273838 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Values.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Values.java @@ -9,21 +9,54 @@ import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.ToString; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.data.type.ExprCoreType; /** AST node class for a sequence of literal values. */ @ToString @Getter @EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor public class Values extends UnresolvedPlan { private final List> values; + private final List columnNames; + + /** + * Optional explicit column types, authoritative for the schema. Required to type a zero-row + * relation (header-only CSV / empty JSON array) where there are no literals to infer from. + */ + private final List columnTypes; + + /** + * When {@code true}, prepend an implicit {@code @timestamp = NOW()} column (from {@code + * makeresults format=json data=}). CSV data= and subsearch callers leave it {@code false}. + */ + private final boolean withImplicitTimestamp; + + public Values(List> values) { + this(values, null, null); + } + + public Values( + List> values, List columnNames, List columnTypes) { + this(values, columnNames, columnTypes, false); + } + + public Values( + List> values, + List columnNames, + List columnTypes, + boolean withImplicitTimestamp) { + this.values = values; + this.columnNames = columnNames; + this.columnTypes = columnTypes; + this.withImplicitTimestamp = withImplicitTimestamp; + } + @Override public UnresolvedPlan attach(UnresolvedPlan child) { throw new UnsupportedOperationException("Values node is supposed to have no child node"); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 0df3c571c6b..94c40e5adb2 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -97,6 +97,7 @@ import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.Argument.ArgumentMap; +import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.Let; @@ -140,6 +141,7 @@ import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.Lookup.OutputStrategy; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; import org.opensearch.sql.ast.tree.MvExpand; @@ -177,6 +179,7 @@ import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.BinUtils; import org.opensearch.sql.calcite.utils.JoinAndLookupUtils; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLHintUtils; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.TimewrapUtils; @@ -186,6 +189,7 @@ import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.common.patterns.PatternUtils; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; @@ -4468,17 +4472,150 @@ public RelNode visitMvExpand(MvExpand mvExpand, CalcitePlanContext context) { @Override public RelNode visitValues(Values values, CalcitePlanContext context) { List> rows = values.getValues(); - if (rows == null || rows.isEmpty()) { + RelBuilder relBuilder = context.relBuilder; + boolean hasExplicitSchema = values.getColumnNames() != null || values.getColumnTypes() != null; + if (!hasExplicitSchema && (rows == null || rows.isEmpty())) { // PPL empty subsearch (e.g., `... | append [ ]`): zero rows, no columns. - context.relBuilder.values(context.relBuilder.getTypeFactory().builder().build()); - return context.relBuilder.peek(); + relBuilder.values(relBuilder.getTypeFactory().builder().build()); + return relBuilder.peek(); } - if (rows.size() == 1 && rows.get(0).isEmpty()) { + if (rows != null && rows.size() == 1 && rows.get(0).isEmpty()) { // SQL FROM-less SELECT (dual table) encoded as Values([[]]): one-row relation for Project. - context.relBuilder.push(LogicalValues.createOneRow(context.relBuilder.getCluster())); - return context.relBuilder.peek(); + relBuilder.push(LogicalValues.createOneRow(relBuilder.getCluster())); + return relBuilder.peek(); + } + // Inline literal rows, e.g. `makeresults format=csv|json data=...`. + return buildLiteralValues( + relBuilder, + values.getColumnNames(), + values.getColumnTypes(), + rows, + values.isWithImplicitTimestamp()); + } + + /** + * Build a typed {@link LogicalValues} (+ a cast {@code Project}) from inline literal rows. Column + * names/types are taken from the explicit lists when provided (authoritative, and required to + * type a zero-row relation); otherwise names are positional and types are inferred from the + * literals. + */ + private RelNode buildLiteralValues( + RelBuilder relBuilder, + List explicitNames, + List explicitTypes, + List> rows, + boolean withImplicitTimestamp) { + int nc; + if (explicitTypes != null) { + nc = explicitTypes.size(); + } else if (explicitNames != null) { + nc = explicitNames.size(); + } else if (!rows.isEmpty() && !rows.get(0).isEmpty()) { + nc = rows.get(0).size(); + } else { + nc = 0; + } + + List names = new java.util.ArrayList<>(); + for (int i = 0; i < nc; i++) { + names.add(explicitNames != null ? explicitNames.get(i) : "column_" + i); + } + + List types = new java.util.ArrayList<>(); + for (int c = 0; c < nc; c++) { + if (explicitTypes != null) { + types.add(explicitTypes.get(c)); + } else { + // infer from the first non-null literal in this column, defaulting to STRING. + ExprCoreType t = ExprCoreType.STRING; + for (List row : rows) { + DataType dt = row.get(c).getType(); + if (dt != DataType.NULL) { + t = dt.getCoreType(); + break; + } + } + types.add(t); + } + } + + boolean prependTimestamp = + withImplicitTimestamp && !names.contains(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP); + RelDataType tsType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.TIMESTAMP, false); + + var typeBuilder = relBuilder.getTypeFactory().builder(); + if (prependTimestamp) { + typeBuilder.add(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP, tsType); } - throw new CalciteUnsupportedException("Inline VALUES with literal rows is unsupported"); + for (int i = 0; i < nc; i++) { + typeBuilder.add( + names.get(i), OpenSearchTypeFactory.convertExprTypeToRelDataType(types.get(i), true)); + } + RelDataType rowType = typeBuilder.build(); + + if (rows.isEmpty()) { + // header-only CSV / empty JSON array: a zero-row relation with the resolved schema. + relBuilder.values(ImmutableList.>of(), rowType); + return relBuilder.peek(); + } + + Object[] flat = new Object[rows.size() * nc]; + int k = 0; + for (List row : rows) { + for (Literal cell : row) { + flat[k++] = cell.getValue(); + } + } + relBuilder.values(names.toArray(new String[0]), flat); + List projects = new java.util.ArrayList<>(); + if (prependTimestamp) { + projects.add( + relBuilder.alias( + relBuilder.call(PPLBuiltinOperators.NOW), + OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP)); + } + for (int i = 0; i < nc; i++) { + projects.add( + relBuilder.alias( + relBuilder.cast( + relBuilder.field(i), + rowType.getField(names.get(i), true, false).getType().getSqlTypeName()), + names.get(i))); + } + relBuilder.project(projects); + return relBuilder.peek(); + } + + @Override + public RelNode visitMakeResults(MakeResults node, CalcitePlanContext context) { + // Count path only: the `format=csv|json data=...` form is parsed into a shared Values node + // (see MakeResultsDataParser) and handled by visitValues. + RelBuilder relBuilder = context.relBuilder; + int count = node.getCount(); + RelDataType tsType = + OpenSearchTypeFactory.convertExprTypeToRelDataType(ExprCoreType.TIMESTAMP, false); + if (count == 0) { + RelDataType rowType = + relBuilder + .getTypeFactory() + .builder() + .add(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP, tsType) + .build(); + relBuilder.values(ImmutableList.>of(), rowType); + return relBuilder.peek(); + } + // The dummy column only carries row multiplicity; project it to @timestamp=NOW(), OpenSearch's + // implicit time field recognized by the time-aware commands. + Object[] dummy = new Object[count]; + for (int i = 0; i < count; i++) { + dummy[i] = i; + } + relBuilder.values(new String[] {"__makeresults_dummy__"}, dummy); + RexNode now = relBuilder.call(PPLBuiltinOperators.NOW); + relBuilder.project( + List.of(relBuilder.alias(now, OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP))); + return relBuilder.peek(); } @Override diff --git a/docs/user/ppl/cmd/makeresults.md b/docs/user/ppl/cmd/makeresults.md new file mode 100644 index 00000000000..370129f1673 --- /dev/null +++ b/docs/user/ppl/cmd/makeresults.md @@ -0,0 +1,88 @@ + +# makeresults + +The `makeresults` command generates in-memory rows. With no arguments it produces a single row containing only the `@timestamp` field, set to the query time. It is commonly used as a seed for `eval` and to generate test data. The time column is named `@timestamp` (OpenSearch's implicit time field) so it is recognized by the time-aware commands such as `timechart`, `reverse`, and `span`. + +> **Note**: The `makeresults` command is a leading command (it opens a query) and is executed only on the coordinating node. It has no backing index. It requires the Calcite engine (`plugins.calcite.enabled=true`). + +## Syntax + +The `makeresults` command has the following syntax: + +```syntax +makeresults [count=] [format=csv|json data=] +``` + +## Parameters + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `count` | Optional | The number of rows to generate. Must be a non-negative integer up to 5000. A negative value produces zero rows. Each row has a single `@timestamp` (timestamp) column. Default is `1`. | +| `format` + `data` | Optional | Generate rows from an inline `csv` or `json` literal instead (up to 5000 cells, where cells = rows x columns, and no single cell value may exceed 60000 characters). When provided, `count` is ignored. | + +### Inline data typing + +Column types for `data=` follow OpenSearch dynamic-mapping semantics: + +- JSON: an integer becomes `long`, a decimal becomes `float`, `true`/`false` becomes `boolean`, and a string becomes `string`. A nested object or array is serialized to its compact JSON string and typed as `string`; use `spath` or the `json_extract` function to re-parse it downstream. +- CSV: a header token of the form `name:type` declares the column type using the same vocabulary as `cast` (for example `age:int`); a bare header token defaults to `string`. + +The `date`, `time`, `timestamp`, `ip`, and `json` inline types are not yet supported on this path; declare the column as `string` and `cast` it downstream, for example `makeresults format=csv data='addr\n192.168.1.1' | eval addr = cast(addr as ip)`. + +### Implicit `@timestamp` column + +`format=json data=` treats each JSON object as an event and prepends an implicit `@timestamp` +(timestamp) column set to the query time, in addition to the object's own fields. If the JSON data +already defines an `@timestamp` field, that value is kept and no implicit column is added. +`format=csv data=` is a pure table and does not add an `@timestamp` column. + +## Example 1: Generate rows for testing + +The following query generates five rows: + +```ppl +makeresults count=5 +``` + +## Example 2: Seed a row for eval + +```ppl +makeresults +| eval message="hello" +``` + +## Example 3: Generate typed rows from JSON + +```ppl +makeresults format=json data='[{"name":"John","age":35},{"name":"Sarah","age":39}]' +``` + +The query returns two rows with an `@timestamp` (timestamp, query time) column followed by a `name` (string) column and an `age` (bigint) column. A JSON integer is typed as a long value; because makeresults rows have no index mapping, the column reports its Calcite type name `bigint` in the response schema. + +## Example 4: Generate typed rows from CSV + +```ppl +makeresults format=csv data='name:string,age:int +John,35 +Sarah,39' +``` + +The query returns two rows with a `name` (string) column and an `age` (int) column. + +## Limitations + +A global aggregate that references no input column, applied directly to `makeresults`, is not +currently supported and raises an error: + +```ppl +makeresults count=5 | stats count() as c +``` + +This is due to an upstream Apache Calcite field-trimming defect on zero-column relations, not a +`makeresults`-specific issue. Use any of the following equivalent forms instead: + +```ppl +makeresults count=5 | stats count(1) as c +makeresults count=5 | stats count() as c by @timestamp +makeresults count=5 | eval g=1 | stats count() as c by g +``` diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 37947113800..1afb162963d 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -80,6 +80,7 @@ source=accounts | [describe command](cmd/describe.md) | 2.1 | stable (since 2.1) | Query the metadata of an index. | | [explain command](cmd/explain.md) | 3.1 | stable (since 3.1) | Explain the plan of query. | | [show datasources command](cmd/showdatasources.md) | 2.4 | stable (since 2.4) | Query datasources configured in the PPL engine. | +| [makeresults command](cmd/makeresults.md) | 3.8 | experimental (since 3.8) | Generate in-memory rows for testing and seeding, optionally from inline CSV/JSON data. | | [addtotals command](cmd/addtotals.md) | 3.5 | stable (since 3.5) | Adds row and column values and appends a totals column and row. | | [addcoltotals command](cmd/addcoltotals.md) | 3.5 | stable (since 3.5) | Adds column values and appends a totals row. | | [transpose command](cmd/transpose.md) | 3.5 | stable (since 3.5) | Transpose rows to columns. | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java new file mode 100644 index 00000000000..ba623c649e0 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLMakeResultsIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for the makeresults leading command. + * + *

The count path output ({@code @timestamp = now()}) is non-deterministic, so it asserts schema + * + row count only. The data= path is deterministic and asserts schema + datarows. + */ +public class CalcitePPLMakeResultsIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + enableCalcite(); + } + + @Test + public void testCount() throws IOException { + JSONObject result = executeQuery("makeresults count=5"); + verifySchema(result, schema("@timestamp", "timestamp")); + assertEquals(5, result.getInt("total")); + } + + @Test + public void testBare() throws IOException { + JSONObject result = executeQuery("makeresults"); + verifySchema(result, schema("@timestamp", "timestamp")); + assertEquals(1, result.getInt("total")); + } + + @Test + public void testJson() throws IOException { + String data = + "makeresults format=json data='[{\"name\":\"John\",\"age\":35,\"score\":3.5}," + + "{\"name\":\"Sarah\",\"age\":39,\"score\":4.0}]'"; + JSONObject result = executeQuery(data); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("name", "string"), + schema("age", "bigint"), + schema("score", "float")); + JSONObject projected = executeQuery(data + " | fields name, age, score"); + verifyDataRows(projected, rows("John", 35, 3.5), rows("Sarah", 39, 4.0)); + } + + @Test + public void testNestedJsonSerializesToString() throws IOException { + String data = + "makeresults format=json data='[{\"name\":\"John\"," + + "\"addr\":{\"city\":\"NYC\",\"zip\":10001},\"tags\":[\"a\",\"b\"]}]'"; + JSONObject result = executeQuery(data); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("name", "string"), + schema("addr", "string"), + schema("tags", "string")); + JSONObject projected = executeQuery(data + " | fields name, addr, tags"); + verifyDataRows(projected, rows("John", "{\"city\":\"NYC\",\"zip\":10001}", "[\"a\",\"b\"]")); + } + + @Test + public void testNestedJsonSpathRoundTrip() throws IOException { + JSONObject result = + executeQuery( + "makeresults format=json data='[{\"addr\":{\"city\":\"NYC\"}}]'" + + " | spath input=addr output=city path=city | fields addr, city"); + verifyDataRows(result, rows("{\"city\":\"NYC\"}", "NYC")); + } + + @Test + public void testTypedCsv() throws IOException { + JSONObject result = + executeQuery("makeresults format=csv data='name:string,age:int\nJohn,35\nSarah,39'"); + verifySchema(result, schema("name", "string"), schema("age", "int")); + verifyDataRows(result, rows("John", 35), rows("Sarah", 39)); + } + + @Test + public void testBareCsv() throws IOException { + JSONObject result = executeQuery("makeresults format=csv data='name,age\nJohn,35\nSarah,39'"); + verifySchema(result, schema("name", "string"), schema("age", "string")); + verifyDataRows(result, rows("John", "35"), rows("Sarah", "39")); + } + + @Test + public void testComposesAsSource() throws IOException { + JSONObject result = executeQuery("makeresults count=3 | eval n=1"); + verifySchema(result, schema("@timestamp", "timestamp"), schema("n", "int")); + assertEquals(3, result.getInt("total")); + } + + @Test + public void testBareGlobalCountIsUnsupported() { + assertThrows(Exception.class, () -> executeQuery("makeresults count=5 | stats count() as c")); + } + + @Test + public void testBareGlobalCountWorkaroundCountArg() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | stats count(1) as c"); + verifySchema(result, schema("c", "bigint")); + verifyDataRows(result, rows(5)); + } + + @Test + public void testBareGlobalCountWorkaroundByTimestamp() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | stats count() as c by @timestamp"); + verifyDataRows(result, rows(5, result.getJSONArray("datarows").getJSONArray(0).get(1))); + } + + @Test + public void testBareGlobalCountWorkaroundEvalGroup() throws IOException { + JSONObject result = executeQuery("makeresults count=5 | eval g=1 | stats count() as c by g"); + verifyDataRows(result, rows(5, 1)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 837865a3585..6b5ac0d4302 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -318,6 +318,27 @@ public void testNoMvUnsupportedInV2() throws IOException { verifyQuery(result); } + @Test + public void testMakeResults() throws IOException { + JSONObject result; + try { + result = executeQuery("makeresults count=2"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + + if (isCalciteEnabled()) { + assertThat(result.getJSONArray("datarows").length(), equalTo(2)); + } else { + JSONObject error = result.getJSONObject("error"); + assertThat( + error.getString("details"), + containsString( + "is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true")); + assertThat(error.getString("type"), equalTo("UnsupportedOperationException")); + } + } + @Test public void testMvExpandCommandBasicExpansion() throws IOException { JSONObject result; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 91d06b1fcd1..b26751ad61b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,10 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +MAKERESULTS: 'MAKERESULTS'; +FORMAT: 'FORMAT'; +CSV: 'CSV'; +DATA: 'DATA'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e05892c5a9d..eeaed6daf52 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -46,6 +46,7 @@ subSearch pplCommands : describeCommand | showDataSourcesCommand + | makeresultsCommand | searchCommand | multisearchCommand | graphLookupCommand @@ -153,6 +154,7 @@ commandName | TRANSPOSE | GRAPHLOOKUP | TIMEWRAP + | MAKERESULTS ; searchCommand @@ -214,6 +216,16 @@ showDataSourcesCommand : SHOW DATASOURCES ; +makeresultsCommand + : MAKERESULTS makeresultsArg* + ; + +makeresultsArg + : COUNT EQUAL integerLiteral + | FORMAT EQUAL (CSV | JSON) + | DATA EQUAL stringLiteral + ; + whereCommand : WHERE logicalExpression ; @@ -1749,6 +1761,9 @@ searchableKeyWord // ARGUMENT KEYWORDS | KEEPEMPTY | CONSECUTIVE + | FORMAT + | CSV + | DATA | DEDUP_SPLITVALUES | PARTITIONS | ALLNUM diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 6d5ca92d167..e87264909c8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -97,6 +97,7 @@ import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.MinSpanBin; import org.opensearch.sql.ast.tree.Multisearch; import org.opensearch.sql.ast.tree.MvCombine; @@ -145,6 +146,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsByClauseContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; +import org.opensearch.sql.ppl.utils.MakeResultsDataParser; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; /** Class of building the AST. Refines the visit path and build the AST nodes */ @@ -257,6 +259,46 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** makeresults command. */ + @Override + public UnresolvedPlan visitMakeresultsCommand(OpenSearchPPLParser.MakeresultsCommandContext ctx) { + int count = 1; + String format = null; + String data = null; + for (OpenSearchPPLParser.MakeresultsArgContext arg : ctx.makeresultsArg()) { + if (arg.integerLiteral() != null) { + String raw = arg.integerLiteral().getText(); + try { + count = Integer.parseInt(raw); + } catch (NumberFormatException e) { + throw new SyntaxCheckException( + "makeresults count \"" + raw + "\" is not a valid integer within the allowed range"); + } + } else if (arg.stringLiteral() != null) { + data = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else if (arg.JSON() != null) { + format = "json"; + } else if (arg.CSV() != null) { + format = "csv"; + } + } + if (data != null || format != null) { + if (data == null || format == null) { + throw new SyntaxCheckException("makeresults format and data must be provided together"); + } + return MakeResultsDataParser.parse(format, data); + } + if (count < 0) { + // Negative count yields zero rows. + count = 0; + } + if (count > 5000) { + // Inline literal rows hit the JVM 64 KB per-method bytecode limit. + throw new SyntaxCheckException("makeresults count must not exceed 5000"); + } + return new MakeResults(count); + } + /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java new file mode 100644 index 00000000000..225f3d0280f --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/MakeResultsDataParser.java @@ -0,0 +1,384 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.opensearch.sql.ast.expression.DataType; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.type.ExprCoreType; + +/** + * Parses the inline {@code makeresults format=csv|json data="..."} literal into a shared {@link + * Values} node of typed literal rows. JSON values infer their type (integer to long, decimal to + * float, boolean, string); a CSV {@code name:type} header declares the type, a bare name is string. + * UDT types (timestamp/date/time/ip/json) are not yet supported on this path. + */ +public final class MakeResultsDataParser { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private MakeResultsDataParser() {} + + public static Values parse(String format, String data) { + String fmt = format == null ? null : format.toLowerCase(Locale.ROOT); + Values result; + if ("json".equals(fmt)) { + result = parseJson(data); + } else if ("csv".equals(fmt)) { + result = parseCsv(data); + } else { + throw new SyntaxCheckException("makeresults format must be 'csv' or 'json'"); + } + // Cap inline cells (rows x columns): the generated literal method hits the JVM 64 KB + // per-method bytecode limit above ~6900 cells, so a flat row cap is unsafe for wide data. + if (result.getValues() != null && !result.getValues().isEmpty()) { + int rows = result.getValues().size(); + int cols = result.getValues().get(0).size(); + long cells = (long) rows * cols; + if (cells > 5000) { + throw new SyntaxCheckException( + "makeresults data must not exceed 5000 cells (rows x columns); got " + + rows + + " rows x " + + cols + + " columns = " + + cells); + } + // A single string literal must fit the 65535-byte constant-pool CONSTANT_Utf8 limit. + for (List row : result.getValues()) { + for (Literal cell : row) { + Object v = cell.getValue(); + if (v instanceof String && ((String) v).length() > 60000) { + throw new SyntaxCheckException( + "makeresults data cell value must not exceed 60000 characters; got " + + ((String) v).length()); + } + } + } + } + return result; + } + + private static Values toValues( + List names, + List types, + List> rows, + boolean withImplicitTimestamp) { + List dataTypes = new ArrayList<>(); + for (ExprCoreType t : types) { + dataTypes.add(exprToDataType(t)); + } + List> literalRows = new ArrayList<>(); + for (List row : rows) { + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + Object v = row.get(i); + out.add(v == null ? new Literal(null, DataType.NULL) : new Literal(v, dataTypes.get(i))); + } + literalRows.add(out); + } + return new Values(literalRows, names, types, withImplicitTimestamp); + } + + private static DataType exprToDataType(ExprCoreType t) { + switch (t) { + case BOOLEAN: + return DataType.BOOLEAN; + case INTEGER: + return DataType.INTEGER; + case LONG: + return DataType.LONG; + case FLOAT: + return DataType.FLOAT; + case DOUBLE: + return DataType.DOUBLE; + case STRING: + default: + return DataType.STRING; + } + } + + private static Values parseJson(String data) { + JsonNode arr; + try { + arr = MAPPER.readTree(data); + } catch (Exception e) { + throw new SyntaxCheckException("makeresults data is not valid JSON: " + e.getMessage()); + } + if (arr == null || !arr.isArray()) { + throw new SyntaxCheckException("makeresults JSON data must be an array of objects"); + } + LinkedHashMap cols = new LinkedHashMap<>(); + List> raw = new ArrayList<>(); + for (JsonNode node : arr) { + if (!node.isObject()) { + throw new SyntaxCheckException("makeresults JSON data must be an array of objects"); + } + Map row = new LinkedHashMap<>(); + for (Iterator> it = node.fields(); it.hasNext(); ) { + Map.Entry f = it.next(); + String name = f.getKey(); + JsonNode v = f.getValue(); + ExprCoreType inferred = inferJsonType(v); + if (inferred == null) { + cols.putIfAbsent(name, null); + } else { + cols.merge(name, inferred, MakeResultsDataParser::widen); + } + row.put(name, jsonValue(v)); + } + raw.add(row); + } + List names = new ArrayList<>(cols.keySet()); + List types = new ArrayList<>(); + for (String n : names) { + ExprCoreType t = cols.get(n); + if (t == null) { + throw new SyntaxCheckException( + "makeresults column '" + + n + + "' has only null values; provide at least one non-null" + + " value so its type can be determined"); + } + types.add(t); + } + List> rows = new ArrayList<>(); + for (Map r : raw) { + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + out.add(coerce(r.get(names.get(i)), types.get(i))); + } + rows.add(out); + } + return toValues(names, types, rows, true); + } + + private static Values parseCsv(String data) { + String[] lines = data.split("\r?\n", -1); + if (lines.length == 0 || lines[0].trim().isEmpty()) { + throw new SyntaxCheckException("makeresults CSV data must start with a header line"); + } + String[] header = splitCsvLine(lines[0]).toArray(new String[0]); + List names = new ArrayList<>(); + List types = new ArrayList<>(); + for (String token : header) { + String t = token.trim(); + String name = t; + ExprCoreType type = ExprCoreType.STRING; + int colon = t.lastIndexOf(':'); + if (colon > 0 && colon < t.length() - 1) { + ExprCoreType declared = resolveType(t.substring(colon + 1).trim()); + if (declared != null) { + name = t.substring(0, colon).trim(); + type = declared; + } + } + if (name.isEmpty()) { + throw new SyntaxCheckException( + "makeresults CSV header has a blank column name: " + lines[0]); + } + names.add(name); + types.add(type); + } + names = uniquify(names); + List> rows = new ArrayList<>(); + for (int li = 1; li < lines.length; li++) { + if (lines[li].trim().isEmpty()) { + continue; + } + List cells = splitCsvLine(lines[li]); + if (cells.size() > names.size()) { + throw new SyntaxCheckException( + "makeresults CSV row has more columns than the header: " + lines[li]); + } + List out = new ArrayList<>(); + for (int i = 0; i < names.size(); i++) { + String cell = i < cells.size() ? cells.get(i).trim() : ""; + out.add(coerce(cell.isEmpty() ? null : cell, types.get(i))); + } + rows.add(out); + } + return toValues(names, types, rows, false); + } + + private static List splitCsvLine(String line) { + List out = new ArrayList<>(); + StringBuilder cur = new StringBuilder(); + boolean inQuotes = false; + for (int i = 0; i < line.length(); i++) { + char c = line.charAt(i); + if (inQuotes) { + if (c == '"') { + if (i + 1 < line.length() && line.charAt(i + 1) == '"') { + cur.append('"'); + i++; + } else { + inQuotes = false; + } + } else { + cur.append(c); + } + } else if (c == '"') { + inQuotes = true; + } else if (c == ',') { + out.add(cur.toString()); + cur.setLength(0); + } else { + cur.append(c); + } + } + if (inQuotes) { + throw new SyntaxCheckException( + "makeresults CSV data has an unterminated quoted field: " + line); + } + out.add(cur.toString()); + return out; + } + + private static List uniquify(List names) { + List out = new ArrayList<>(); + Set seen = new HashSet<>(); + for (String n : names) { + String candidate = n; + int suffix = 0; + while (!seen.add(candidate)) { + candidate = n + suffix++; + } + out.add(candidate); + } + return out; + } + + private static ExprCoreType inferJsonType(JsonNode v) { + if (v.isNull()) { + return null; + } + if (v.isObject() || v.isArray()) { + return ExprCoreType.STRING; + } + if (v.isBoolean()) { + return ExprCoreType.BOOLEAN; + } + if (v.isIntegralNumber()) { + // A JSON integer wider than long keeps full precision as a string rather than overflowing. + return v.canConvertToLong() ? ExprCoreType.LONG : ExprCoreType.STRING; + } + if (v.isNumber()) { + return ExprCoreType.FLOAT; + } + return ExprCoreType.STRING; + } + + private static ExprCoreType widen(ExprCoreType a, ExprCoreType b) { + if (a == null) { + return b; + } + if (b == null || a == b) { + return a; + } + boolean an = a == ExprCoreType.LONG || a == ExprCoreType.FLOAT; + boolean bn = b == ExprCoreType.LONG || b == ExprCoreType.FLOAT; + if (an && bn) { + return ExprCoreType.FLOAT; + } + return ExprCoreType.STRING; + } + + private static Object jsonValue(JsonNode v) { + if (v.isNull()) { + return null; + } + if (v.isObject() || v.isArray()) { + return v.toString(); + } + if (v.isBoolean()) { + return v.booleanValue(); + } + if (v.isIntegralNumber()) { + return v.canConvertToLong() ? v.longValue() : v.asText(); + } + if (v.isNumber()) { + return v.doubleValue(); + } + return v.asText(); + } + + private static ExprCoreType resolveType(String name) { + switch (name.toLowerCase(Locale.ROOT)) { + case "string": + return ExprCoreType.STRING; + case "boolean": + return ExprCoreType.BOOLEAN; + case "int": + case "integer": + return ExprCoreType.INTEGER; + case "long": + return ExprCoreType.LONG; + case "float": + return ExprCoreType.FLOAT; + case "double": + return ExprCoreType.DOUBLE; + case "date": + case "time": + case "timestamp": + case "ip": + case "json": + throw new SyntaxCheckException( + "makeresults inline type '" + name + "' is not yet supported; use string and cast"); + default: + return null; + } + } + + private static Object coerce(Object value, ExprCoreType type) { + if (value == null) { + return null; + } + String s = String.valueOf(value); + try { + switch (type) { + case STRING: + return s; + case BOOLEAN: + return value instanceof Boolean ? value : parseBooleanStrict(s.trim()); + case INTEGER: + return Integer.parseInt(s.trim()); + case LONG: + return value instanceof Long ? value : Long.parseLong(s.trim()); + case FLOAT: + case DOUBLE: + return value instanceof Double ? value : Double.parseDouble(s.trim()); + default: + return s; + } + } catch (NumberFormatException e) { + throw new SyntaxCheckException( + "makeresults cannot parse \"" + s + "\" as " + type.typeName()); + } + } + + private static Boolean parseBooleanStrict(String s) { + if ("true".equalsIgnoreCase(s)) { + return Boolean.TRUE; + } + if ("false".equalsIgnoreCase(s)) { + return Boolean.FALSE; + } + throw new SyntaxCheckException( + "makeresults cannot parse \"" + s + "\" as boolean; expected true or false"); + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 8adb5b79ce3..11c47d137e8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -884,6 +884,11 @@ public String visitValues(Values node, String context) { return ""; } + @Override + public String visitMakeResults(org.opensearch.sql.ast.tree.MakeResults node, String context) { + return "makeresults"; + } + private String visitFieldList(List fieldList) { return fieldList.stream().map(this::visitExpression).collect(Collectors.joining(",")); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java new file mode 100644 index 00000000000..8f2b61685f0 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMakeResultsTest.java @@ -0,0 +1,254 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +/** Logical-plan tests for the makeresults leading command (count path + format/data path). */ +public class CalcitePPLMakeResultsTest extends CalcitePPLAbstractTest { + public CalcitePPLMakeResultsTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + private void expectError(String ppl, String messageFragment) { + try { + getRelNode(ppl); + fail("expected an error for: " + ppl); + } catch (Exception e) { + String msg = String.valueOf(e.getMessage()); + assertTrue( + "expected message containing '" + messageFragment + "' but got: " + msg, + msg.contains(messageFragment)); + } + } + + @Test + public void testMakeResultsBare() { + RelNode root = getRelNode("makeresults"); + verifyLogical(root, "LogicalProject(@timestamp=[NOW()])\n LogicalValues(tuples=[[{ 0 }]])\n"); + } + + @Test + public void testMakeResultsCount() { + RelNode root = getRelNode("makeresults count=3"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()])\n" + + " LogicalValues(tuples=[[{ 0 }, { 1 }, { 2 }]])\n"); + } + + @Test + public void testMakeResultsCountZero() { + RelNode root = getRelNode("makeresults count=0"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsJson() { + RelNode root = + getRelNode( + "makeresults format=json data='[{\"name\":\"John\",\"age\":35,\"score\":3.5}," + + "{\"name\":\"Sarah\",\"age\":39,\"score\":4.0}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], name=[CAST($0):VARCHAR NOT NULL]," + + " age=[CAST($1):BIGINT NOT NULL], score=[CAST($2):REAL NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 35, 3.5E0 }, { 'Sarah', 39, 4.0E0 }]])\n"); + } + + @Test + public void testMakeResultsTypedCsv() { + RelNode root = + getRelNode("makeresults format=csv data='name:string,age:int\nJohn,35\nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[$1])\n" + + " LogicalValues(tuples=[[{ 'John', 35 }, { 'Sarah', 39 }]])\n"); + } + + @Test + public void testMakeResultsBareCsv() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', '39' }]])\n"); + } + + @Test + public void testMakeResultsHeaderOnlyCsv() { + RelNode root = getRelNode("makeresults format=csv data='name,age'"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsCsvQuotedComma() { + RelNode root = getRelNode("makeresults format=csv data='name,note\nJohn,\"a,b\"'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], note=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 'a,b' }]])\n"); + } + + @Test + public void testMakeResultsJsonBigIntKeepsPrecision() { + RelNode root = getRelNode("makeresults format=json data='[{\"n\":99999999999999999999}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], n=[CAST($0):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ '99999999999999999999' }]])\n"); + } + + @Test + public void testMakeResultsSerializesNestedJson() { + RelNode root = getRelNode("makeresults format=json data='[{\"a\":{\"x\":1},\"b\":[1,2]}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[NOW()], a=[CAST($0):VARCHAR NOT NULL]," + + " b=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ '{\"x\":1}', '[1,2]' }]])\n"); + } + + @Test + public void testMakeResultsRejectsAllNullColumn() { + expectError("makeresults format=json data='[{\"a\":null},{\"a\":null}]'", "only null values"); + } + + @Test + public void testMakeResultsRejectsDataWithoutFormat() { + expectError("makeresults data='[{\"a\":1}]'", "format and data must be provided together"); + } + + @Test + public void testMakeResultsNegativeCountYieldsZeroRows() { + // A negative count silently yields zero rows rather than an error. + RelNode root = getRelNode("makeresults count=-1"); + verifyLogical(root, "LogicalValues(tuples=[[]])\n"); + } + + @Test + public void testMakeResultsRejectsCountOverCap() { + // count > 5000 is rejected cleanly, not with a Janino 64 KB codegen failure. + expectError("makeresults count=6000", "must not exceed 5000"); + } + + @Test + public void testMakeResultsRejectsCellBudget() { + // 50 columns x 120 rows = 6000 cells > 5000; a flat row cap would miss this. + expectError(csvData(50, 120), "cells (rows x columns)"); + } + + @Test + public void testMakeResultsAllowsAtCellBudget() { + // 50 columns x 100 rows = 5000 cells is exactly at budget and must be accepted. + assertNotNull(getRelNode(csvData(50, 100))); + } + + @Test + public void testMakeResultsRejectsOversizedCellValue() { + // A single value over the 60000 per-value guard is rejected, not a codegen error. + String wide = "x".repeat(60001); + expectError( + "makeresults format=csv data='c0\n" + wide + "'", "cell value must not exceed 60000"); + } + + @Test + public void testMakeResultsRejectsCsvRowWithMoreColumns() { + expectError( + "makeresults format=csv data='name,age\nJohn,35,extra'", "more columns than the header"); + } + + @Test + public void testMakeResultsCsvRowWithFewerColumnsPadsNull() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\nSarah'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', null }]])\n"); + } + + @Test + public void testMakeResultsCountAtCap() { + assertNotNull(getRelNode("makeresults count=5000")); + } + + @Test + public void testMakeResultsAllowsCellValueAtLimit() { + assertNotNull(getRelNode("makeresults format=csv data='c0\n" + "x".repeat(60000) + "'")); + } + + private static String csvData(int cols, int rows) { + StringBuilder sb = new StringBuilder("makeresults format=csv data='"); + for (int j = 0; j < cols; j++) sb.append(j == 0 ? "" : ",").append("c").append(j); + for (int r = 0; r < rows; r++) { + sb.append("\n"); + for (int j = 0; j < cols; j++) sb.append(j == 0 ? "" : ",").append("1"); + } + return sb.append("'").toString(); + } + + @Test + public void testMakeResultsRejectsCountOverflow() { + // T3: a count outside int range yields a clean validation error, not a raw + // NumberFormatException. + expectError("makeresults count=99999999999999", "not a valid integer"); + } + + @Test + public void testMakeResultsJsonUserTimestampWins() { + RelNode root = getRelNode("makeresults format=json data='[{\"@timestamp\":\"2020\",\"x\":1}]'"); + verifyLogical( + root, + "LogicalProject(@timestamp=[CAST($0):VARCHAR NOT NULL], x=[CAST($1):BIGINT NOT NULL])\n" + + " LogicalValues(tuples=[[{ '2020', 1 }]])\n"); + } + + @Test + public void testMakeResultsRejectsInvalidBoolean() { + expectError( + "makeresults format=csv data='active:boolean\nnot true'", "cannot parse \"not true\""); + } + + @Test + public void testMakeResultsAcceptsBooleanCaseInsensitive() { + RelNode root = getRelNode("makeresults format=csv data='active:boolean\nTRUE\nFalse'"); + verifyLogical(root, "LogicalValues(tuples=[[{ true }, { false }]])\n"); + } + + @Test + public void testMakeResultsRejectsUnterminatedQuote() { + expectError("makeresults format=csv data='name\n\"unterminated'", "unterminated quoted field"); + } + + @Test + public void testMakeResultsUniquifiesDuplicateCsvHeaders() { + RelNode root = getRelNode("makeresults format=csv data='name,name\nJohn,Doe'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], name0=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', 'Doe' }]])\n"); + } + + @Test + public void testMakeResultsRejectsBlankCsvHeader() { + expectError("makeresults format=csv data=',field\n1,2'", "blank column name"); + } + + @Test + public void testMakeResultsSkipsWhitespaceOnlyCsvLines() { + RelNode root = getRelNode("makeresults format=csv data='name,age\nJohn,35\n \nSarah,39'"); + verifyLogical( + root, + "LogicalProject(name=[CAST($0):VARCHAR NOT NULL], age=[CAST($1):VARCHAR NOT NULL])\n" + + " LogicalValues(tuples=[[{ 'John', '35' }, { 'Sarah', '39' }]])\n"); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9d70487c741..d5f45a96f8c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -81,6 +81,7 @@ import org.opensearch.sql.ast.tree.Join; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.ML; +import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings.Key; @@ -1107,6 +1108,12 @@ public void testDescribeCommand() { assertEqual("describe t", describe(mappingTable("t"))); } + @Test + public void testMakeResultsCommand() { + assertEqual("makeresults", new MakeResults(1)); + assertEqual("makeresults count=5", new MakeResults(5)); + } + @Test public void testDescribeMatchAllCrossClusterSearchCommand() { assertEqual("describe *:t", describe(mappingTable("*:t"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 286f601b52c..eba4f57112b 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -36,6 +36,11 @@ public void testSearchCommand() { assertEquals("source=table identifier = ***", anonymize("search source=t a=1")); } + @Test + public void testMakeResultsCommand() { + assertEquals("makeresults", anonymize("makeresults count=5")); + } + @Test public void testTableFunctionCommand() { assertEquals( From 29c7ecb6b8d663c3424a80669452db49696ee39f Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Tue, 21 Jul 2026 00:54:35 +0800 Subject: [PATCH 03/78] Fix foreach JSON array type coercion (#5637) Signed-off-by: Songkan Tang --- .../sql/calcite/ForeachPlanner.java | 6 ++--- .../jsonUDF/ForeachJsonArrayFunctionImpl.java | 25 +++++++++++-------- .../ForeachFunctionImplTest.java | 7 ++++++ docs/user/ppl/cmd/foreach.md | 4 +-- docs/user/ppl/index.md | 1 + .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../calcite/remote/ForeachFieldJsonIT.java | 15 ++++++++--- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java index d9ef71b2220..25a8ddef642 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java +++ b/core/src/main/java/org/opensearch/sql/calcite/ForeachPlanner.java @@ -432,9 +432,9 @@ private UnresolvedExpression asArrayExpression( * VARCHAR. * *

For {@code json_array()} calls and string literals the content is visible at plan time; - * mixed content is rejected. For opaque expressions — typically a field holding JSON text, the - * primary Splunk use of json_array mode — content is unknowable, so infer from usage: an item - * placeholder consumed by arithmetic means numeric elements, anything else means strings. + * mixed content is rejected. For opaque expressions, typically a field holding JSON text, content + * is unknowable, so infer from usage: an item placeholder consumed by arithmetic means numeric + * elements, anything else means strings. */ private SqlTypeName jsonElementType( UnresolvedExpression collection, CalcitePlanContext context, Foreach node) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java index 28dff66a8ae..df8783a311e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ForeachJsonArrayFunctionImpl.java @@ -10,7 +10,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonSyntaxException; -import java.math.BigDecimal; import java.util.List; import java.util.stream.StreamSupport; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -20,6 +19,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; @@ -97,14 +97,19 @@ private static Object cast(Object value, SqlTypeName elementType) { if (value == null) { return null; } - return switch (elementType) { - case DOUBLE -> ((Number) value).doubleValue(); - case VARCHAR -> - value instanceof List || value instanceof java.util.Map - ? gson.toJson(value) - : String.valueOf(value); - case DECIMAL -> BigDecimal.valueOf(((Number) value).doubleValue()); - default -> value; - }; + // Match Calcite SAFE_CAST semantics for runtime values whose type is unknown during planning. + try { + return switch (elementType) { + case DOUBLE -> SqlFunctions.toDouble(value); + case VARCHAR -> + value instanceof List || value instanceof java.util.Map + ? gson.toJson(value) + : String.valueOf(value); + case DECIMAL -> SqlFunctions.toBigDecimal(value); + default -> value; + }; + } catch (RuntimeException e) { + return null; + } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java index e3c3724e0aa..5a633a1a9bb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CollectionUDF/ForeachFunctionImplTest.java @@ -40,6 +40,13 @@ public void testMalformedJsonArrayIsEmpty() { assertEquals(List.of(), ForeachJsonArrayFunctionImpl.eval("not-json", "VARCHAR")); } + @Test + public void testJsonArraySafelyCoercesNumericElements() { + assertEquals( + Arrays.asList(10.0, 20.0, null), + ForeachJsonArrayFunctionImpl.eval("[10,\"20\",\"not-a-number\"]", "DOUBLE")); + } + @Test public void testStatePreservesHeterogeneousAndNullSlots() { assertEquals( diff --git a/docs/user/ppl/cmd/foreach.md b/docs/user/ppl/cmd/foreach.md index 45dcb939f8a..ba6b758fa5d 100644 --- a/docs/user/ppl/cmd/foreach.md +++ b/docs/user/ppl/cmd/foreach.md @@ -41,9 +41,9 @@ Placeholders renamed via `itemstr`/`iterstr` may be written without the `<<...>> The following considerations apply when using the `foreach` command: * In collection modes, the bracketed `eval` acts as an accumulator: each target field must already exist (typically initialized with a preceding `eval`), and the expressions are applied once per element. Multiple assignments run from left to right, so a later assignment in the same iteration sees an earlier assignment's updated value. -* In `json_array` mode the element type is inferred: a `json_array(...)` call or JSON string literal is inspected at plan time; for a field holding JSON text, elements are treated as numbers when `<>` is used in arithmetic and as strings otherwise. Mixed string/number JSON arrays are rejected. +* In `json_array` mode the element type is inferred: a `json_array(...)` call or JSON string literal is inspected at plan time; for a field holding JSON text, elements are treated as numbers when `<>` is used in arithmetic and as strings otherwise. Plan-time arrays with mixed string/number elements are rejected. When numeric use is inferred for field-backed JSON text, elements that cannot be converted to numbers evaluate to `null`. * Placeholders are also substituted inside string literals. For example, `eval <> = '<>'` replaces each selected field value with that field's name. -* As in Splunk, `multivalue` mode applied to a non-array value and `json_array` mode applied to a native array are no-ops. A field whose mapping is scalar cannot be identified as multivalue at plan time even when a document stores several values, so `multivalue` mode also no-ops for that mapping. +* `multivalue` mode applied to a non-array value and `json_array` mode applied to a native array are no-ops. A field whose mapping is scalar cannot be identified as multivalue at plan time even when a document stores several values, so `multivalue` mode also no-ops for that mapping. ## Example 1: Apply the same calculation to multiple fields diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 1afb162963d..3eed4181c61 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -43,6 +43,7 @@ source=accounts | [fields command](cmd/fields.md) | 1.0 | stable (since 1.0) | Keep or remove fields from the search result. | | [rename command](cmd/rename.md) | 1.0 | stable (since 1.0) | Rename one or more fields in the search result. | | [eval command](cmd/eval.md) | 1.0 | stable (since 1.0) | Evaluate an expression and append the result to the search result. | +| [foreach command](cmd/foreach.md) | 3.8 | experimental (since 3.8) | Run a templated evaluation for each selected field or collection element. | | [convert command](cmd/convert.md) | 3.5 | experimental (since 3.5) | Transform field values to numeric values using specialized conversion functions. | | [replace command](cmd/replace.md) | 3.4 | experimental (since 3.4) | Replace text in one or more fields in the search result | | [fillnull command](cmd/fillnull.md) | 3.0 | experimental (since 3.0) | Fill null with provided value in one or more fields in the search result. | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 907f91fa98e..6d0c88bf773 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -38,6 +38,7 @@ CalciteExpandCommandIT.class, CalciteFieldFormatCommandIT.class, CalciteForeachCommandIT.class, + ForeachFieldJsonIT.class, CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java index 2fea0858abd..08243595fd8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/ForeachFieldJsonIT.java @@ -17,7 +17,7 @@ import org.opensearch.sql.legacy.TestUtils; import org.opensearch.sql.ppl.PPLIntegTestCase; -/** Foreach collection modes over index fields (Splunk-parity scenarios). */ +/** Foreach collection modes over index fields. */ public class ForeachFieldJsonIT extends PPLIntegTestCase { @Override @@ -42,7 +42,6 @@ public void init() throws Exception { @Test public void testJsonArrayModeOnFieldWithNumericContent() throws IOException { - // Splunk: field holding "[10,20,30]" with foreach mode=json_array sums to 60. JSONObject result = executeQuery( "source=test_foreach_field2 | eval total = 0 | foreach mode=json_array jsonfield [" @@ -81,6 +80,16 @@ public void testJsonArrayModeOnFieldWithStringContent() throws IOException { verifyDataRows(result, rows("ab")); } + @Test + public void testJsonArrayFieldSafelyCoercesNonNumericItems() throws IOException { + JSONObject result = + executeQuery( + "source=test_foreach_field2 | eval total = 0 | foreach mode=json_array jsonstrs [" + + " eval total = total + <> ] | fields total"); + verifySchema(result, schema("total", "double")); + verifyDataRows(result, rows((Object) null)); + } + /** * Native OpenSearch array fields (a long field holding [1,2,3]) are typed as scalar BIGINT at * plan time because OpenSearch mappings do not distinguish scalars from arrays. foreach @@ -111,7 +120,7 @@ public void testNestedFieldMultivalueIterates() throws IOException { verifyDataRows(result, rows(2)); } - /** Splunk silently no-ops when a mode is fed the wrong collection shape. */ + /** Collection modes are no-ops when the input has a different collection shape. */ @Test public void testJsonArrayModeOnRealArrayIsNoOp() throws IOException { JSONObject result = From daf6afd856cf63d82bbcf2708a37f73559ce9018 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 21 Jul 2026 09:49:27 -0700 Subject: [PATCH 04/78] Support constant_keyword field type in PPL (#5639) * Support constant_keyword field type in PPL Register `constant_keyword` in the OpenSearch mapping parser so PPL recognises the type and treats it as a string, matching the semantics of the OpenSearch `constant_keyword` field type (a single value shared by every document in an index). Resolves #3703 Signed-off-by: Peng Huo * Document constant_keyword in PPL data types table Signed-off-by: Peng Huo * Index tenant value explicitly in constant_keyword integ test The v2 engine reads field values from _source. OpenSearch does not back-fill the mapping's constant value into _source when a document omits the field, so the row came back with tenant=null on the v2 path even though the mapping declares the value. Index the tenant field in the document so both v2 and Calcite paths return the same value. Signed-off-by: Peng Huo --------- Signed-off-by: Peng Huo --- docs/user/general/datatypes.rst | 2 ++ docs/user/ppl/general/datatypes.md | 1 + .../org/opensearch/sql/ppl/DataTypeIT.java | 29 +++++++++++++++++++ .../data/type/OpenSearchDataType.java | 1 + .../data/type/OpenSearchDataTypeTest.java | 2 ++ 5 files changed, 35 insertions(+) diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 3e115b249ec..adefd59397b 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -87,6 +87,8 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da +-----------------+---------------------+-----------+ | keyword | keyword | VARCHAR | +-----------------+---------------------+-----------+ +| constant_keyword| keyword | VARCHAR | ++-----------------+---------------------+-----------+ | text | text | VARCHAR | +-----------------+---------------------+-----------+ | date* | timestamp | TIMESTAMP | diff --git a/docs/user/ppl/general/datatypes.md b/docs/user/ppl/general/datatypes.md index 85c24bd7dec..b2f38418ea8 100644 --- a/docs/user/ppl/general/datatypes.md +++ b/docs/user/ppl/general/datatypes.md @@ -42,6 +42,7 @@ The table below list the mapping between OpenSearch Data Type, PPL Data Type and | scaled_float | float | DOUBLE | | double | double | DOUBLE | | keyword | string | VARCHAR | +| constant_keyword | string | VARCHAR | | text | string | VARCHAR | | match_only_text | string | VARCHAR | | date | timestamp | TIMESTAMP | diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 1af872a8ab6..5b4181d4e33 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -207,6 +207,35 @@ public void testBooleanFieldFromNumberAcrossWildcardIndices() throws Exception { } } + @Test + public void test_constant_keyword_data_type() throws Exception { + String index = "test_constant_keyword"; + try { + Request createIndex = new Request("PUT", "/" + index); + createIndex.setJsonEntity( + "{\"mappings\":{\"properties\":{" + + "\"tenant\":{\"type\":\"constant_keyword\",\"value\":\"acme\"}," + + "\"message\":{\"type\":\"text\"}}}}"); + client().performRequest(createIndex); + + Request insertDoc = new Request("PUT", "/" + index + "/_doc/1?refresh=true"); + insertDoc.setJsonEntity("{\"tenant\":\"acme\",\"message\":\"hello\"}"); + client().performRequest(insertDoc); + + JSONObject result = executeQuery(String.format("source=%s | fields tenant, message", index)); + verifySchema(result, schema("tenant", "string"), schema("message", "string")); + verifyDataRows(result, rows("acme", "hello")); + + // constant_keyword should filter like a regular string. + JSONObject filtered = + executeQuery( + String.format("source=%s | where tenant='acme' | fields tenant, message", index)); + verifyDataRows(filtered, rows("acme", "hello")); + } finally { + client().performRequest(new Request("DELETE", "/" + index)); + } + } + @Test @RequiresCapability( value = DOC_MUTATION, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 76de0c30a08..2a70502f392 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -31,6 +31,7 @@ public enum MappingType { Text("text", ExprCoreType.UNKNOWN), MatchOnlyText("match_only_text", ExprCoreType.UNKNOWN), Keyword("keyword", ExprCoreType.STRING), + ConstantKeyword("constant_keyword", ExprCoreType.STRING), Ip("ip", ExprCoreType.IP), GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 1479ccfb615..247b7a754e0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -91,6 +91,7 @@ public void typeName() { assertEquals("DOUBLE", OpenSearchDataType.of(MappingType.Double).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); assertEquals("KEYWORD", OpenSearchDataType.of(MappingType.Keyword).typeName()); + assertEquals("CONSTANT_KEYWORD", OpenSearchDataType.of(MappingType.ConstantKeyword).typeName()); } @Test @@ -115,6 +116,7 @@ private static Stream getTestDataWithType() { return Stream.of( Arguments.of(MappingType.Text, "text", OpenSearchTextType.of()), Arguments.of(MappingType.Keyword, "keyword", STRING), + Arguments.of(MappingType.ConstantKeyword, "constant_keyword", STRING), Arguments.of(MappingType.Byte, "byte", BYTE), Arguments.of(MappingType.Short, "short", SHORT), Arguments.of(MappingType.Integer, "integer", INTEGER), From 6d3c58d641b680179bb42b15e43fc1b6b4fecf4c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Jul 2026 14:51:01 -0700 Subject: [PATCH 05/78] Xyseries command implementation (#5343) * xy series implmentation Signed-off-by: Asif Bashar * xy series implmentation Signed-off-by: Asif Bashar * xy series implmentation Signed-off-by: Asif Bashar * xy series implmentation Signed-off-by: Asif Bashar * xy series implmentation Signed-off-by: Asif Bashar * index.md updated Signed-off-by: Asif Bashar * removed duplicate format added during merging Signed-off-by: Asif Bashar * fix compile issue Signed-off-by: Asif Bashar * fix test failure Signed-off-by: Asif Bashar * added missing explain expection output files missed during merge conflict. Signed-off-by: Asif Bashar * removed extra formatting changes Signed-off-by: Asif Bashar * removed extra formatting changes Signed-off-by: Asif Bashar * fix explain test failure Signed-off-by: Asif Bashar --------- Signed-off-by: Asif Bashar --- .../org/opensearch/sql/analysis/Analyzer.java | 6 + .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/tree/Xyseries.java | 64 ++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 127 ++++++++++++++ docs/user/ppl/cmd/xyseries.md | 155 ++++++++++++++++++ docs/user/ppl/index.md | 3 +- .../sql/calcite/remote/CalciteExplainIT.java | 42 +++++ .../sql/ppl/NewAddedCommandsIT.java | 75 +++++++++ .../sql/security/CrossClusterSearchIT.java | 24 +++ .../calcite/explain_xyseries.yaml | 15 ++ ...explain_xyseries_multiple_data_fields.yaml | 16 ++ .../calcite/explain_xyseries_with_format.yaml | 15 ++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 16 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 34 ++++ .../sql/ppl/parser/AstBuilderTest.java | 73 +++++++++ 16 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java create mode 100644 docs/user/ppl/cmd/xyseries.md create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 701d1545b76..916cc00bc4a 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -112,6 +112,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -842,6 +843,11 @@ public LogicalPlan visitChart(Chart node, AnalysisContext context) { throw getOnlyForCalciteException("Chart"); } + @Override + public LogicalPlan visitXyseries(Xyseries node, AnalysisContext context) { + throw getOnlyForCalciteException("Xyseries"); + } + @Override public LogicalPlan visitWindow(Window node, AnalysisContext context) { throw getOnlyForCalciteException("Window"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index acb6e105661..266f8f46f7d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -100,6 +100,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; /** AST nodes visitor Defines the traverse path. */ public abstract class AbstractNodeVisitor { @@ -520,4 +521,8 @@ public T visitMvExpand(MvExpand node, C context) { public T visitGraphLookup(GraphLookup node, C context) { return visitChildren(node, context); } + + public T visitXyseries(Xyseries node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java new file mode 100644 index 00000000000..84fb3019e0f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Xyseries.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node representing the xyseries command. Converts row-oriented grouped results into a wide + * table where one field is the X axis (row key), one field provides pivot values for column naming, + * and one or more data fields fill the pivoted cells. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class Xyseries extends UnresolvedPlan { + + /** The x-axis field (row key in output). */ + private final UnresolvedExpression xField; + + /** The y-name field whose values become part of the output column names. */ + private final UnresolvedExpression yNameField; + + /** Explicit pivot values from the IN (...) clause. */ + private final List pivotValues; + + /** One or more y-data fields whose values fill the pivoted cells. */ + private final List yDataFields; + + /** Separator between y-data-field name and pivot value in column names. Default ":". */ + private final String separator; + + /** Optional format template for output column names using $AGG$ and $VAL$ placeholders. */ + private final String format; + + @Setter private UnresolvedPlan child; + + @Override + public Xyseries attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitXyseries(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 94c40e5adb2..9995895bfa1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -171,6 +172,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.AliasFieldsWrappable; import org.opensearch.sql.calcite.plan.HighlightPushDown; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -4071,6 +4073,131 @@ static ChartConfig fromArguments(ArgumentMap argMap) { } } + @Override + public RelNode visitXyseries(Xyseries node, CalcitePlanContext context) { + visitChildren(node, context); + + RelBuilder b = context.relBuilder; + RexBuilder rx = context.rexBuilder; + + // Resolve x-field and y-name-field names + String xFieldName = resolveFieldName(node.getXField()); + String yNameFieldName = resolveFieldName(node.getYNameField()); + + // Resolve y-data field names + List yDataFieldNames = + node.getYDataFields().stream().map(this::resolveFieldName).collect(Collectors.toList()); + + List pivotValues = node.getPivotValues() != null ? node.getPivotValues() : List.of(); + String separator = node.getSeparator(); + String format = node.getFormat(); + + // Build the pivot axis - cast to VARCHAR if needed for string comparison + RexNode yNameRef = b.field(yNameFieldName); + RelDataType yNameType = yNameRef.getType(); + RexNode axis; + if (!SqlTypeUtil.isCharacter(yNameRef.getType())) { + if (!SqlTypeUtil.isAtomic(yNameType)) { + throw new IllegalArgumentException( + "xyseries y-name-field must be a scalar type, got: " + yNameType.getSqlTypeName()); + } + RelDataType varchar = + rx.getTypeFactory() + .createTypeWithNullability( + rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), true); + axis = rx.makeCast(varchar, yNameRef, true); + } else { + axis = yNameRef; + } + + // Build aggregate calls - MAX for each y-data field + List aggCalls = + yDataFieldNames.stream() + .map(name -> b.max(b.field(name)).as(name)) + .collect(Collectors.toList()); + + // Build pivot value entries: alias -> [literal(value)] + // LinkedHashMap preserves insertion order for deterministic column ordering + LinkedHashMap> pivotValueMap = new LinkedHashMap<>(); + for (String val : pivotValues) { + pivotValueMap.put(val, ImmutableList.of(b.literal(val))); + } + + // Execute pivot: decomposes into GROUP BY x-field with FILTER-based aggregation + // Produces columns: x-field, {val1}_{agg1}, {val1}_{agg2}, {val2}_{agg1}, ... + b.pivot( + b.groupKey(b.field(xFieldName)), + aggCalls, + ImmutableList.of(axis), + pivotValueMap.entrySet()); + + // Pivot produces value-first column ordering: val1_agg1, val1_agg2, val2_agg1, ... + // Reorder to agg-first and apply custom column naming: agg1: val1, agg1: val2, ... + List reorderProjections = new ArrayList<>(); + List reorderNames = new ArrayList<>(); + + reorderProjections.add(b.field(xFieldName)); + reorderNames.add(xFieldName); + + for (String aggName : yDataFieldNames) { + for (String pivotVal : pivotValues) { + // Reference pivot output column by its generated name: {value}_{agg} + String pivotColName = pivotVal + "_" + aggName; + try { + reorderProjections.add(b.field(pivotColName)); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "xyseries: expected pivot output column '" + pivotColName + "' not found", e); + } + boolean singleDataField = yDataFieldNames.size() == 1; + reorderNames.add(generateColumnName(aggName, pivotVal, separator, format, singleDataField)); + } + } + // Fail fast with a clear message if the naming scheme produced collisions + // (e.g. a format template that omits $VAL$ or $AGG$ with multiple series). + Set seenNames = new HashSet<>(); + for (String name : reorderNames) { + if (!seenNames.add(name)) { + throw new IllegalArgumentException( + "xyseries produced duplicate output column name '" + + name + + "'. Use a format template containing both $AGG$ and $VAL$ so column names" + + " are unique."); + } + } + b.project(reorderProjections, reorderNames, true); + + // Order by x-field + b.sort(b.field(0)); + + return b.peek(); + } + + private String resolveFieldName(UnresolvedExpression expr) { + if (expr instanceof Field) { + return ((Field) expr).getField().toString(); + } + if (expr instanceof Alias) { + return ((Alias) expr).getName(); + } + return expr.toString(); + } + + private String generateColumnName( + String yDataFieldName, + String pivotValue, + String separator, + String format, + boolean singleDataField) { + if (format != null) { + return format.replace("$AGG$", yDataFieldName).replace("$VAL$", pivotValue); + } + if (singleDataField) { + return pivotValue; + } + return yDataFieldName + separator + pivotValue; + } + @Override public RelNode visitTrendline(Trendline node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/docs/user/ppl/cmd/xyseries.md b/docs/user/ppl/cmd/xyseries.md new file mode 100644 index 00000000000..a81ea7d604c --- /dev/null +++ b/docs/user/ppl/cmd/xyseries.md @@ -0,0 +1,155 @@ +# xyseries + +## Description + +The `xyseries` command converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. + +## Syntax + +```syntax +xyseries [sep=] [format=] in (, , ...) [, , ...] +``` + +## Parameters + +| Parameter | Required/Optional | Description | Default | +| --- | --- | --- | --- | +| `` | Required | The field used as the row key in the output. Results are grouped and sorted by this field. | N/A | +| `` | Required | The field whose values are used to generate output column names. Only the values listed in the `in` clause are pivoted into columns. | N/A | +| `in (, , ...)` | Required | Explicit list of pivot values to select from ``. Each value generates one output column per ``. Values must be quoted strings. | N/A | +| `` | Required (at least one) | One or more fields containing the data to pivot. If multiple fields are specified, separate them with commas. | N/A | +| `sep` | Optional | Separator between the `` name and the pivot value in output column names. Ignored if `format` is specified. | `": "` | +| `format` | Optional | Naming template for output column names. Use `$AGG$` as a placeholder for the `` name and `$VAL$` as a placeholder for the pivot value. When specified, overrides `sep`. | N/A | + +## Notes + +The following considerations apply when using the `xyseries` command: + +* The `xyseries` command is typically used after a `stats` command that groups results by both the `` and ``. +* Output column names follow the pattern `` by default (for example, `host_cnt: 200`). Use the `format` option to customize this pattern. +* When a pivot value has no matching data for a given `` row, the output cell is `null`. +* The `` values are compared as strings. Non-string fields are cast to string automatically. +* Results are sorted by `` in ascending order. +* This command requires the Calcite engine to be enabled (`plugins.calcite.enabled: true`). + +## Example 1: Basic xyseries with a single data field + +This example pivots HTTP response codes into columns for a count of hosts per URL: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | +|--------+---------------+---------------+---------------| +| /page1 | 3 | 1 | null | +| /page2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+ +``` + +## Example 2: Multiple data fields + +This example pivots multiple aggregated fields at once: + +```ppl +source=weblogs +| stats count(host) as host_cnt, count(method) as method_cnt by url, response +| xyseries url response in ("200", "404", "500") host_cnt, method_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +| url | host_cnt: 200 | host_cnt: 404 | host_cnt: 500 | method_cnt: 200 | method_cnt: 404 | method_cnt: 500 | +|--------+---------------+---------------+---------------+------------------+------------------+------------------| +| /page1 | 3 | 1 | null | 3 | 1 | null | +| /page2 | 5 | null | 2 | 5 | null | 2 | ++--------+---------------+---------------+---------------+------------------+------------------+------------------+ +``` + +## Example 3: Custom separator + +This example uses a custom separator between the data field name and pivot value in column names: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries sep="-" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | host_cnt-200 | host_cnt-404 | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 4: Format template + +This example uses a format template to customize output column names. `$VAL$` is replaced with the pivot value and `$AGG$` is replaced with the data field name: + +```ppl +source=weblogs +| stats count(host) as host_cnt by url, response +| xyseries format="$VAL$_$AGG$" url response in ("200", "404") host_cnt +``` + +The query returns the following results: + +```text +fetched rows / total rows = 2/2 ++--------+--------------+--------------+ +| url | 200_host_cnt | 404_host_cnt | +|--------+--------------+--------------| +| /page1 | 3 | 1 | +| /page2 | 5 | null | ++--------+--------------+--------------+ +``` + +## Example 5: Partial pivot values + +When only a subset of values is specified in the `in` clause, rows with unmatched `` values produce `null` for the corresponding `` rows: + +```ppl +source=accounts +| stats avg(balance) as avg_balance by gender, state +| xyseries state gender in ("F") avg_balance +``` + +The query returns the following results: + +```text +fetched rows / total rows = 7/7 ++-------+-----------------+ +| state | avg_balance: F | +|-------+-----------------| +| IL | null | +| IN | 48086.0 | +| MD | null | +| PA | 40540.0 | +| TN | null | +| VA | 32838.0 | +| WA | null | ++-------+-----------------+ +``` + +## Limitations + +The `xyseries` command has the following limitations: + +* Pivot values must be explicitly provided in the `in` clause. Dynamic pivot (deriving column names from data at runtime) is not supported. +* This command is only available when the Calcite engine is enabled. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 3eed4181c61..067d5f524fd 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -89,7 +89,8 @@ source=accounts | [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | | [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | | [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| - +| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | + - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** - [Aggregation Functions](functions/aggregations.md) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index b78a71e534c..04dc2b0e74b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -3028,6 +3028,48 @@ public void testHighlightOsdObjectFormatExplain() throws IOException { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void testXyseriesExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesMultipleDataFieldsExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance, count() as cnt by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance, cnt", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_multiple_data_fields.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testXyseriesWithFormatExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String query = + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | stats avg(balance) as avg_balance by gender, state | xyseries" + + " format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK)); + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_xyseries_with_format.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void testExplainConsecutiveSortsAfterAggIssue5125() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 6b5ac0d4302..a32dd9fb990 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; import java.io.IOException; +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -567,4 +568,78 @@ public void testUnionUnsupportedInV2() throws IOException { } verifyQuery(result); } + + @Test + public void testXyseriesCommand() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandMultipleDataFields() throws IOException { + + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance, count() as cnt by" + + " gender, state | xyseries state gender in (\"F\", \"M\") avg_balance," + + " cnt", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithSep() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries sep=\"-\" state gender in (\"F\", \"M\") avg_balance", + TEST_INDEX_BANK))); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } + + @Test + public void testXyseriesCommandWithFormat() throws IOException { + JSONObject result; + try { + result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries format=\"$VAL$_$AGG$\" state gender in (\"F\", \"M\")" + + " avg_balance", + TEST_INDEX_BANK))); + + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + verifyQuery(result); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index 0029921c1fc..86e98c1523f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -237,4 +237,28 @@ public void testCrossClusterConvertWithAlias() throws IOException { disableCalcite(); } + + @Test + public void testCrossClusterXyseries() throws IOException { + enableCalcite(); + + JSONObject result = + executeQuery( + String.format( + "search source=%s | stats avg(balance) as avg_balance by gender, state" + + " | xyseries state gender in ('F', 'M') avg_balance", + TEST_INDEX_BANK_REMOTE)); + verifyColumn(result, columnName("state"), columnName("F"), columnName("M")); + verifyDataRows( + result, + rows("IL", null, 39225.0), + rows("IN", 48086.0, null), + rows("MD", null, 4180.0), + rows("PA", 40540.0, null), + rows("TN", null, 5686.0), + rows("VA", 32838.0, null), + rows("WA", null, 16418.0)); + + disableCalcite(); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml new file mode 100644 index 00000000000..610d9aa1410 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F=[$1], M=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml new file mode 100644 index 00000000000..a8419925e99 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_multiple_data_fields.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], avg_balance: F=[$1], avg_balance: M=[$3], cnt: F=[$2], cnt: M=[$4]) + LogicalAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + LogicalProject(avg_balance=[$2], cnt=[$3], state=[$1], $f4=[IS TRUE(=($0, 'F'))], $f5=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)], cnt=[COUNT()]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableCalc(expr#0..4=[{inputs}], proj#0..1=[{exprs}], avg_balance: M=[$t3], cnt: F=[$t2], cnt: M=[$t4]) + EnumerableAggregate(group=[{2}], F_avg_balance=[MAX($0) FILTER $3], F_cnt=[MAX($1) FILTER $3], M_avg_balance=[MAX($0) FILTER $4], M_cnt=[MAX($1) FILTER $4]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=['F'], expr#5=[=($t0, $t4)], expr#6=[IS TRUE($t5)], expr#7=['M'], expr#8=[=($t0, $t7)], expr#9=[IS TRUE($t8)], avg_balance=[$t2], cnt=[$t3], state=[$t1], $f4=[$t6], $f5=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml new file mode 100644 index 00000000000..eaf89c53c2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_xyseries_with_format.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], dir0=[ASC]) + LogicalProject(state=[$0], F_avg_balance=[$1], M_avg_balance=[$2]) + LogicalAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + LogicalProject(avg_balance=[$2], state=[$1], $f3=[IS TRUE(=($0, 'F'))], $f4=[IS TRUE(=($0, 'M'))]) + LogicalAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) + LogicalProject(gender=[$4], state=[$9], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableTopK(sort0=[$0], dir0=[ASC], fetch=[10000]) + EnumerableAggregate(group=[{1}], F_avg_balance=[MAX($0) FILTER $2], M_avg_balance=[MAX($0) FILTER $3]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=['F'], expr#4=[=($t0, $t3)], expr#5=[IS TRUE($t4)], expr#6=['M'], expr#7=[=($t0, $t6)], expr#8=[IS TRUE($t7)], avg_balance=[$t2], state=[$t1], $f3=[$t5], $f4=[$t8]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index b26751ad61b..fb072ae134f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -60,6 +60,8 @@ APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; GRAPHLOOKUP: 'GRAPHLOOKUP'; +XYSERIES: 'XYSERIES'; +SEP: 'SEP'; TIMEWRAP: 'TIMEWRAP'; ALIGN: 'ALIGN'; SERIES: 'SERIES'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index eeaed6daf52..a23a314537d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -99,6 +99,7 @@ commands | fieldformatCommand | nomvCommand | graphLookupCommand + | xyseriesCommand | unionCommand | timewrapCommand ; @@ -153,6 +154,7 @@ commandName | NOMV | TRANSPOSE | GRAPHLOOKUP + | XYSERIES | TIMEWRAP | MAKERESULTS ; @@ -764,6 +766,19 @@ graphLookupArgs | (FILTER EQUAL LT_PRTHS logicalExpression RT_PRTHS) ; +xyseriesCommand + : XYSERIES xyseriesOption* xField = fieldExpression yNameField = fieldExpression IN LT_PRTHS xyseriesPivotValues RT_PRTHS yDataFields = fieldList + ; + +xyseriesOption + : SEP EQUAL sep = stringLiteral + | FORMAT EQUAL format = stringLiteral + ; + +xyseriesPivotValues + : stringLiteral (COMMA stringLiteral)* + ; + // clauses fromClause : SOURCE EQUAL tableOrSubqueryClause @@ -1856,4 +1871,5 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + | SEP ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e87264909c8..efdbf26a205 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -129,6 +129,7 @@ import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.antlr.AstBuildGuard; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -1837,4 +1838,37 @@ public UnresolvedPlan visitGraphLookupCommand(OpenSearchPPLParser.GraphLookupCom .filter(filter) .build(); } + + /** Xyseries command. */ + @Override + public UnresolvedPlan visitXyseriesCommand(OpenSearchPPLParser.XyseriesCommandContext ctx) { + UnresolvedExpression xField = internalVisitExpression(ctx.xField); + UnresolvedExpression yNameField = internalVisitExpression(ctx.yNameField); + + // Parse pivot values from IN (...) clause + List pivotValues = + ctx.xyseriesPivotValues().stringLiteral().stream() + .map(s -> StringUtils.unquoteText(s.getText())) + .distinct() + .collect(Collectors.toList()); + + // Parse y-data fields + List yDataFields = + ctx.yDataFields.fieldExpression().stream() + .map(this::internalVisitExpression) + .collect(Collectors.toList()); + + // Parse options + String separator = ": "; + String format = null; + for (OpenSearchPPLParser.XyseriesOptionContext optCtx : ctx.xyseriesOption()) { + if (optCtx.SEP() != null) { + separator = StringUtils.unquoteText(optCtx.sep.getText()); + } else if (optCtx.FORMAT() != null) { + format = StringUtils.unquoteText(optCtx.format.getText()); + } + } + + return new Xyseries(xField, yNameField, pivotValues, yDataFields, separator, format); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5f45a96f8c..0650b4d1e4c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -83,6 +83,7 @@ import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.MakeResults; import org.opensearch.sql.ast.tree.RareTopN.CommandType; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.exception.SemanticCheckException; @@ -1827,6 +1828,78 @@ public void testMalformedPipeProducesSyntaxError() { plan("source=t | invalidCmd |"); } + // Xyseries tests + + @Test + public void testXyseriesCommandBasic() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries url response in (\"200\", \"404\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandMultipleDataFields() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt"), field("method_cnt")), + ": ", + null); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries url response in (\"200\", \"404\") host_cnt, method_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSep() { + Xyseries expected = + new Xyseries( + field("url"), field("response"), List.of("200"), List.of(field("host_cnt")), "-", null); + expected.attach(relation("t")); + assertEqual("source=t | xyseries sep=\"-\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200"), + List.of(field("host_cnt")), + ": ", + "$VAL$+$AGG$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries format=\"$VAL$+$AGG$\" url response in (\"200\") host_cnt", expected); + } + + @Test + public void testXyseriesCommandWithSepAndFormat() { + Xyseries expected = + new Xyseries( + field("url"), + field("response"), + List.of("200", "404"), + List.of(field("host_cnt")), + "-", + "$AGG$_$VAL$"); + expected.attach(relation("t")); + assertEqual( + "source=t | xyseries sep=\"-\" format=\"$AGG$_$VAL$\" url response in (\"200\", \"404\")" + + " host_cnt", + expected); + } + @Test public void testUnionWithSubsearches() { plan("| union [search source=t1 | where age > 30] " + "[search source=t2 | where age < 20]"); From 7c430dda6bbc48092efd3bd2955ff6d1e94e8889 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 22 Jul 2026 08:09:55 -0700 Subject: [PATCH 06/78] Decouple Calcite PPL planning from ExprType (#5633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Decouple Calcite PPL planning from ExprType Rewrite the Calcite-side PPL planning surface so RelNode/RexNode code, coercion, type checking, and UDF implementations operate on RelDataType instead of bouncing through the v2 ExprType system. UDT identity at planning time: - UDTs (ExprDateType, ExprTimeType, ExprTimeStampType, ExprIPType, ExprBinaryType) are recognised via instanceof rather than getExprType(). Subclasses are preserved through createTypeWithNullability / createTypeWithCharsetAndCollation via a cloneWith hook on ExprSqlType / ExprJavaType so the instanceof checks survive type-factory operations. Calcite-side rewrites: - CoercionUtils: new RelDataType-typed common-type resolver with an internal CoercionTag widening DAG that mirrors v2 semantics. Widening produces UDT-normalized temporal types and always makes the cast target nullable so safe cast + primitive aggregators do not NPE on parse failure. - PPLTypeChecker: signatures expressed as List>; adds renderTypeName() for error messages; folds DECIMAL to DOUBLE. isComparable() accepts UDT temporal vs standard temporal of same kind. - PPLOperandTypes: exposes RelDataType signature constants (DATE_UDT, INTEGER_T, ...). - PPLFuncImpTable.requiresNumericArgument: uses SqlTypeUtil.isNumeric and SqlTypeName.ANY for the "unknown-type" check. - ExtendedRexBuilder, AddSubDate/Extract/Format/LastDay/PeriodName/ TimestampAdd/TimestampDiff/Weekday/Span/WidthBucket and the ip UDFs branch on UDT classes and pass RelDataType through their implementors. - visitCast in CalciteRexNodeVisitor maps AST DataType directly to RelDataType, removing the DataType.getCoreType() round-trip. - CurrentFunction / FormatFunction take an internal Kind / boolean discriminator rather than an ExprCoreType. - DatetimeExtension switches from ExprUDT enum to type-class checks. Signed-off-by: Peng Huo * Fix PPLTypeChecker.typesMatch UDT comparison and revert CalciteRexNodeVisitor churn typesMatch compared UDTs via getClass(), but addCharsetAndCollation strips concrete subclass identity from VARCHAR-backed UDTs (ExprDateType, ExprTimeType, ExprTimeStampType, ExprBinaryType) — they all collapse to ExprSqlType. That let wrapUDT accept mismatched UDTs at the signature gate: `cidrmatch(date_field, "1.2.3.4/24")` matched the [BINARY_UDT, STRING_T] signature and crashed at runtime trying to parse a date as an IP. Compare via the ExprUDT tag instead, matching what PPLComparableTypeChecker.isComparable already does. Also revert CalciteRexNodeVisitor to upstream/main — the earlier inlined visitCast switch duplicated convertExprTypeToRelDataType, and the visitBetween comment tweak had no code change. Signed-off-by: Peng Huo * spotless: reflow typesMatch javadoc Signed-off-by: Peng Huo * CoercionUtils: convert RelDataType→ExprType at boundary, reuse main-branch lattice The parallel CoercionTag lattice duplicated ExprCoreType.getParent() (BYTE→SHORT→...→DOUBLE; STRING→DATE/TIME/TIMESTAMP/BOOLEAN/IP; DATE/TIME→TIMESTAMP), and adding STRING→TIMESTAMP as a widening edge to make ranking tie with STRING→DOUBLE conflated widening truth with signature preference. Also, normalizeTemporalToUdt was a downstream fixup for plain-Calcite TIMESTAMP results that only handled bare TIMESTAMP/DATE/TIME and missed the TZ variants, and the public hasString(List) used SqlTypeUtil.isCharacter which incorrectly classified VARCHAR-backed UDTs (DATE/TIME/TIMESTAMP/BINARY) as STRING. Replace all of it with a boundary conversion: convertRelDataTypeToExprType at the entry, run the widening + rule set exactly as upstream/main does over ExprCoreType, and round-trip results through convertExprTypeToRelDataType — which already returns the UDT variant for temporals and IP, so the "normalize to UDT" step happens for free. TZ variants are handled correctly (convertSqlTypeNameToExprType folds them into TIMESTAMP/TIME). The hasString public API now matches the private one (both check ExprCoreType.STRING). Deletes ~90 lines of duplicated lattice. Signed-off-by: Peng Huo * CoercionUtils: restore upstream body verbatim, keep only boundary adapter Reduce this file to its minimal delta vs main: the only necessary change is that PPLTypeChecker.getParameterTypes() on this branch returns List> (upstream returns List>). Adapt at the entry of castArguments; everything else is main-branch code. Delete the parallel CoercionTag lattice, PARENTS map, normalizeTemporalToUdt, and the RelDataType-flavored resolveCommonType / max / distance helpers introduced in the earlier version of this file. The upstream ExprType-based lattice is the source of truth for widening, and the RelDataType→ExprType conversion via convertRelDataTypeToExprType at the public boundary handles UDTs uniformly. Test: StubTypeChecker.getParameterTypes now returns List> to match the branch's PPLTypeChecker interface. Substitute SqlTypeName.GEOMETRY for ExprCoreType.GEO_POINT in the no-compatible-signature test since GEO_POINT isn't in convertExprTypeToRelDataType's switch. Signed-off-by: Peng Huo * PPLTypeChecker.isComparable: match upstream semantics, add unit tests Replace the custom "same UDT kind → temporal kind → same SqlTypeFamily" branches with the upstream approach: convert both sides to ExprType via convertRelDataTypeToExprType and use ExprType.shouldCast to decide comparability. Two regressions surface without this: - plain VARBINARY vs EXPR_BINARY UDT (both map to ExprCoreType.BINARY) was rejected because the family-match guard excluded UDTs, so the branch never triggered on a UDT-plain pair. - day-time interval vs year-month interval (both map to INTERVAL) was rejected because Calcite splits their SqlTypeFamily into INTERVAL_DAY_TIME / INTERVAL_YEAR_MONTH. Both compare equal under shouldCast so upstream semantics preserves them. The unused temporalKind helper is removed; new PPLComparableTypeCheckerTest covers numeric, same-UDT, plain-vs-UDT temporal/binary, cross-UDT rejection, interval mixing, ANY fallback, struct field-by-field, and the IP-outer-checker rejection. Signed-off-by: Peng Huo * PPLTypeChecker.isComparable: revert to upstream, exercise via public checker Restore isComparable and its javadoc byte-for-byte to upstream/main — the previous adaptation added no behavior and the private visibility is correct. Drive the tests through the public PPLComparableTypeChecker.checkOperandTypes entry point instead of lifting isComparable to package-private just for tests. Signed-off-by: Peng Huo * PPLComparableTypeCheckerTest: reword regression comments as guardrails These pairs don't fix a shipped bug — the upstream isComparable already handles them. Reword the comments to reflect that the tests exist to guard against future edits that would classify via SqlTypeFamily or Java class, either of which would break these specific pairs. Signed-off-by: Peng Huo --------- Signed-off-by: Peng Huo --- .../sql/calcite/utils/PPLOperandTypes.java | 77 ++++--- .../expression/function/CoercionUtils.java | 9 +- .../function/PPLBuiltinOperators.java | 7 +- .../expression/function/PPLFuncImpTable.java | 13 +- .../expression/function/PPLTypeChecker.java | 212 ++++++++++-------- .../function/UDFOperandMetadata.java | 6 +- .../udf/datetime/CurrentFunction.java | 38 ++-- .../udf/datetime/PeriodNameFunction.java | 5 - .../function/udf/ip/CidrMatchFunction.java | 8 +- .../function/udf/ip/CompareIpFunction.java | 5 +- .../function/udf/ip/IPFunction.java | 17 +- .../function/CoercionUtilsTest.java | 21 +- .../PPLComparableTypeCheckerTest.java | 141 ++++++++++++ .../opensearch/functions/GeoIpFunction.java | 6 +- 14 files changed, 375 insertions(+), 190 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index fcd361ba229..fd233c83dd5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -5,10 +5,17 @@ package org.opensearch.sql.calcite.utils; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + +import java.util.ArrayList; +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.type.CompositeOperandTypeChecker; import org.apache.calcite.sql.type.FamilyOperandTypeChecker; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** @@ -20,41 +27,51 @@ public class PPLOperandTypes { // This class is not meant to be instantiated. private PPLOperandTypes() {} + // Convenience RelDataType constants used to express UDF signatures via wrapUDT(...). + // UDT-backed scalar types: + public static final RelDataType DATE_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_DATE); + public static final RelDataType TIME_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIME); + public static final RelDataType TIMESTAMP_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIMESTAMP); + public static final RelDataType IP_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_IP); + public static final RelDataType BINARY_UDT = TYPE_FACTORY.createUDT(ExprUDT.EXPR_BINARY); + // Plain SQL scalar types: + public static final RelDataType BYTE_T = TYPE_FACTORY.createSqlType(SqlTypeName.TINYINT); + public static final RelDataType SHORT_T = TYPE_FACTORY.createSqlType(SqlTypeName.SMALLINT); + public static final RelDataType INTEGER_T = TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER); + public static final RelDataType LONG_T = TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT); + public static final RelDataType FLOAT_T = TYPE_FACTORY.createSqlType(SqlTypeName.REAL); + public static final RelDataType DOUBLE_T = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); + public static final RelDataType STRING_T = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); + public static final RelDataType BOOLEAN_T = TYPE_FACTORY.createSqlType(SqlTypeName.BOOLEAN); + /** List of all scalar type signatures (single parameter each) */ - private static final java.util.List> - SCALAR_TYPES = - java.util.List.of( - // Numeric types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), - // String type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), - // Boolean type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), - // Temporal types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), - // Special scalar types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY)); + private static final List> SCALAR_TYPES = + List.of( + // Numeric types + List.of(BYTE_T), + List.of(SHORT_T), + List.of(INTEGER_T), + List.of(LONG_T), + List.of(FLOAT_T), + List.of(DOUBLE_T), + // String type + List.of(STRING_T), + // Boolean type + List.of(BOOLEAN_T), + // Temporal types + List.of(DATE_UDT), + List.of(TIME_UDT), + List.of(TIMESTAMP_UDT), + // Special scalar types + List.of(IP_UDT), + List.of(BINARY_UDT)); /** Helper method to create scalar types with optional integer parameter */ - private static java.util.List> - createScalarWithOptionalInteger() { - java.util.List> result = - new java.util.ArrayList<>(SCALAR_TYPES); + private static List> createScalarWithOptionalInteger() { + List> result = new ArrayList<>(SCALAR_TYPES); // Add scalar + integer combinations - SCALAR_TYPES.forEach( - scalarType -> - result.add( - java.util.List.of( - scalarType.get(0), org.opensearch.sql.data.type.ExprCoreType.INTEGER))); + SCALAR_TYPES.forEach(scalarType -> result.add(List.of(scalarType.get(0), INTEGER_T))); return result; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java index 7562dac74d8..3ec4911821c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/CoercionUtils.java @@ -37,7 +37,14 @@ public final class CoercionUtils { */ public static @Nullable List castArguments( RexBuilder builder, PPLTypeChecker typeChecker, List arguments) { - List> paramTypeCombinations = typeChecker.getParameterTypes(); + List> paramTypeCombinations = + typeChecker.getParameterTypes().stream() + .map( + types -> + types.stream() + .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) + .toList()) + .toList(); List sourceTypes = arguments.stream() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index d64f04bb9ad..2a670af3fee 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -246,11 +246,12 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator SECOND = new DatePartFunction(TimeUnit.SECOND).toUDF("SECOND"); public static final SqlOperator MICROSECOND = new DatePartFunction(TimeUnit.MICROSECOND).toUDF("MICROSECOND"); - public static final SqlOperator NOW = new CurrentFunction(ExprCoreType.TIMESTAMP).toUDF("NOW"); + public static final SqlOperator NOW = + new CurrentFunction(CurrentFunction.Kind.TIMESTAMP).toUDF("NOW"); public static final SqlOperator CURRENT_TIME = - new CurrentFunction(ExprCoreType.TIME).toUDF("CURRENT_TIME"); + new CurrentFunction(CurrentFunction.Kind.TIME).toUDF("CURRENT_TIME"); public static final SqlOperator CURRENT_DATE = - new CurrentFunction(ExprCoreType.DATE).toUDF("CURRENT_DATE"); + new CurrentFunction(CurrentFunction.Kind.DATE).toUDF("CURRENT_DATE"); public static final SqlOperator DATE_FORMAT = new FormatFunction(ExprCoreType.DATE).toUDF("DATE_FORMAT"); public static final SqlOperator TIME_FORMAT = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 151c4a96655..64f29906829 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -309,6 +309,7 @@ import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.calcite.tools.RelBuilder; @@ -319,8 +320,6 @@ import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.CollectionUDF.MVIndexFunctionImp; @@ -450,7 +449,7 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { return false; } try { - List> signatures = + List> signatures = checker.getParameterTypes().stream() .filter(parameters -> argumentIndex < parameters.size()) .toList(); @@ -458,12 +457,12 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { return false; } foundArgument = true; - List acceptedTypes = + List acceptedTypes = signatures.stream().map(parameters -> parameters.get(argumentIndex)).toList(); - if (acceptedTypes.stream().allMatch(ExprCoreType.numberTypes()::contains)) { + if (acceptedTypes.stream().allMatch(SqlTypeUtil::isNumeric)) { continue; } - if (acceptedTypes.stream().anyMatch(type -> type != ExprCoreType.UNKNOWN) + if (acceptedTypes.stream().anyMatch(type -> type.getSqlTypeName() != SqlTypeName.ANY) || !requiresNumericByValidation(checker, signatures, argumentIndex)) { return false; } @@ -475,7 +474,7 @@ public boolean requiresNumericArgument(String functionName, int argumentIndex) { } private boolean requiresNumericByValidation( - PPLTypeChecker checker, List> signatures, int argumentIndex) { + PPLTypeChecker checker, List> signatures, int argumentIndex) { RelDataType numericType = TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE); RelDataType stringType = TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR); return signatures.stream() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java index 521764ba7bb..c3de664443d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java @@ -27,8 +27,10 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.Pair; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -61,12 +63,9 @@ public interface PPLTypeChecker { /** * Get a list of all possible parameter type combinations for the function. * - *

This method is used to generate the allowed signatures for the function based on the - * parameter types. - * * @return a list of lists, where each inner list represents an allowed parameter type combination */ - List> getParameterTypes(); + List> getParameterTypes(); private static boolean validateOperands( List funcTypeFamilies, List operandTypes) { @@ -111,8 +110,8 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - return PPLTypeChecker.getExprSignatures(families); + public List> getParameterTypes() { + return PPLTypeChecker.getRelDataTypeSignatures(families); } @Override @@ -150,17 +149,17 @@ public boolean checkOperandTypes(List types) { @Override public String getAllowedSignatures() { if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - var allowedExprSignatures = getExprSignatures(familyOperandTypeChecker); - return PPLTypeChecker.formatExprSignatures(allowedExprSignatures); + var allowedSignatures = getRelDataTypeSignatures(familyOperandTypeChecker); + return PPLTypeChecker.formatSignatures(allowedSignatures); } else { return ""; } } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { if (innerTypeChecker instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - return getExprSignatures(familyOperandTypeChecker); + return getRelDataTypeSignatures(familyOperandTypeChecker); } else { // If the inner type checker is not a FamilyOperandTypeChecker, we cannot provide // parameter types. @@ -232,11 +231,11 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - List> parameterTypes = new ArrayList<>(); + public List> getParameterTypes() { + List> parameterTypes = new ArrayList<>(); for (SqlOperandTypeChecker rule : allowedRules) { if (rule instanceof FamilyOperandTypeChecker familyOperandTypeChecker) { - parameterTypes.addAll(getExprSignatures(familyOperandTypeChecker)); + parameterTypes.addAll(getRelDataTypeSignatures(familyOperandTypeChecker)); } else { throw new IllegalArgumentException( "Currently only compositions of FamilyOperandTypeChecker are supported"); @@ -337,9 +336,10 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { - // Should not be used - return List.of(List.of(ExprCoreType.UNKNOWN, ExprCoreType.UNKNOWN)); + public List> getParameterTypes() { + // Should not be used by coercion since comparable operators don't drive type widening here. + RelDataType anyType = OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY); + return List.of(List.of(anyType, anyType)); } } @@ -397,23 +397,23 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { if (internal instanceof FamilyOperandTypeChecker familyChecker) { - return getExprSignatures(familyChecker); + return getRelDataTypeSignatures(familyChecker); } else { - // For unknown type checkers, return UNKNOWN types + // For unknown type checkers, return ANY-typed signatures. int min = internal.getOperandCountRange().getMin(); int max = internal.getOperandCountRange().getMax(); + RelDataType anyType = OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY); if (min == -1 || max == -1) { - // Variable arguments - return a single signature with UNKNOWN - return List.of(List.of(ExprCoreType.UNKNOWN)); + return List.of(List.of(anyType)); } else { - List> parameterTypes = new ArrayList<>(); + List> parameterTypes = new ArrayList<>(); final int MAX_ARGS = 10; max = Math.min(MAX_ARGS, max); for (int i = min; i <= max; i++) { - parameterTypes.add(Collections.nCopies(i, ExprCoreType.UNKNOWN)); + parameterTypes.add(Collections.nCopies(i, anyType)); } return parameterTypes; } @@ -513,28 +513,27 @@ static PPLDefaultTypeChecker wrapDefault(SqlOperandTypeChecker typeChecker) { } /** - * Create a {@link PPLTypeChecker} from a list of allowed signatures consisted of {@link - * ExprType}. This is useful to validate arguments against user-defined types (UDT) that does not - * match any Calcite {@link SqlTypeFamily}. + * Create a {@link PPLTypeChecker} from a list of allowed signatures composed of {@link + * RelDataType}. This is used for functions whose argument types include user-defined types (UDTs) + * that don't fit any standard {@link SqlTypeFamily}. * * @param allowedSignatures a list of allowed signatures, where each signature is a list of {@link - * ExprType} representing the expected types of the function arguments. + * RelDataType} representing the expected types of the function arguments. * @return a {@link PPLTypeChecker} that checks if the operand types match any of the allowed * signatures */ - static PPLTypeChecker wrapUDT(List> allowedSignatures) { + static PPLTypeChecker wrapUDT(List> allowedSignatures) { return new PPLTypeChecker() { @Override public boolean checkOperandTypes(List types) { - List argExprTypes = - types.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); for (var allowedSignature : allowedSignatures) { if (allowedSignature.size() != types.size()) { continue; // Skip signatures that do not match the operand count } - // Check if the argument types match the allowed signature + // Match each operand against the allowed signature using nullability-insensitive + // equality with UDT-class-aware comparison. if (IntStream.range(0, allowedSignature.size()) - .allMatch(i -> allowedSignature.get(i).equals(argExprTypes.get(i)))) { + .allMatch(i -> typesMatch(allowedSignature.get(i), types.get(i)))) { return true; } } @@ -543,17 +542,36 @@ public boolean checkOperandTypes(List types) { @Override public String getAllowedSignatures() { - return PPLTypeChecker.formatExprSignatures(allowedSignatures); + return PPLTypeChecker.formatSignatures(allowedSignatures); } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { return allowedSignatures; } }; } + /** + * Compares two RelDataTypes for signature matching. Two UDTs match if they share the same {@link + * ExprUDT} tag — comparing {@code getClass()} is unsafe because addCharsetAndCollation collapses + * ExprDateType/ExprTimeType/ExprTimeStampType/ExprBinaryType down to ExprSqlType, so different + * UDTs would appear equal. Plain types match by SqlTypeName. + */ + private static boolean typesMatch(RelDataType expected, RelDataType actual) { + if (expected instanceof AbstractExprRelDataType expUdt + && actual instanceof AbstractExprRelDataType actUdt) { + return expUdt.getUdt() == actUdt.getUdt(); + } + if (expected instanceof AbstractExprRelDataType + || actual instanceof AbstractExprRelDataType) { + return false; + } + return expected.getSqlTypeName() == actual.getSqlTypeName(); + } + // Util Functions + /** * Generates a list of allowed function signatures based on the provided {@link * FamilyOperandTypeChecker}. The signatures are generated by iterating through the operand count @@ -563,14 +581,15 @@ public List> getParameterTypes() { * to 10 to avoid excessive enumeration. * * @param typeChecker the {@link FamilyOperandTypeChecker} to use for generating signatures - * @return a list of allowed function signatures + * @return a string representation of allowed function signatures */ private static String getFamilySignatures(FamilyOperandTypeChecker typeChecker) { - var allowedExprSignatures = getExprSignatures(typeChecker); - return formatExprSignatures(allowedExprSignatures); + var allowedSignatures = getRelDataTypeSignatures(typeChecker); + return formatSignatures(allowedSignatures); } - private static List> getExprSignatures(FamilyOperandTypeChecker typeChecker) { + private static List> getRelDataTypeSignatures( + FamilyOperandTypeChecker typeChecker) { var operandCountRange = typeChecker.getOperandCountRange(); int min = operandCountRange.getMin(); int max = operandCountRange.getMax(); @@ -578,91 +597,79 @@ private static List> getExprSignatures(FamilyOperandTypeChecker t for (int i = 0; i < min; i++) { families.add(typeChecker.getOperandSqlTypeFamily(i)); } - List> allowedSignatures = new ArrayList<>(getExprSignatures(families)); + List> allowedSignatures = new ArrayList<>(getRelDataTypeSignatures(families)); // Avoid enumerating signatures for infinite args final int MAX_ARGS = 10; max = Math.min(max, MAX_ARGS); for (int i = min; i < max; i++) { families.add(typeChecker.getOperandSqlTypeFamily(i)); - allowedSignatures.addAll(getExprSignatures(families)); + allowedSignatures.addAll(getRelDataTypeSignatures(families)); } return allowedSignatures; } /** - * Converts a {@link SqlTypeFamily} to a list of {@link ExprType}. This method is used to display - * the allowed signatures for functions based on their type families. - * - * @param family the {@link SqlTypeFamily} to convert - * @return a list of {@link ExprType} corresponding to the concrete types of the family + * Converts a {@link SqlTypeFamily} to a list of concrete {@link RelDataType} representatives. + * Used to enumerate allowed signatures and to drive widening in PPL coercion. */ - private static List getExprTypes(SqlTypeFamily family) { - List concreteTypes = - switch (family) { - case DATETIME -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DATE), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIME)); - case NUMERIC -> - List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE)); - // Integer is mapped to BIGINT in family.getDefaultConcreteType - case INTEGER -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER)); - case ANY, IGNORE -> - List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY)); - case DATETIME_INTERVAL -> - SqlTypeName.INTERVAL_TYPES.stream() - .map( - type -> - OpenSearchTypeFactory.TYPE_FACTORY.createSqlIntervalType( + private static List getRelDataTypes(SqlTypeFamily family) { + OpenSearchTypeFactory tf = OpenSearchTypeFactory.TYPE_FACTORY; + return switch (family) { + case DATETIME -> + List.of( + tf.createUDT(ExprUDT.EXPR_TIMESTAMP), + tf.createUDT(ExprUDT.EXPR_DATE), + tf.createUDT(ExprUDT.EXPR_TIME)); + case TIMESTAMP -> List.of(tf.createUDT(ExprUDT.EXPR_TIMESTAMP)); + case DATE -> List.of(tf.createUDT(ExprUDT.EXPR_DATE)); + case TIME -> List.of(tf.createUDT(ExprUDT.EXPR_TIME)); + case NUMERIC -> + List.of(tf.createSqlType(SqlTypeName.INTEGER), tf.createSqlType(SqlTypeName.DOUBLE)); + // Integer is mapped to BIGINT in family.getDefaultConcreteType + case INTEGER -> List.of(tf.createSqlType(SqlTypeName.INTEGER)); + case ANY, IGNORE -> List.of(tf.createSqlType(SqlTypeName.ANY)); + // ARRAY of nullable ANY, matching convertExprTypeToRelDataType(ARRAY, nullable=true). + // Calcite's default concrete type for ARRAY has a NOT NULL element, which diverges from + // PPL semantics (see #5175: null literals must remain nullable inside an array). + case ARRAY -> List.of(tf.createArrayType(tf.createSqlType(SqlTypeName.ANY, true), -1)); + case BINARY -> List.of(tf.createUDT(ExprUDT.EXPR_BINARY)); + case DATETIME_INTERVAL -> + SqlTypeName.INTERVAL_TYPES.stream() + .map( + type -> + (RelDataType) + tf.createSqlIntervalType( new SqlIntervalQualifier( type.getStartUnit(), type.getEndUnit(), SqlParserPos.ZERO))) - .collect(Collectors.toList()); - default -> { - RelDataType type = family.getDefaultConcreteType(OpenSearchTypeFactory.TYPE_FACTORY); - if (type == null) { - yield List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.OTHER)); - } - yield List.of(type); - } - }; - return concreteTypes.stream() - .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) - .distinct() - .collect(Collectors.toList()); + .collect(Collectors.toList()); + default -> { + RelDataType type = family.getDefaultConcreteType(tf); + if (type == null) { + yield List.of(tf.createSqlType(SqlTypeName.OTHER)); + } + yield List.of(type); + } + }; } /** - * Generates a list of all possible {@link ExprType} signatures based on the provided {@link - * SqlTypeFamily} list. - * - * @param families the list of {@link SqlTypeFamily} to generate signatures for - * @return a list of lists, where each inner list contains {@link ExprType} signatures + * Generates a list of all possible {@link RelDataType} signatures based on the provided {@link + * SqlTypeFamily} list (cartesian product per-position). */ - private static List> getExprSignatures(List families) { - List> exprTypes = - families.stream().map(PPLTypeChecker::getExprTypes).collect(Collectors.toList()); - - // Do a cartesian product of all ExprTypes in the family - return Lists.cartesianProduct(exprTypes); + private static List> getRelDataTypeSignatures(List families) { + List> perPosition = + families.stream().map(PPLTypeChecker::getRelDataTypes).collect(Collectors.toList()); + return Lists.cartesianProduct(perPosition); } /** * Generates a string representation of the function signature based on the provided type * families. The format is a list of type families enclosed in square brackets, e.g.: "[INTEGER, * STRING]". - * - * @param families the list of type families to include in the signature - * @return a string representation of the function signature */ private static String getFamilySignature(List families) { - List> signatures = getExprSignatures(families); - // Convert each signature to a string representation and then concatenate them - return formatExprSignatures(signatures); + return formatSignatures(getRelDataTypeSignatures(families)); } /** @@ -686,13 +693,20 @@ private static boolean isCompositionOr(CompositeOperandTypeChecker typeChecker) return composition == CompositeOperandTypeChecker.Composition.OR; } - private static String formatExprSignatures(List> signatures) { + /** + * Renders a list of {@link RelDataType} signatures as a pipe-separated string of bracketed + * signatures, e.g. {@code [INTEGER,STRING]|[DOUBLE,STRING]}. Each type is rendered through {@link + * OpenSearchTypeFactory#convertRelDataTypeToExprType} so plain SQL types come out as their PPL + * names ({@code STRING}, {@code LONG}, ...) and UDTs as their {@code ExprCoreType} names; {@code + * UNDEFINED} (Calcite {@code NULL}/{@code ANY}) is displayed as {@code ANY}. + */ + static String formatSignatures(List> signatures) { return signatures.stream() .map( types -> "[" + types.stream() - // Display ExprCoreType.UNDEFINED as "ANY" for better interpretability + .map(OpenSearchTypeFactory::convertRelDataTypeToExprType) .map(t -> t == ExprCoreType.UNDEFINED ? "ANY" : t.toString()) .collect(Collectors.joining(",")) + "]") diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java index dc4761b26e7..d67b612e0ea 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UDFOperandMetadata.java @@ -17,7 +17,6 @@ import org.apache.calcite.sql.type.SqlOperandMetadata; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.opensearch.sql.data.type.ExprType; /** * This class is created for the compatibility with {@link SqlUserDefinedFunction} constructors when @@ -106,11 +105,12 @@ public String getAllowedSignatures(SqlOperator op, String opName) { }; } - static UDFOperandMetadata wrapUDT(List> allowSignatures) { + static UDFOperandMetadata wrapUDT(List> allowSignatures) { return new UDTOperandMetadata(allowSignatures); } - record UDTOperandMetadata(List> allowedParamTypes) implements UDFOperandMetadata { + record UDTOperandMetadata(List> allowedParamTypes) + implements UDFOperandMetadata { @Override public SqlOperandTypeChecker getInnerTypeChecker() { return this; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java index 49e06afa3d6..f620ffd21c5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/CurrentFunction.java @@ -20,8 +20,6 @@ import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.function.FunctionProperties; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -39,21 +37,28 @@ *

It returns the current date, time, or timestamp based on the specified return type. */ public class CurrentFunction extends ImplementorUDF { - private final ExprType returnType; - public CurrentFunction(ExprType returnType) { - super(new CurrentFunctionImplementor(returnType), NullPolicy.NONE); - this.returnType = returnType; + /** Discriminates the temporal flavour at registration time, decoupled from {@code ExprType}. */ + public enum Kind { + DATE, + TIME, + TIMESTAMP + } + + private final Kind kind; + + public CurrentFunction(Kind kind) { + super(new CurrentFunctionImplementor(kind), NullPolicy.NONE); + this.kind = kind; } @Override public SqlReturnTypeInference getReturnTypeInference() { return opBinding -> - switch (returnType) { - case ExprCoreType.DATE -> UserDefinedFunctionUtils.NULLABLE_DATE_UDT; - case ExprCoreType.TIME -> UserDefinedFunctionUtils.NULLABLE_TIME_UDT; - case ExprCoreType.TIMESTAMP -> UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; - default -> throw new IllegalArgumentException("Unsupported return type: " + returnType); + switch (kind) { + case DATE -> UserDefinedFunctionUtils.NULLABLE_DATE_UDT; + case TIME -> UserDefinedFunctionUtils.NULLABLE_TIME_UDT; + case TIMESTAMP -> UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; }; } @@ -64,18 +69,17 @@ public UDFOperandMetadata getOperandMetadata() { @RequiredArgsConstructor public static class CurrentFunctionImplementor implements NotNullImplementor { - private final ExprType returnType; + private final Kind kind; @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { String functionName = - switch (returnType) { - case ExprCoreType.DATE -> "currentDate"; - case ExprCoreType.TIME -> "currentTime"; - case ExprCoreType.TIMESTAMP -> "currentTimestamp"; - default -> throw new IllegalArgumentException("Unsupported return type: " + returnType); + switch (kind) { + case DATE -> "currentDate"; + case TIME -> "currentTime"; + case TIMESTAMP -> "currentTimestamp"; }; Expression properties = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java index 109bad16bf1..a53c733de66 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/PeriodNameFunction.java @@ -17,11 +17,9 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprDateValue; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -65,9 +63,6 @@ public PeriodNameFunctionImplementor(TimeUnit periodUnit) { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - ExprType dateType = - OpenSearchTypeFactory.convertRelDataTypeToExprType( - call.getOperands().getFirst().getType()); return Expressions.call( PeriodNameFunctionImplementor.class, "name", diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java index 11fdd7947af..b213181038b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CidrMatchFunction.java @@ -14,10 +14,10 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.sql.expression.ip.IPFunctions; @@ -51,9 +51,9 @@ public UDFOperandMetadata getOperandMetadata() { // We use a specific type checker to serve return UDFOperandMetadata.wrapUDT( List.of( - List.of(ExprCoreType.IP, ExprCoreType.STRING), - List.of(ExprCoreType.STRING, ExprCoreType.STRING), - List.of(ExprCoreType.BINARY, ExprCoreType.STRING))); + List.of(PPLOperandTypes.IP_UDT, PPLOperandTypes.STRING_T), + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.STRING_T), + List.of(PPLOperandTypes.BINARY_UDT, PPLOperandTypes.STRING_T))); } public static class CidrMatchImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index ce200323f60..34d5fec2f3e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -25,8 +25,8 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -120,7 +120,8 @@ public SqlReturnTypeInference getReturnTypeInference() { @Override public UDFOperandMetadata getOperandMetadata() { - return UDFOperandMetadata.wrapUDT(List.of(List.of(ExprCoreType.IP, ExprCoreType.IP))); + return UDFOperandMetadata.wrapUDT( + List.of(List.of(PPLOperandTypes.IP_UDT, PPLOperandTypes.IP_UDT))); } public static class CompareImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java index baf6b8a37e1..13dccfee6ae 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java @@ -11,13 +11,14 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.opensearch.sql.calcite.type.ExprIPType; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprIpValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -41,7 +42,7 @@ public IPFunction() { @Override public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrapUDT( - List.of(List.of(ExprCoreType.IP), List.of(ExprCoreType.STRING))); + List.of(List.of(PPLOperandTypes.IP_UDT), List.of(PPLOperandTypes.STRING_T))); } @Override @@ -57,12 +58,10 @@ public Expression implement( if (call.getOperands().size() != 1) { throw new IllegalArgumentException("IP function requires exactly one operand"); } - ExprType argType = - OpenSearchTypeFactory.convertRelDataTypeToExprType( - call.getOperands().getFirst().getType()); - if (argType == ExprCoreType.IP) { + RelDataType argType = call.getOperands().getFirst().getType(); + if (argType instanceof ExprIPType) { return translatedOperands.getFirst(); - } else if (argType == ExprCoreType.STRING) { + } else if (SqlTypeUtil.isCharacter(argType)) { return Expressions.new_(ExprIpValue.class, translatedOperands); } else { throw new ExpressionEvaluationException( diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java index 5533fd90915..2727710fc18 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; class CoercionUtilsTest { @@ -88,7 +87,8 @@ void widenArgumentsUnifiesPlainTimestampWithDateUdtBounds() { @Test void castArgumentsReturnsExactMatchWhenAvailable() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); + PPLTypeChecker typeChecker = + new StubTypeChecker(List.of(List.of(sqlType(INTEGER)), List.of(sqlType(DOUBLE)))); List arguments = List.of(nullLiteral(INTEGER)); List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); @@ -102,7 +102,7 @@ void castArgumentsReturnsExactMatchWhenAvailable() { @Test void castArgumentsFallsBackToWidestCandidate() { PPLTypeChecker typeChecker = - new StubTypeChecker(List.of(List.of(ExprCoreType.LONG), List.of(DOUBLE))); + new StubTypeChecker(List.of(List.of(sqlType(ExprCoreType.LONG)), List.of(sqlType(DOUBLE)))); List arguments = List.of(nullLiteral(STRING)); List result = CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments); @@ -114,16 +114,23 @@ void castArgumentsFallsBackToWidestCandidate() { @Test void castArgumentsReturnsNullWhenNoCompatibleSignatureExists() { - PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(ExprCoreType.GEO_POINT))); + PPLTypeChecker typeChecker = + new StubTypeChecker( + List.of( + List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.GEOMETRY)))); List arguments = List.of(nullLiteral(INTEGER)); assertNull(CoercionUtils.castArguments(REX_BUILDER, typeChecker, arguments)); } + private static RelDataType sqlType(ExprCoreType type) { + return OpenSearchTypeFactory.convertExprTypeToRelDataType(type); + } + private static class StubTypeChecker implements PPLTypeChecker { - private final List> signatures; + private final List> signatures; - private StubTypeChecker(List> signatures) { + private StubTypeChecker(List> signatures) { this.signatures = signatures; } @@ -138,7 +145,7 @@ public String getAllowedSignatures() { } @Override - public List> getParameterTypes() { + public List> getParameterTypes() { return signatures; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java b/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java new file mode 100644 index 00000000000..ba03a8b6280 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/PPLComparableTypeCheckerTest.java @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SameOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.expression.function.PPLTypeChecker.PPLComparableTypeChecker; + +/** + * Exercises {@link PPLComparableTypeChecker#checkOperandTypes} against representative type pairs. + */ +class PPLComparableTypeCheckerTest { + + private static final OpenSearchTypeFactory TF = OpenSearchTypeFactory.TYPE_FACTORY; + private static final PPLComparableTypeChecker CHECKER = + new PPLComparableTypeChecker((SameOperandTypeChecker) OperandTypes.SAME_SAME); + + private static RelDataType sql(SqlTypeName name) { + return TF.createSqlType(name); + } + + private static RelDataType udt(ExprUDT udt) { + return TF.createUDT(udt); + } + + private static RelDataType udt(ExprUDT udt, boolean nullable) { + return TF.createUDT(udt, nullable); + } + + private static RelDataType interval(TimeUnit unit) { + return TF.createSqlIntervalType(new SqlIntervalQualifier(unit, unit, SqlParserPos.ZERO)); + } + + private static boolean comparable(RelDataType a, RelDataType b) { + return CHECKER.checkOperandTypes(List.of(a, b)); + } + + @Test + void numericAndNumericAreComparable() { + assertTrue(comparable(sql(SqlTypeName.INTEGER), sql(SqlTypeName.DOUBLE))); + assertTrue(comparable(sql(SqlTypeName.TINYINT), sql(SqlTypeName.BIGINT))); + } + + @Test + void sameUdtIsComparable() { + assertTrue(comparable(udt(ExprUDT.EXPR_DATE), udt(ExprUDT.EXPR_DATE))); + assertTrue(comparable(udt(ExprUDT.EXPR_TIMESTAMP, true), udt(ExprUDT.EXPR_TIMESTAMP, false))); + } + + @Test + void plainBinaryVsBinaryUdtAreComparable() { + // Guardrail: any future refactor of isComparable that classifies via SqlTypeFamily or Java + // class would break this pair — VARBINARY (family=BINARY) and EXPR_BINARY (VARCHAR-backed + // UDT) look unrelated at that level, but both map to ExprCoreType.BINARY and must compare. + assertTrue(comparable(sql(SqlTypeName.VARBINARY), udt(ExprUDT.EXPR_BINARY))); + assertTrue(comparable(udt(ExprUDT.EXPR_BINARY), sql(SqlTypeName.VARBINARY))); + } + + @Test + void dayTimeAndYearMonthIntervalsAreComparable() { + // Guardrail: Calcite splits INTERVAL into day-time and year-month SqlTypeFamilies, so a + // family-based check would reject this pair. convertRelDataTypeToExprType collapses every + // interval SqlTypeName to ExprCoreType.INTERVAL, so shouldCast is false and both compare. + assertTrue(comparable(interval(TimeUnit.DAY), interval(TimeUnit.YEAR))); + assertTrue(comparable(interval(TimeUnit.HOUR), interval(TimeUnit.MONTH))); + } + + @Test + void plainTemporalVsMatchingTemporalUdtIsComparable() { + assertTrue(comparable(sql(SqlTypeName.TIMESTAMP), udt(ExprUDT.EXPR_TIMESTAMP))); + assertTrue(comparable(sql(SqlTypeName.DATE), udt(ExprUDT.EXPR_DATE))); + assertTrue(comparable(sql(SqlTypeName.TIME), udt(ExprUDT.EXPR_TIME))); + } + + @Test + void udtVsUnrelatedPlainTypeIsNotComparable() { + assertFalse(comparable(udt(ExprUDT.EXPR_DATE), sql(SqlTypeName.VARCHAR))); + assertFalse(comparable(udt(ExprUDT.EXPR_TIMESTAMP), sql(SqlTypeName.INTEGER))); + } + + @Test + void stringVsNumericIsNotComparable() { + assertFalse(comparable(sql(SqlTypeName.VARCHAR), sql(SqlTypeName.INTEGER))); + } + + @Test + void anyIsComparableWithAnything() { + assertTrue(comparable(sql(SqlTypeName.ANY), sql(SqlTypeName.INTEGER))); + assertTrue(comparable(sql(SqlTypeName.ANY), udt(ExprUDT.EXPR_DATE))); + } + + @Test + void structVsNonStructIsNotComparable() { + RelDataType struct = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + assertFalse(comparable(struct, sql(SqlTypeName.INTEGER))); + } + + @Test + void structsWithMatchingFieldsAreComparable() { + RelDataType s1 = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + RelDataType s2 = + TF.createStructType( + List.of(sql(SqlTypeName.BIGINT), sql(SqlTypeName.CHAR)), List.of("x", "y")); + assertTrue(comparable(s1, s2)); + } + + @Test + void structsWithMismatchedFieldCountsAreNotComparable() { + RelDataType s1 = TF.createStructType(List.of(sql(SqlTypeName.INTEGER)), List.of("a")); + RelDataType s2 = + TF.createStructType( + List.of(sql(SqlTypeName.INTEGER), sql(SqlTypeName.VARCHAR)), List.of("a", "b")); + assertFalse(comparable(s1, s2)); + } + + @Test + void ipTypesAreRejectedByOuterChecker() { + // IP UDTs are explicitly filtered out in PPLComparableTypeChecker.checkOperandTypes so that + // built-in comparable functions (COALESCE, NULLIF, IFNULL, IF) cannot accept them. + assertFalse(comparable(udt(ExprUDT.EXPR_IP), udt(ExprUDT.EXPR_IP))); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 83b1915f6b5..d2e3c52bb6b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -19,12 +19,12 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.geospatial.action.IpEnrichmentActionClient; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; import org.opensearch.transport.client.node.NodeClient; @@ -60,8 +60,8 @@ public SqlReturnTypeInference getReturnTypeInference() { public UDFOperandMetadata getOperandMetadata() { return UDFOperandMetadata.wrapUDT( List.of( - List.of(ExprCoreType.STRING, ExprCoreType.IP), - List.of(ExprCoreType.STRING, ExprCoreType.IP, ExprCoreType.STRING))); + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.IP_UDT), + List.of(PPLOperandTypes.STRING_T, PPLOperandTypes.IP_UDT, PPLOperandTypes.STRING_T))); } public static class GeoIPImplementor implements NotNullImplementor { From 2b329a752193c6a9f06ca52b38b8bd773a50de5f Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 22 Jul 2026 10:22:26 -0700 Subject: [PATCH 07/78] Anonymize xyseries command and mark it experimental in docs (#5643) Add visitXyseries to PPLQueryDataAnonymizer so the xyseries stage appears in the anonymized query logged for every PPL request, with pivot literals and options masked. Previously the visitor fell through to visitChildren and the entire xyseries clause was silently dropped from log output. Also flip the xyseries entry in docs/user/ppl/index.md from stable to experimental (since 3.8), matching how other newly-introduced commands are listed. Signed-off-by: Peng Huo --- docs/user/ppl/index.md | 2 +- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 21 +++++++++++++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 23 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 067d5f524fd..647c4568446 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -89,7 +89,7 @@ source=accounts | [nomv command](cmd/nomv.md) | 3.6 | stable (since 3.6) | Converts a multivalue field to a single-value string by joining elements with newlines. | | [mvexpand command](cmd/mvexpand.md) | 3.6 | stable (since 3.6) | Expand a multi-valued field into separate documents (one per value). | | [graphlookup command](cmd/graphlookup.md) | 3.6 | experimental (since 3.6) | Performs recursive graph traversal on a collection using a BFS algorithm.| -| [xyseries command](cmd/xyseries.md) | 3.8 | stable (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | +| [xyseries command](cmd/xyseries.md) | 3.8 | experimental (since 3.8) | Converts row-oriented grouped results into a wide table format suitable for chart visualizations. One field serves as the X axis (row key), one field provides pivot values for generating output column names, and one or more data fields fill the pivoted cells. Only rows matching the explicitly provided pivot values in the `in` clause are included. | - [Syntax](cmd/syntax.md) - PPL query structure and command syntax formatting * **Functions** diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 11c47d137e8..398361fcea7 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -113,6 +113,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; +import org.opensearch.sql.ast.tree.Xyseries; import org.opensearch.sql.calcite.plan.OpenSearchConstants; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.StringUtils; @@ -824,6 +825,26 @@ public String visitTranspose(Transpose node, String context) { return anonymized.toString(); } + @Override + public String visitXyseries(Xyseries node, String context) { + String child = node.getChild().get(0).accept(this, context); + StringBuilder command = new StringBuilder(); + command.append(" | xyseries"); + if (node.getSeparator() != null && !": ".equals(node.getSeparator())) { + command.append(" sep=").append(MASK_LITERAL); + } + if (node.getFormat() != null) { + command.append(" format=").append(MASK_LITERAL); + } + command.append(" ").append(visitExpression(node.getXField())); + command.append(" ").append(visitExpression(node.getYNameField())); + command.append(" in (").append(MASK_LITERAL).append(")"); + String dataFields = + node.getYDataFields().stream().map(this::visitExpression).collect(Collectors.joining(",")); + command.append(" ").append(dataFields); + return StringUtils.format("%s%s", child, command.toString()); + } + @Override public String visitAppendCol(AppendCol node, String context) { String child = node.getChild().get(0).accept(this, context); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index eba4f57112b..63f7dea81d7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -327,6 +327,29 @@ public void testChartCommandOverBy() { anonymize("source=t | chart sum(amount) over gender by age")); } + @Test + public void testXyseriesCommand() { + assertEquals( + "source=table | stats avg(identifier) by identifier,identifier" + + " | xyseries identifier identifier in (***) identifier", + anonymize( + "source=t | stats avg(balance) by gender, state" + + " | xyseries state gender in (\"F\",\"M\") avg_balance")); + } + + @Test + public void testXyseriesCommandWithOptions() { + assertEquals( + "source=table | stats avg(identifier),max(identifier) by identifier,identifier" + + " | xyseries sep=*** format=*** identifier identifier in (***)" + + " identifier,identifier", + anonymize( + "source=t | stats avg(balance) as avg_balance, max(balance) as max_balance" + + " by gender, state" + + " | xyseries sep=\"_\" format=\"$AGG$_$VAL$\" state gender" + + " in (\"F\",\"M\") avg_balance, max_balance")); + } + // todo, sort order is ignored, it doesn't impact the log analysis. @Test public void testSortCommandWithOptions() { From fff8c42de51011903cfa5cf37bad8c3050b2ccb4 Mon Sep 17 00:00:00 2001 From: opensearch-ci <83309141+opensearch-ci-bot@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:31:08 -0400 Subject: [PATCH 08/78] Add release notes for 3.8.0 (#5644) Signed-off-by: opensearch-ci-bot --- .../opensearch-sql.release-notes-3.8.0.0.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-3.8.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-3.8.0.0.md b/release-notes/opensearch-sql.release-notes-3.8.0.0.md new file mode 100644 index 00000000000..61db95af498 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-3.8.0.0.md @@ -0,0 +1,80 @@ +## Version 3.8.0 Release Notes + +Compatible with OpenSearch and OpenSearch Dashboards version 3.8.0 + +### Features + +* Add PPL `xyseries` command for pivoting row-oriented grouped results into wide tables ([#5343](https://github.com/opensearch-project/sql/pull/5343)) +* Add PPL `timewrap` command for time-period comparison over timechart output ([#5241](https://github.com/opensearch-project/sql/pull/5241)) +* Add PPL `foreach` command for iterating over field lists, multivalue fields, and JSON arrays ([#5613](https://github.com/opensearch-project/sql/pull/5613)) +* Add PPL `makeresults` command for generating in-memory rows without an index scan ([#5622](https://github.com/opensearch-project/sql/pull/5622)) + +### Enhancements + +* Anonymize `xyseries` command and mark it as experimental in documentation ([#5643](https://github.com/opensearch-project/sql/pull/5643)) +* Suggest similar field names in 'field not found' error messages ([#5402](https://github.com/opensearch-project/sql/pull/5402)) +* Support `constant_keyword` field type in PPL, treating it as a string ([#5639](https://github.com/opensearch-project/sql/pull/5639)) +* Decouple Calcite PPL planning from ExprType, operating on RelDataType directly ([#5633](https://github.com/opensearch-project/sql/pull/5633)) +* Support bare-field join criteria shorthand (`join on `) in PPL ([#5517](https://github.com/opensearch-project/sql/pull/5517)) +* Classify unsupported-feature errors as client errors (4xx) on the SQL path ([#5569](https://github.com/opensearch-project/sql/pull/5569)) +* Reject unsupported output formats on the analytics-engine route with a 4xx error ([#5570](https://github.com/opensearch-project/sql/pull/5570)) +* Widen narrow integer operands in PPL arithmetic to prevent silent overflow ([#5603](https://github.com/opensearch-project/sql/pull/5603)) +* Add configurable expression depth limit during AST building to prevent stack overflow ([#5602](https://github.com/opensearch-project/sql/pull/5602)) +* Add `json_tree` machine-readable explain format accessible via `_explain?format=json_tree` ([#5576](https://github.com/opensearch-project/sql/pull/5576)) +* Onboard new backport-pr reusable GitHub workflow ([#5586](https://github.com/opensearch-project/sql/pull/5586)) +* Return all columns including struct and nested fields when using `head` command ([#5518](https://github.com/opensearch-project/sql/pull/5518)) +* Bring `CalcitePPLBasicIT` to parity on the analytics-engine route ([#5542](https://github.com/opensearch-project/sql/pull/5542)) +* Bring `CalciteWhereCommandIT` to parity on the analytics-engine route ([#5546](https://github.com/opensearch-project/sql/pull/5546)) +* Stabilize order-dependent PPL ITs with explicit sort for multi-shard analytics runs ([#5537](https://github.com/opensearch-project/sql/pull/5537)) +* Align `DateTimeComparisonIT` today's date computation to UTC for analytics-engine compatibility ([#5543](https://github.com/opensearch-project/sql/pull/5543)) +* Fix NPE on `case()` with incompatible branch types, returning a clean 400 error ([#5575](https://github.com/opensearch-project/sql/pull/5575)) +* Fix NPE when `rex` sits inside `appendcol` subsearch for the analytics engine ([#5574](https://github.com/opensearch-project/sql/pull/5574)) + +### Bug Fixes + +* Fix `ClassCastException` in PPL multisearch on indexes with `@timestamp` alias field ([#5577](https://github.com/opensearch-project/sql/pull/5577)) +* Fix PPL `foreach` JSON array type coercion to handle non-numeric elements gracefully ([#5637](https://github.com/opensearch-project/sql/pull/5637)) +* Detect long (BIGINT) arithmetic overflow instead of silently wrapping ([#5604](https://github.com/opensearch-project/sql/pull/5604)) +* Preserve SQL-layer profiling alongside the analytics-engine profile ([#5571](https://github.com/opensearch-project/sql/pull/5571)) +* Propagate request-task cancellation into the analytics PPL route ([#5563](https://github.com/opensearch-project/sql/pull/5563)) +* Return 4xx instead of 500 for unsupported window functions ([#5587](https://github.com/opensearch-project/sql/pull/5587)) +* Fix `SHOW`/`DESCRIBE` statement routing under `cluster.pluggable.dataformat` setting ([#5528](https://github.com/opensearch-project/sql/pull/5528)) +* Handle opaque `NullPointerException` for unresolvable alias-type field path with a clear error ([#5536](https://github.com/opensearch-project/sql/pull/5536)) +* Fix invalid field or index error misclassified as internal 500 failures ([#5532](https://github.com/opensearch-project/sql/pull/5532)) +* Fix `GROUP BY` expression resolution in `SELECT`/`HAVING`/`ORDER BY` ([#5548](https://github.com/opensearch-project/sql/pull/5548)) +* Fix SQL window functions with `ORDER BY`/`LIMIT` on unified query path ([#5592](https://github.com/opensearch-project/sql/pull/5592)) +* Fix dedup field name mapping to handle alias collision when rename and eval resolve to the same source field ([#5593](https://github.com/opensearch-project/sql/pull/5593)) +* Allow partial pushdown for semi-scripted predicates so pushable filters are not blocked by unsupported ones ([#5565](https://github.com/opensearch-project/sql/pull/5565)) +* Gracefully handle malformed documents in result scanning instead of crashing ([#5618](https://github.com/opensearch-project/sql/pull/5618)) +* Honor PPL `fetch_size` on the analytics-engine route ([#5567](https://github.com/opensearch-project/sql/pull/5567)) +* Strip analytics-engine-unsupported fields from test data and exclude affected ITs ([#5541](https://github.com/opensearch-project/sql/pull/5541)) +* Repair two pre-existing IT failures on main (error type assertion and explain flake) ([#5545](https://github.com/opensearch-project/sql/pull/5545)) +* Revert PPL `rest` command ([#5635](https://github.com/opensearch-project/sql/pull/5635)) + +### Infrastructure + +* Bring `CalciteBinCommandIT` and `CalciteMultisearchCommandIT` to parity on the analytics-engine route ([#5551](https://github.com/opensearch-project/sql/pull/5551)) +* Bring `CalcitePPLEnhancedCoalesceIT` to parity on the analytics-engine route ([#5552](https://github.com/opensearch-project/sql/pull/5552)) +* Bring `CalcitePPLJoinIT` to parity on the analytics-engine route ([#5554](https://github.com/opensearch-project/sql/pull/5554)) +* Stabilize `CalcitePPLConditionBuiltinFunctionIT` on the analytics-engine route ([#5556](https://github.com/opensearch-project/sql/pull/5556)) +* Stabilize `CalciteStreamstatsCommandIT` on the analytics-engine route ([#5582](https://github.com/opensearch-project/sql/pull/5582)) +* Stabilize PPL ITs on the analytics-engine route (array/map-path/datatype/basic) ([#5562](https://github.com/opensearch-project/sql/pull/5562)) +* Stabilize PPL ITs on the analytics-engine route (case/string/full-text/like/appendpipe) ([#5561](https://github.com/opensearch-project/sql/pull/5561)) +* Stabilize PPL ITs on the analytics-engine route (percentile/float/datetime/json/dedup/union/rename/chart) ([#5564](https://github.com/opensearch-project/sql/pull/5564)) +* Stabilize PPL ITs on the analytics-engine route (sort/streamstats/IP-UDT/metadata/strip-verifier) ([#5566](https://github.com/opensearch-project/sql/pull/5566)) +* Stabilize subquery PPL ITs on the analytics-engine route ([#5555](https://github.com/opensearch-project/sql/pull/5555)) +* Recover concrete schema type for ANY-typed columns on the analytics route (fixes eval max/min) ([#5557](https://github.com/opensearch-project/sql/pull/5557)) +* Fix SQL IT test queries, assertions, and data for engine-agnostic compatibility ([#5584](https://github.com/opensearch-project/sql/pull/5584)) +* Gate analytics-engine incompatible IT tests with capability matrix annotations ([#5585](https://github.com/opensearch-project/sql/pull/5585)) +* Decouple IT from execution backend with capability-based gating ([#5560](https://github.com/opensearch-project/sql/pull/5560)) +* Fix doctest job-scheduler dependency resolution for 3.8.0 ([#5540](https://github.com/opensearch-project/sql/pull/5540)) +* Bump Apache Calcite 1.41.0 → 1.42.0 (CVE-2026-46718) ([#5619](https://github.com/opensearch-project/sql/pull/5619)) +* Bump `get-ci-image-tag.yml` ref to SHA-pinned opensearch-build commit to unblock CI ([#5583](https://github.com/opensearch-project/sql/pull/5583)) +* Case test patches for missed optimizations ([#5531](https://github.com/opensearch-project/sql/pull/5531)) +* Use engine-zone today in `DateTimeFunctionIT` now()-based assertions ([#5553](https://github.com/opensearch-project/sql/pull/5553)) +* Update datetime tests to stay within analytics-engine epoch bounds ([#5534](https://github.com/opensearch-project/sql/pull/5534)) + +### Maintenance + +* Fix flaky TPC-H Q15 floating-point assertion ([#5629](https://github.com/opensearch-project/sql/pull/5629)) +* Fix lychee link checker ([#5451](https://github.com/opensearch-project/sql/pull/5451)) From 9368cb59bb4f8fdbf3637c74b8ae03ad626b4f2c Mon Sep 17 00:00:00 2001 From: Ajimelec <144399074+AjimelecGonzalez@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:43:03 -0700 Subject: [PATCH 09/78] [BUG] PPL `LIKE` function: no way to escape the escape character (`\`) (#5653) Signed-off-by: Ajimelec Gonzalez --- .../opensearch/storage/script/StringUtils.java | 10 ++++++++-- .../storage/script/StringUtilsTest.java | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java index 05e0907d934..dd88cf7a6c6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java @@ -43,8 +43,14 @@ private static String convert(String text, boolean escapeStarQuestion) { for (char currentChar : text.toCharArray()) { switch (currentChar) { case DEFAULT_ESCAPE: - escaped = true; - convertedString.append(currentChar); + if (escaped) { + convertedString.deleteCharAt(convertedString.length() - 1); + convertedString.append(currentChar); + escaped = false; + } else { + escaped = true; + convertedString.append(currentChar); + } break; case '%': if (escaped) { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java index 24ee9b12907..50fb6a5ea8f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java @@ -47,4 +47,22 @@ public void test_escape_sql_wildcards_safe() { assertEquals("foo\\*bar", StringUtils.convertSqlWildcardToLuceneSafe("foo*bar")); assertEquals("foo\\?bar", StringUtils.convertSqlWildcardToLuceneSafe("foo?bar")); } + + @Test + public void test_escaping_backslash_itself() { + assertEquals("\\", StringUtils.convertSqlWildcardToLucene("\\\\")); + assertEquals("\\", StringUtils.convertSqlWildcardToLuceneSafe("\\\\")); + assertEquals("*\\*", StringUtils.convertSqlWildcardToLucene("%\\\\%")); + assertEquals("*\\*", StringUtils.convertSqlWildcardToLuceneSafe("%\\\\%")); + assertEquals("\\*", StringUtils.convertSqlWildcardToLucene("\\\\%")); + assertEquals("\\*", StringUtils.convertSqlWildcardToLuceneSafe("\\\\%")); + assertEquals("*\\", StringUtils.convertSqlWildcardToLucene("%\\\\")); + assertEquals("*\\", StringUtils.convertSqlWildcardToLuceneSafe("%\\\\")); + assertEquals("\\\\", StringUtils.convertSqlWildcardToLucene("\\\\\\\\")); + assertEquals("\\\\", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\\\")); + assertEquals("\\\\*", StringUtils.convertSqlWildcardToLucene("\\\\\\\\%")); + assertEquals("\\\\*", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\\\%")); + assertEquals("\\%", StringUtils.convertSqlWildcardToLucene("\\\\\\%")); + assertEquals("\\%", StringUtils.convertSqlWildcardToLuceneSafe("\\\\\\%")); + } } From 47149f0250aa1bb2b32d6e301ad0939d109a3257 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:18:02 -0700 Subject: [PATCH 10/78] Surface PIT-context exhaustion with an actionable error message (#5631) * Surface PIT-context exhaustion with an actionable error message The Calcite engine opens a Point-In-Time (PIT) context to page over a query it cannot push down to OpenSearch -- for example a stats/aggregation that groups by a text field with no keyword sub-field, which forces a full doc scan. A PIT allocates one reader context per shard, so a single such query over a many-shard index can exhaust the node's search.max_open_pit_context budget on its own; concurrent load makes it more likely. Previously that failure surfaced to the user as the opaque internal message "exception while executing query: Error occurred while creating PIT for internal plugin operation" with no hint of the cause or the remedy. Detect the PIT-context-limit rejection in the execute-time cause chain and rethrow it as a PointInTimeLimitExceededException wrapped in an ErrorReport, so the reason names the search.max_open_pit_context setting and the details explain the two remedies (raise the setting, or optimize the query). Match on the "too many Point In Time contexts" marker rather than the exception class, since OpenSearchRejectedExecutionException is also raised for scroll and thread-pool rejections. The scan walks the whole cause chain because the rejection surfaces several layers deep (SQLException -> RuntimeException -> ExecutionException -> per-shard rejection). This targets the Calcite path only; the marker check guards against self-referential cause loops. Signed-off-by: Kai Huang * Address review: trim PIT remedy text, generalize resource-limit exception Drop the "optimize the query" remedy from the PIT-context-limit details, leaving the actionable "increase [search.max_open_pit_context]" instruction. Replace the one-message PointInTimeLimitExceededException with a reusable ResourceLimitExceededException in common/error, alongside ErrorReport and ErrorCode.RESOURCE_LIMIT_EXCEEDED, so the type is not one-class-per-message and is reachable across modules. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../error/ResourceLimitExceededException.java | 24 ++++++++++ .../executor/OpenSearchExecutionEngine.java | 39 ++++++++++++++++ .../OpenSearchExecutionEngineTest.java | 46 ++++++++++++++++++- 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java diff --git a/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java b/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java new file mode 100644 index 00000000000..b201fb0c8ed --- /dev/null +++ b/common/src/main/java/org/opensearch/sql/common/error/ResourceLimitExceededException.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.common.error; + +/** + * Raised when a query cannot proceed because it would exceed a node or cluster resource budget -- + * e.g. the per-node Point-In-Time (PIT) context limit ({@code search.max_open_pit_context}). Pairs + * with {@link ErrorCode#RESOURCE_LIMIT_EXCEEDED}: the code is the machine-readable classifier while + * this type gives clients a stable, semantic name to match on. The message is the customer-facing + * {@code reason}; put the explanation and remedy in the {@link ErrorReport} details. + */ +public class ResourceLimitExceededException extends RuntimeException { + + public ResourceLimitExceededException(String message) { + super(message); + } + + public ResourceLimitExceededException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index e8c7cfc7c68..2e37782b72b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -46,6 +46,9 @@ import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.TimewrapPivot; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.ResourceLimitExceededException; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; @@ -336,11 +339,47 @@ public void execute( listener.onResponse(response); } catch (SQLException e) { + if (isPitContextLimitReached(e)) { + // reason (title) comes from the wrapped cause's message; keep it short and put the + // explanation and remedy in details. + ResourceLimitExceededException pitException = + new ResourceLimitExceededException( + "Too many open Point-In-Time (PIT) contexts on this node.", e); + throw ErrorReport.wrap(pitException) + .code(ErrorCode.RESOURCE_LIMIT_EXCEEDED) + .details( + "This query opened a Point-In-Time (PIT) context on each shard and reached" + + " the limit set by [search.max_open_pit_context]. Increase that" + + " setting.") + .build(); + } throw new RuntimeException(e); } }); } + /** + * Substring of the error OpenSearch's {@code SearchService} raises when a node has no free PIT + * contexts. The engine opens a PIT (one reader context per shard) to page over a query it cannot + * push down -- e.g. a {@code stats} that groups by a text field with no {@code keyword} sub-field + * -- and a busy node exhausts its per-node budget. The raw failure is an opaque internal message, + * so it is replaced with an actionable one when this marker appears anywhere in the cause chain. + */ + private static final String PIT_CONTEXT_LIMIT_MARKER = "too many Point In Time contexts"; + + /** Package-private for testing. Walks the cause chain guarding against self-referential loops. */ + static boolean isPitContextLimitReached(Throwable t) { + for (Throwable cause = t; + cause != null && cause != cause.getCause(); + cause = cause.getCause()) { + String message = cause.getMessage(); + if (message != null && message.contains(PIT_CONTEXT_LIMIT_MARKER)) { + return true; + } + } + return false; + } + /** * Process values recursively, handling geo points, nested maps, structs and arrays. When a {@link * RelDataType} is provided, struct values (StructImpl) are converted to Maps keyed by field diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java index 01d61288173..3da901b567a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngineTest.java @@ -12,7 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -22,6 +22,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; +import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -71,7 +72,9 @@ class OpenSearchExecutionEngineTest { @BeforeEach void setUp() { - doAnswer( + // lenient: the static PIT-detection tests below exercise no mock, so this stub is unused there. + lenient() + .doAnswer( invocation -> { // Run task immediately Runnable task = invocation.getArgument(0); @@ -262,6 +265,45 @@ public void onFailure(Exception e) { assertTrue(plan.hasClosed); } + @Test + void detects_pit_context_limit_in_nested_cause() { + // The create-PIT rejection surfaces wrapped several layers deep, mirroring the real chain: + // SQLException -> RuntimeException -> ExecutionException(all shards failed) -> rejection. + Throwable rejection = + new IllegalStateException( + "Trying to create too many Point In Time contexts. Must be less than or equal to: [0]." + + " This limit can be set by changing the [search.max_open_pit_context] setting."); + Throwable chain = + new SQLException( + "exception while executing query: Error occurred while creating PIT", + new RuntimeException("all shards failed", rejection)); + + assertTrue(OpenSearchExecutionEngine.isPitContextLimitReached(chain)); + } + + @Test + void does_not_flag_unrelated_failures_as_pit_context_limit() { + Throwable chain = + new SQLException( + "exception while executing query", new RuntimeException("all shards failed")); + + assertFalse(OpenSearchExecutionEngine.isPitContextLimitReached(chain)); + } + + @Test + void pit_context_limit_check_survives_self_referential_cause() { + // A throwable whose cause is itself must not loop forever (getCause() == this). + RuntimeException selfReferential = + new RuntimeException("boom") { + @Override + public synchronized Throwable getCause() { + return this; + } + }; + + assertFalse(OpenSearchExecutionEngine.isPitContextLimitReached(selfReferential)); + } + @RequiredArgsConstructor private static class FakePhysicalPlan extends TableScanOperator implements SerializablePlan { private final Iterator it; From f09ab9b6de9c67b1979fe4a65325542aed7c9f4f Mon Sep 17 00:00:00 2001 From: Krish Gandhi Date: Fri, 24 Jul 2026 13:24:57 -0700 Subject: [PATCH 11/78] Overriding profile endpoint with analyze endpoint with operator tree and profiling (#5568) * Overriding profile endpoint with analyze endpoint with operator tree and profiling Signed-off-by: Krish Gandhi * Updating AnalyzeResponse format Signed-off-by: Krish Gandhi * Cleaner method to combine profile and analyze Signed-off-by: Krish Gandhi * Adding tests and CI test fixes Signed-off-by: Krish Gandhi * Ran ./gradlew spotlessApply for formatting Signed-off-by: Krish Gandhi * Fixing narrowing type conversion Signed-off-by: Krish Gandhi * Adding querySegments back to AnalyzeResponse body Signed-off-by: Krish Gandhi * Adding docs, integ-test, handling when Calcite is disabled, 'fallback' on complex queries Signed-off-by: Krish Gandhi * Updating integ-test correct version, previously tested recommendation, which is not a part of this PR Signed-off-by: Krish Gandhi * Deleted FQN, updated/added testing Signed-off-by: Krish Gandhi * Rebased and fixed type mismatch Signed-off-by: Krish Gandhi * Updating integ-test to ignore recommendations (next PR) Signed-off-by: Krish Gandhi * Updating unit test to ignore \r\n and \n differences Signed-off-by: Krish Gandhi * fixing spotless check Signed-off-by: Krish Gandhi --------- Signed-off-by: Krish Gandhi --- .../sql/calcite/CalcitePlanContext.java | 16 + .../sql/calcite/CalciteRelNodeVisitor.java | 36 +- .../sql/executor/AnalyzeResponse.java | 56 ++ .../opensearch/sql/executor/QueryService.java | 479 ++++++++++++++++++ .../sql/executor/execution/AnalyzePlan.java | 54 ++ .../executor/execution/QueryPlanFactory.java | 13 + docs/user/ppl/interfaces/endpoint.md | 116 ++++- .../sql/calcite/remote/CalciteAnalyzeIT.java | 350 +++++++++++++ .../rest-api-spec/test/api/ppl.analyze.yml | 129 +++++ .../rest-api-spec/test/api/ppl.profile.yml | 11 +- .../request/PPLQueryRequestFactory.java | 11 +- .../transport/TransportPPLQueryAction.java | 31 ++ .../transport/TransportPPLQueryRequest.java | 10 +- .../org/opensearch/sql/ppl/PPLService.java | 95 ++++ .../sql/ppl/domain/PPLQueryRequest.java | 21 +- .../ppl/calcite/CalcitePPLTrackingTest.java | 266 ++++++++++ 16 files changed, 1684 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 162a4895805..b715fdac86d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -119,6 +119,22 @@ public class CalcitePlanContext { /** Whether we're currently inside a lambda context. */ @Getter @Setter private boolean inLambdaContext = false; + /** + * When enabled, tracks which RelNode ids were produced by each AST command. Each entry maps an + * AST node class name to the list of RelNode ids it produced (excluding children). + */ + @Getter @Setter private boolean trackingEnabled = false; + + @Getter private final List nodeIdMappings = new ArrayList<>(); + + /** Records a mapping from an AST command to the RelNode ids it produced. */ + public void recordMapping(String astNodeType, List relNodeIds) { + nodeIdMappings.add(new NodeIdMapping(astNodeType, relNodeIds)); + } + + /** A mapping from one AST command to the RelNode ids it produced. */ + public record NodeIdMapping(String astNodeType, List relNodeIds) {} + private CalcitePlanContext(FrameworkConfig config, SysLimit sysLimit, QueryType queryType) { this.config = config; this.sysLimit = sysLimit; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 9995895bfa1..b7b52e16d82 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -232,15 +232,47 @@ public CalciteRelNodeVisitor(DataSourceService dataSourceService) { } public RelNode analyze(UnresolvedPlan unresolved, CalcitePlanContext context) { + if (context.isTrackingEnabled()) { + int idBefore = context.relBuilder.size() > 0 ? context.relBuilder.peek().getId() : -1; + RelNode result = unresolved.accept(this, context); + int idAfter = context.relBuilder.peek().getId(); + List producedIds = new ArrayList<>(); + for (int id = idBefore + 1; id <= idAfter; id++) { + producedIds.add(id); + } + context.recordMapping(unresolved.getClass().getSimpleName(), producedIds); + return result; + } return unresolved.accept(this, context); } @Override public RelNode visitChildren(Node node, CalcitePlanContext context) { + if (context.isTrackingEnabled() && node instanceof UnresolvedPlan) { + // Track each child's total contribution (the subtree it produces) + RelNode result = null; + for (Node child : node.getChild()) { + int idBefore = context.relBuilder.size() > 0 ? context.relBuilder.peek().getId() : -1; + RelNode childResult = child.accept(this, context); + result = childResult; + // After child.accept returns, the child's visit* method has fully completed, + // so all RelNodes produced by that child (including ITS children) are on the stack. + int idAfter = context.relBuilder.peek().getId(); + if (child instanceof UnresolvedPlan) { + List producedIds = new ArrayList<>(); + for (int id = idBefore + 1; id <= idAfter; id++) { + producedIds.add(id); + } + context.recordMapping(child.getClass().getSimpleName(), producedIds); + } + } + if (node instanceof UnresolvedPlan plan) { + mapPathMaterializer.materializePaths(plan, context); + } + return result; + } RelNode result = super.visitChildren(node, context); if (node instanceof UnresolvedPlan plan) { - // Materialize MAP dotted paths as flat columns after children are analyzed - // (so MAP/struct types are known) but before the command's own visit logic runs. mapPathMaterializer.materializePaths(plan, context); } return result; diff --git a/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java b/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java new file mode 100644 index 00000000000..d8e0b12a8e7 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/AnalyzeResponse.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import java.util.List; +import lombok.Builder; +import lombok.Data; +import org.opensearch.sql.monitor.profile.QueryProfile; + +@Data +@Builder +public class AnalyzeResponse { + + private final String query; + private final List querySegments; + // private final String ast; + private final List logicalPlan; + private final List physicalPlan; + private final QueryProfile profile; + private final List operator_tree; + private final List recommendations; + private final List schema; + private final Object[][] datarows; + private final long total; + private final long size; + + @Data + @Builder + public static class SchemaColumn { + private final String name; + private final String type; + } + + @Data + @Builder + public static class QuerySegment { + private final String nodeType; + private final String source; + } + + @Data + @Builder + public static class OperatorNode { + private final String source; + private final List node_type; + private final List description; + private final String estimated_cost; + private final Long estimated_rows; + private final String actual_time_ms; + private final Long actual_rows; + private final Boolean is_pushed_down; + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index b97a679cbd3..123fe7a10fe 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -5,25 +5,37 @@ package org.opensearch.sql.executor; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitDef; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.runtime.Hook; import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; @@ -42,6 +54,7 @@ import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.CalciteClassLoaderHelper; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.common.error.StageErrorHandler; @@ -54,6 +67,7 @@ import org.opensearch.sql.monitor.profile.MetricName; import org.opensearch.sql.monitor.profile.ProfileContext; import org.opensearch.sql.monitor.profile.ProfileMetric; +import org.opensearch.sql.monitor.profile.QueryProfile; import org.opensearch.sql.monitor.profile.QueryProfiling; import org.opensearch.sql.planner.PlanContext; import org.opensearch.sql.planner.Planner; @@ -273,6 +287,471 @@ public void explainWithCalcite( settings); } + public void analyzeWithCalcite( + String query, + List querySegments, + UnresolvedPlan plan, + QueryType queryType, + ResponseListener listener) { + if (!shouldUseCalcite(queryType)) { + listener.onFailure( + new UnsupportedOperationException( + "Analyze requires the Calcite engine to be enabled" + + " (plugins.calcite.enabled=true) and a PPL query type")); + return; + } + // Phase 1: Execute via the exact same path as executeWithCalcite + executionEngine.execute + // to get identical profile timings. Use a latch to synchronize the async callback. + // Force profiling on so executeWithCalcite activates QueryProfiling. + QueryContext.setProfile(true); + AtomicReference queryResponseRef = new AtomicReference<>(); + AtomicReference profileRef = new AtomicReference<>(); + AtomicReference errorRef = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + + executeWithCalcite( + plan, + queryType, + null, + new ResponseListener<>() { + @Override + public void onResponse(ExecutionEngine.QueryResponse response) { + ProfileMetric formatMetric = + QueryProfiling.current().getOrCreateMetric(MetricName.FORMAT); + long formatStart = System.nanoTime(); + int resultSize = response.getResults().size(); + for (var exprValue : response.getResults()) { + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + formatMetric.set(System.nanoTime() - formatStart); + profileRef.set(QueryProfiling.current().finish()); + queryResponseRef.set(response); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + errorRef.set(e); + latch.countDown(); + } + }); + + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + listener.onFailure(new RuntimeException("Interrupted while waiting for query execution", e)); + return; + } + + if (errorRef.get() != null) { + listener.onFailure(errorRef.get()); + return; + } + + ExecutionEngine.QueryResponse queryResponse = queryResponseRef.get(); + QueryProfile profile = profileRef.get(); + + // If the profile plan tree has branching (any node with >1 child), our linear + // operator tree logic won't work. Return a response that 'fallsback' on `profile` + // by only including fields mirroring the `profile` endpoint. + if (profile != null && profile.getPlan() != null && !isLinearPlanTree(profile)) { + List schema = new ArrayList<>(); + if (queryResponse.getSchema() != null) { + for (ExecutionEngine.Schema.Column col : queryResponse.getSchema().getColumns()) { + schema.add( + AnalyzeResponse.SchemaColumn.builder() + .name(col.getName()) + .type(col.getExprType().typeName()) + .build()); + } + } + Object[][] datarows = new Object[queryResponse.getResults().size()][]; + int rowIdx = 0; + for (var exprValue : queryResponse.getResults()) { + datarows[rowIdx++] = + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + listener.onResponse( + AnalyzeResponse.builder() + // .query(query) + .profile(profile) + .schema(schema) + .datarows(datarows) + .total(datarows.length) + .size(datarows.length) + .build()); + return; + } + + // Phase 2: Re-run with tracking to capture logical/physical plans and node mappings. + // This run benefits from warm caches but we don't report its timings. + CalcitePlanContext.run( + () -> { + try { + QueryProfiling.noop(); + CalciteClassLoaderHelper.withCalciteClassLoader( + () -> { + CalcitePlanContext context = + CalcitePlanContext.create( + buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + context.setTrackingEnabled(true); + RelNode relNode = analyze(plan, context); + RelNode calcitePlan = convertToCalcitePlan(relNode, context); + + AtomicReference physicalPlanRef = new AtomicReference<>(); + AtomicReference physicalRelRef = new AtomicReference<>(); + try (Hook.Closeable closeable = + Hook.PLAN_BEFORE_IMPLEMENTATION.addThread( + obj -> { + RelRoot relRoot = (RelRoot) obj; + physicalRelRef.set(relRoot.rel); + physicalPlanRef.set( + RelOptUtil.toString(relRoot.rel, SqlExplainLevel.ALL_ATTRIBUTES)); + })) { + try (java.sql.PreparedStatement ignored = + OpenSearchRelRunners.run(context, calcitePlan)) { + } catch (java.sql.SQLException e) { + throw new RuntimeException(e); + } + } + + String logicalPlanStr = + RelOptUtil.toString(calcitePlan, SqlExplainLevel.ALL_ATTRIBUTES); + List logicalPlanNodes = + java.util.Arrays.stream(logicalPlanStr.split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + List physicalPlanNodes = + java.util.Arrays.stream(physicalPlanRef.get().split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + + // Build operator tree using phase 2's tracking data + phase 1's profile. + List operatorTree = + buildOperatorTree( + querySegments, + logicalPlanNodes, + context.getNodeIdMappings(), + calcitePlan, + physicalRelRef.get(), + profile); + + // Convert QueryResponse results to analyze format. + List schema = new ArrayList<>(); + if (queryResponse.getSchema() != null) { + for (ExecutionEngine.Schema.Column col : + queryResponse.getSchema().getColumns()) { + schema.add( + AnalyzeResponse.SchemaColumn.builder() + .name(col.getName()) + .type(col.getExprType().typeName()) + .build()); + } + } + + Object[][] datarows = new Object[queryResponse.getResults().size()][]; + int rowIdx = 0; + for (var exprValue : queryResponse.getResults()) { + datarows[rowIdx++] = + exprValue.tupleValue().entrySet().stream() + .map(e -> e.getValue().value()) + .toArray(Object[]::new); + } + + AnalyzeResponse response = + AnalyzeResponse.builder() + .query(query) + .querySegments(querySegments) + .logicalPlan(logicalPlanNodes) + .physicalPlan(physicalPlanNodes) + .operator_tree(operatorTree) + .recommendations(List.of()) + .profile(profile) + .schema(schema) + .datarows(datarows) + .total(datarows.length) + .size(datarows.length) + .build(); + listener.onResponse(response); + }, + QueryService.class); + } catch (Throwable t) { + if (t instanceof Exception) { + listener.onFailure((Exception) t); + } else { + listener.onFailure(new RuntimeException(t)); + } + } + }, + settings); + } + + private List buildOperatorTree( + List querySegments, + List logicalPlanNodes, + List nodeIdMappings, + RelNode logicalPlan, + RelNode physicalPlan, + QueryProfile profile) { + // Build a map from RelNode id to its logical plan description string. + Map idToDescription = new HashMap<>(); + for (String node : logicalPlanNodes) { + int idIdx = node.lastIndexOf("id = "); + if (idIdx >= 0) { + String idStr = node.substring(idIdx + 5).trim(); + try { + int id = Integer.parseInt(idStr); + idToDescription.put(id, node); + } catch (NumberFormatException ignored) { + } + } + } + + // Compute exclusive ids per mapping by subtracting the previous mapping's ids. + // Mappings are recorded bottom-up: [Relation:[0], Filter:[0,1], Project:[0,1,2]] + // Exclusive: Relation=[0], Filter=[1], Project=[2] + List> exclusiveIds = new ArrayList<>(); + Set previousIds = new HashSet<>(); + for (CalcitePlanContext.NodeIdMapping mapping : nodeIdMappings) { + Set current = new HashSet<>(mapping.relNodeIds()); + Set exclusive = new HashSet<>(current); + exclusive.removeAll(previousIds); + exclusiveIds.add(exclusive); + previousIds = current; + } + + // Determine how many segments from the bottom were pushed into the physical scan. + // The physical plan's leaf node (the scan) absorbs logical nodes from the bottom up. + // Physical depth tells us how many separate physical operators exist; everything else + // was pushed down. We count segments bottom-up until we've covered all pushed logical nodes. + int physicalDepth = getLinearDepth(physicalPlan); + int logicalDepth = getLinearDepth(logicalPlan); + int pushedNodeCount = logicalDepth - physicalDepth; + + // log.info( + // "buildOperatorTree: logicalDepth={}, physicalDepth={}, pushedNodeCount={}," + // + " segments={}, exclusiveIds={}", + // logicalDepth, + // physicalDepth, + // pushedNodeCount, + // querySegments.size(), + // exclusiveIds); + + // Walk segments bottom-up (they're already in bottom-up order) and greedily assign + // them to the pushed group until we've accounted for all pushed logical nodes. + // The LogicalSystemLimit added by convertToCalcitePlan counts toward the logical depth + // but has no segment, so we only count nodes that appear in exclusiveIds. + long pushedLogicalNodes = 0; + int pushedSegments = 0; + for (int idx = 0; idx < querySegments.size() && pushedLogicalNodes < pushedNodeCount; idx++) { + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + long planNodeCount = ids.stream().filter(idToDescription::containsKey).count(); + pushedLogicalNodes += planNodeCount; + pushedSegments++; + } + + // log.info( + // "buildOperatorTree: pushedSegments={}, pushedLogicalNodes={}", + // pushedSegments, + // pushedLogicalNodes); + + // Compute estimated row counts from the logical plan using RelMetadataQuery. + // Walk the logical plan bottom-up to get rowcount per node by id. + org.apache.calcite.rel.metadata.RelMetadataQuery mq = + logicalPlan.getCluster().getMetadataQuery(); + Map idToRowCount = new HashMap<>(); + collectRowCounts(logicalPlan, mq, idToRowCount); + + // Compute exclusive time and rows per physical node from the profile plan tree. + // The plan tree is top-down; we flatten it bottom-up to match operator tree order. + List physicalTimings = new ArrayList<>(); + if (profile != null && profile.getPlan() != null) { + List planNodes = new ArrayList<>(); + QueryProfile.PlanNode current = (QueryProfile.PlanNode) profile.getPlan(); + while (current != null) { + planNodes.add(current); + current = + (current.getChildren() != null && !current.getChildren().isEmpty()) + ? current.getChildren().get(0) + : null; + } + // planNodes is top-down; reverse to bottom-up + java.util.Collections.reverse(planNodes); + for (int p = 0; p < planNodes.size(); p++) { + double inclusive = planNodes.get(p).getTimeMillis(); + double childInclusive = (p > 0) ? planNodes.get(p - 1).getTimeMillis() : 0; + double exclusive = Math.max(0, inclusive - childInclusive); + long rows = planNodes.get(p).getRows(); + physicalTimings.add(new double[] {exclusive, rows}); + } + } + + List operators = new ArrayList<>(); + int physicalIdx = 0; + + // Build the pushed-down merged entry (first pushedSegments segments) + if (pushedSegments > 1) { + List mergedSegments = querySegments.subList(0, pushedSegments); + List descriptions = new ArrayList<>(); + for (int idx = 0; idx < pushedSegments; idx++) { + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + ids.stream() + .sorted() + .map(idToDescription::get) + .filter(Objects::nonNull) + .forEach(descriptions::add); + } + String combinedSource = + mergedSegments.stream() + .map(AnalyzeResponse.QuerySegment::getSource) + .reduce((a, b) -> a + " | " + b) + .orElse(""); + List nodeTypes = + mergedSegments.stream().map(AnalyzeResponse.QuerySegment::getNodeType).toList(); + // Collect all plan node ids in the pushed group for estimated_rows + Set allPushedPlanIds = new HashSet<>(); + for (int i = 0; i < pushedSegments; i++) { + Set ids = i < exclusiveIds.size() ? exclusiveIds.get(i) : Set.of(); + ids.stream().filter(idToDescription::containsKey).forEach(allPushedPlanIds::add); + } + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(combinedSource) + .node_type(nodeTypes) + .description(descriptions.isEmpty() ? null : descriptions) + .is_pushed_down(true) + .estimated_rows(getEstimatedRows(allPushedPlanIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } else if (pushedSegments == 1) { + AnalyzeResponse.QuerySegment seg = querySegments.get(0); + Set ids = !exclusiveIds.isEmpty() ? exclusiveIds.get(0) : Set.of(); + Set planIds = + ids.stream() + .filter(idToDescription::containsKey) + .collect(java.util.stream.Collectors.toSet()); + List descriptions = + ids.stream().sorted().map(idToDescription::get).filter(Objects::nonNull).toList(); + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(seg.getSource()) + .node_type(List.of(seg.getNodeType())) + .description(descriptions.isEmpty() ? null : descriptions) + .estimated_rows(getEstimatedRows(planIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } + + // Remaining segments map to non-scan physical nodes (physicalDepth - 1 of them). + // Each physical node corresponds to one logical plan node. Group segments so that each + // group covers exactly one logical plan node; segments with 0 plan nodes merge into the + // next group that has one. + int idx = pushedSegments; + while (idx < querySegments.size()) { + List group = new ArrayList<>(); + List descriptions = new ArrayList<>(); + Set groupPlanIds = new HashSet<>(); + long logicalNodesInGroup = 0; + while (idx < querySegments.size() && logicalNodesInGroup < 1) { + group.add(querySegments.get(idx)); + Set ids = idx < exclusiveIds.size() ? exclusiveIds.get(idx) : Set.of(); + ids.stream() + .sorted() + .map(idToDescription::get) + .filter(Objects::nonNull) + .forEach(descriptions::add); + ids.stream().filter(idToDescription::containsKey).forEach(groupPlanIds::add); + logicalNodesInGroup += ids.stream().filter(idToDescription::containsKey).count(); + idx++; + } + String combinedSource = + group.stream() + .map(AnalyzeResponse.QuerySegment::getSource) + .reduce((a, b) -> a + " | " + b) + .orElse(""); + List nodeTypes = + group.stream().map(AnalyzeResponse.QuerySegment::getNodeType).toList(); + double[] timing = + physicalIdx < physicalTimings.size() ? physicalTimings.get(physicalIdx) : null; + physicalIdx++; + operators.add( + AnalyzeResponse.OperatorNode.builder() + .source(combinedSource) + .node_type(nodeTypes) + .description(descriptions.isEmpty() ? null : descriptions) + .estimated_rows(getEstimatedRows(groupPlanIds, idToRowCount)) + .actual_time_ms(timing != null ? String.format("%.2f ms", timing[0]) : null) + .actual_rows(timing != null ? (long) timing[1] : null) + .build()); + } + + return operators; + } + + private static boolean isLinearPlanTree(QueryProfile profile) { + QueryProfile.PlanNode current = (QueryProfile.PlanNode) profile.getPlan(); + while (current != null) { + if (current.getChildren() != null && current.getChildren().size() > 1) { + return false; + } + current = + (current.getChildren() != null && !current.getChildren().isEmpty()) + ? current.getChildren().get(0) + : null; + } + return true; + } + + private static int getLinearDepth(RelNode node) { + int depth = 0; + RelNode current = node; + while (current != null) { + depth++; + List inputs = current.getInputs(); + current = inputs.isEmpty() ? null : inputs.get(0); + } + return depth; + } + + private void collectRowCounts( + RelNode node, + org.apache.calcite.rel.metadata.RelMetadataQuery mq, + Map idToRowCount) { + try { + Double rowCount = mq.getRowCount(node); + if (rowCount != null) { + idToRowCount.put(node.getId(), rowCount); + } + } catch (Exception ignored) { + } + for (RelNode input : node.getInputs()) { + collectRowCounts(input, mq, idToRowCount); + } + } + + private Long getEstimatedRows(Set ids, Map idToRowCount) { + return ids.stream() + .filter(idToRowCount::containsKey) + .max(Integer::compareTo) + .map(id -> Math.round(idToRowCount.get(id))) + .orElse(null); + } + public void executeWithLegacy( UnresolvedPlan plan, QueryType queryType, diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java new file mode 100644 index 00000000000..a43bc32792e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AnalyzePlan.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor.execution; + +import java.util.List; +import org.opensearch.sql.ast.statement.ExplainMode; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.executor.AnalyzeResponse; +import org.opensearch.sql.executor.AnalyzeResponse.QuerySegment; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryId; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.QueryType; + +/** Plan that produces an AnalyzeResponse (AST + logical plan). */ +public class AnalyzePlan extends AbstractPlan { + + private final String query; + private final List querySegments; + private final UnresolvedPlan plan; + private final QueryService queryService; + private final ResponseListener listener; + + public AnalyzePlan( + QueryId queryId, + QueryType queryType, + String query, + List querySegments, + UnresolvedPlan plan, + QueryService queryService, + ResponseListener listener) { + super(queryId, queryType); + this.query = query; + this.querySegments = querySegments; + this.plan = plan; + this.queryService = queryService; + this.listener = listener; + } + + @Override + public void execute() { + queryService.analyzeWithCalcite(query, querySegments, plan, getQueryType(), listener); + } + + @Override + public void explain( + ResponseListener listener, ExplainMode mode) { + throw new UnsupportedOperationException("Explain is not supported for analyze plan"); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 93c73a2315b..0e44dd02e7e 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -7,6 +7,7 @@ import static java.util.Objects.requireNonNull; +import java.util.List; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -19,6 +20,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.exception.UnsupportedCursorRequestException; +import org.opensearch.sql.executor.AnalyzeResponse; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryId; import org.opensearch.sql.executor.QueryService; @@ -147,4 +149,15 @@ public AbstractPlan visitExplain( node.getFormat(), context.getRight()); } + + /** Create an AnalyzePlan that produces AST node and logical plan RelNode. */ + public AbstractPlan createAnalyzePlan( + String query, + List querySegments, + UnresolvedPlan plan, + QueryType queryType, + ResponseListener listener) { + return new AnalyzePlan( + QueryId.queryId(), queryType, query, querySegments, plan, queryService, listener); + } } diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 9360d5198ca..3ef3ccbf37d 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -152,8 +152,122 @@ calcite: physical: | CalciteEnumerableIndexScan(table=[[OpenSearch, state_country]], PushDownContext=[[PROJECT->[name, country, state, month, year, age], FILTER->>($5, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["name","country","state","month","year","age"]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) ``` +## Analyze (Experimental) -## Profile (Experimental) +You can enable analysis on the PPL endpoint to capture query execution details including per-stage timings, logical and physical plans, operator tree with pushdown visibility, and optimization recommendations. Analysis is returned only for regular query execution (not explain) and only when using the default `format=jdbc`. + +### Example + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ + -X POST localhost:9200/_plugins/_ppl \ + -d '{ + "query": "source=accounts | where age < 30 | eval full_name = firstname + \" \" + lastname | fields full_name, email, age", + "analyze": true + }' +``` + +Expected output (trimmed): + +```json +{ + "query": "source=accounts | where age < 30 | eval full_name = firstname + \" \" + lastname | fields full_name, email, age", + "querySegments": [ + {"nodeType": "SearchFrom", "source": "source=accounts"}, + {"nodeType": "WhereCommand", "source": "where age < 30"}, + ], + "logicalPlan": [ + "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]): rowcount = 5000.0, cumulative cost = {114000.0 rows, 145000.0 cpu, 0.0 io}, id = 4229", + "LogicalProject(full_name=[||(||($0, ' '), $4)], email=[$3], age=[$2]): rowcount = 5000.0, cumulative cost = {109000.0 rows, 25000.0 cpu, 0.0 io}, id = 4228", + "LogicalFilter(condition=[<($2, 30)]): rowcount = 5000.0, cumulative cost = {104000.0 rows, 10000.0 cpu, 0.0 io}, id = 4226", + "CalciteLogicalIndexScan(table=[[OpenSearch, accounts]]): rowcount = 10000.0, cumulative cost = {99000.0 rows, 0.0 cpu, 0.0 io}, id = 4225" + ], + "physicalPlan": [ + "EnumerableCalc(expr#0..3=[{inputs}], expr#4=[' '], expr#5=[||($t0, $t4)], expr#6=[||($t5, $t3)], full_name=[$t6], email=[$t2], age=[$t1]): rowcount = 5000.0, cumulative cost = {22996.4 rows, 50000.0 cpu, 0.0 io}, id = 4319", + "CalciteEnumerableIndexScan(table=[[OpenSearch, accounts]], PushDownContext=[[PROJECT->[firstname, age, email, lastname], FILTER-><($1, 30), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{ + ], + "profile": { + "summary": { + "total_time_ms": 37.13 + }, + "phases": { + "analyze": { "time_ms": 7.06 }, + "optimize": { "time_ms": 25.29 }, + "execute": { "time_ms": 4.73 }, + "format": { "time_ms": 0.03 } + }, + "plan": { + "node": "EnumerableCalc", + "time_ms": 3.44, + "rows": 3, + "children": [ + { "node": "CalciteEnumerableIndexScan", "time_ms": 3.31, "rows": 3 } + ] + } + }, + "operator_tree": [ + { + "source": "source=accounts | where age < 30", + "node_type": [ + "SearchFrom", + "WhereCommand" + ], + "description": [ + "CalciteLogicalIndexScan(table=[[OpenSearch, accounts]]): rowcount = 10000.0, cumulative cost = {99000.0 rows, 0.0 cpu, 0.0 io}, id = 4225", + "LogicalFilter(condition=[<($2, 30)]): rowcount = 5000.0, cumulative cost = {104000.0 rows, 10000.0 cpu, 0.0 io}, id = 4226" + ], + "estimated_rows": 5000, + "actual_time_ms": "3.31 ms", + "actual_rows": 3, + "is_pushed_down": true + }, + ], + "recommendations": [] +} +``` + +### Response fields + +| Field | Type | Description | +|-------|------|-------------| +| `query` | String | The original PPL query. | +| `querySegments` | Array | Breakdown of the query into AST segments with `nodeType` and `source`. | +| `logicalPlan` | Array | Calcite logical plan nodes (top-down). | +| `physicalPlan` | Array | Calcite physical plan nodes after optimization. | +| `operator_tree` | Array | Per-stage execution details linking query segments to plan operators. | +| `recommendations` | Array | Optimization suggestions generated from the execution profile. | +| `profile` | Object | Per-phase timing breakdown (same format as the profile endpoint). | +| `schema` | Array | Column names and types of the query result. | +| `datarows` | Array | Query result rows. | +| `total` | Integer | Total number of result rows. | +| `size` | Integer | Number of result rows returned. | + +### Operator tree fields + +| Field | Type | Description | +|-------|------|-------------| +| `source` | String | The PPL query fragment(s) that produced this operator. | +| `node_type` | Array | AST node type(s) (e.g. `Relation`, `Filter`, `Project`). | +| `description` | Array | Logical plan node descriptions. | +| `estimated_rows` | Long | Estimated row count from Calcite metadata. | +| `actual_time_ms` | String | Exclusive wall-clock time for this operator. | +| `actual_rows` | Long | Actual rows produced by this operator. | +| `is_pushed_down` | Boolean | Whether the operator was pushed down to the storage engine. | + + + +### Notes +- Analyze output is only returned when the query finishes successfully. +- Analyze requires the Calcite engine to be enabled (`plugins.calcite.enabled=true`). +- Operator tree nodes with `is_pushed_down: true` were executed within the OpenSearch storage engine (single network round-trip). Remaining operators ran in-memory on the coordinating node. +- This endpoint is meant to replace/override the existing `profile` endpoint. As a result, any POST requests with either `"analyze": true` or `"profile": true` (or both) will be routed to this endpoint. + - The `profile` section uses the same format as the previous `profile` endpoint. This means current consumers of `profile` should not face any breaking changes. +- The logic for `analyze` doesn't hold for queries that produce non-linear physical plan trees (for example, JOINs). In this scenario, `analyze` will return an output identical to the previous `profile` endpoint. + + +## Profile (Experimental) (Deprecated) + +**This endpoint is outdated, see the `analyze` section above.** You can enable profiling on the PPL endpoint to capture per-stage timings in milliseconds. Profiling is returned only for regular query execution (not explain) and only when using the default `format=jdbc`. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java new file mode 100644 index 00000000000..df2e1b88e9a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyzeIT.java @@ -0,0 +1,350 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteAnalyzeIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.ACCOUNT); + } + + // === Helper === + + private JSONObject executeAnalyze(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"analyze\": true}", query)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/json"); + request.setOptions(opts); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } + + private JSONObject executeProfile(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/json"); + request.setOptions(opts); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } + + // === A. Query result correctness === + + @Test + public void analyzeResultsMatchNormalExecution() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"; + JSONObject normal = executeQuery(query); + JSONObject analyzed = executeAnalyze(query); + + // Schema should match + assertEquals(normal.getJSONArray("schema").length(), analyzed.getJSONArray("schema").length()); + // Row counts should match + assertEquals(normal.getInt("total"), analyzed.getInt("total")); + assertEquals(normal.getInt("size"), analyzed.getInt("size")); + // Datarows should have same length + assertEquals( + normal.getJSONArray("datarows").length(), analyzed.getJSONArray("datarows").length()); + } + + @Test + public void analyzeResultsMatchWithAggregation() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | stats count() by gender"; + JSONObject normal = executeQuery(query); + JSONObject analyzed = executeAnalyze(query); + + assertEquals(normal.getInt("total"), analyzed.getInt("total")); + assertEquals( + normal.getJSONArray("datarows").length(), analyzed.getJSONArray("datarows").length()); + } + + // === B. Operator tree — all pushed down === + + @Test + public void operatorTreeAllPushedDown() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // Single physical node → all segments merged into one entry + assertEquals(1, tree.length()); + JSONObject node = tree.getJSONObject(0); + assertTrue(node.getBoolean("is_pushed_down")); + + JSONArray nodeTypes = node.getJSONArray("node_type"); + assertTrue(nodeTypes.toString().contains("SearchFrom")); + assertTrue(nodeTypes.toString().contains("WhereCommand")); + assertTrue(nodeTypes.toString().contains("FieldsCommand")); + } + + @Test + public void operatorTreeAllPushedDownWithStats() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | stats count() by gender"); + JSONArray tree = result.getJSONArray("operator_tree"); + + assertEquals(1, tree.length()); + JSONObject node = tree.getJSONObject(0); + assertTrue(node.getBoolean("is_pushed_down")); + + JSONArray nodeTypes = node.getJSONArray("node_type"); + assertTrue(nodeTypes.toString().contains("SearchFrom")); + assertTrue(nodeTypes.toString().contains("WhereCommand")); + assertTrue(nodeTypes.toString().contains("StatsCommand")); + } + + // === C. Operator tree — partial pushdown === + + @Test + public void operatorTreePartialPushdown() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + + TEST_INDEX_ACCOUNT + + " | where age > 30 | eval name = firstname | fields name, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // At least 2 entries: pushed-down group + non-pushed group + assertTrue(tree.length() >= 2); + // First entry should be pushed down + assertTrue(tree.getJSONObject(0).optBoolean("is_pushed_down", false)); + // Last entry should NOT be pushed down + assertFalse(tree.getJSONObject(tree.length() - 1).optBoolean("is_pushed_down", false)); + } + + // === D. Profile structure === + + @Test + public void analyzeIncludesProfileWithAllPhases() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + assertTrue(result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue(profile.has("summary")); + assertTrue(profile.has("phases")); + assertTrue(profile.has("plan")); + + JSONObject phases = profile.getJSONObject("phases"); + assertTrue(phases.has("analyze")); + assertTrue(phases.has("optimize")); + assertTrue(phases.has("execute")); + assertTrue(phases.has("format")); + + // All phase times should be non-negative + assertTrue(phases.getJSONObject("analyze").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("optimize").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("execute").getDouble("time_ms") >= 0); + assertTrue(phases.getJSONObject("format").getDouble("time_ms") >= 0); + } + + @Test + public void analyzeProfilePlanHasNodeInfo() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONObject plan = result.getJSONObject("profile").getJSONObject("plan"); + + assertTrue(plan.has("node")); + assertTrue(plan.has("time_ms")); + assertTrue(plan.has("rows")); + assertTrue(plan.getDouble("time_ms") >= 0); + assertTrue(plan.getLong("rows") >= 0); + } + + // === E. Timing correctness === + + @Test + public void operatorTreeHasTimings() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + + for (int i = 0; i < tree.length(); i++) { + JSONObject node = tree.getJSONObject(i); + assertTrue("node " + i + " has actual_time_ms", node.has("actual_time_ms")); + assertTrue("node " + i + " has actual_rows", node.has("actual_rows")); + assertTrue(node.getLong("actual_rows") >= 0); + } + } + + @Test + public void operatorTreeTimingsSumApproximatesPlanRoot() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + + TEST_INDEX_ACCOUNT + + " | where age > 30 | eval x = age * 2 | fields x, firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + JSONObject profile = result.getJSONObject("profile"); + + double totalOperatorTime = 0; + for (int i = 0; i < tree.length(); i++) { + String timeStr = tree.getJSONObject(i).getString("actual_time_ms"); + totalOperatorTime += Double.parseDouble(timeStr.replace(" ms", "")); + } + double planRootTime = profile.getJSONObject("plan").getDouble("time_ms"); + + // Exclusive times should sum to roughly the root inclusive time. + // Allow generous tolerance for off-spine subtree time not captured. + assertTrue( + "operator times (" + totalOperatorTime + ") roughly match plan root (" + planRootTime + ")", + totalOperatorTime <= planRootTime * 2.0 && totalOperatorTime >= planRootTime * 0.1); + } + + // === F. Estimated rows === + + @Test + public void operatorTreeHasEstimatedRows() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + JSONArray tree = result.getJSONArray("operator_tree"); + + for (int i = 0; i < tree.length(); i++) { + JSONObject node = tree.getJSONObject(i); + assertTrue("node " + i + " has estimated_rows", node.has("estimated_rows")); + assertTrue(node.getLong("estimated_rows") > 0); + } + } + + // === G. Logical and physical plan presence === + + @Test + public void analyzeIncludesLogicalAndPhysicalPlan() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname"); + + assertTrue(result.has("logicalPlan")); + assertTrue(result.has("physicalPlan")); + + JSONArray logicalPlan = result.getJSONArray("logicalPlan"); + JSONArray physicalPlan = result.getJSONArray("physicalPlan"); + + assertTrue(logicalPlan.length() > 0); + assertTrue(physicalPlan.length() > 0); + + // Logical plan should contain known node types + String logicalStr = logicalPlan.toString(); + assertTrue(logicalStr.contains("LogicalFilter") || logicalStr.contains("LogicalProject")); + } + + // === H. Degenerate cases === + + @Test + public void analyzeEmptyResults() throws IOException { + JSONObject result = + executeAnalyze("source=" + TEST_INDEX_ACCOUNT + " | where age > 99999 | fields firstname"); + + assertEquals(0, result.getInt("total")); + assertEquals(0, result.getJSONArray("datarows").length()); + // Profile and operator tree should still be present + assertTrue(result.has("profile")); + assertTrue(result.has("operator_tree")); + assertTrue(result.getJSONArray("operator_tree").length() > 0); + } + + @Test + public void analyzeNonexistentIndexReturnsError() { + assertThrows( + ResponseException.class, () -> executeAnalyze("source=nonexistent_index_xyz | fields a")); + } + + @Test + public void analyzeSyntaxErrorReturnsError() { + assertThrows(ResponseException.class, () -> executeAnalyze("this is not valid ppl")); + } + + // === I. Schema correctness === + + @Test + public void analyzeSchemaMatchesQueryFields() throws IOException { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray schema = result.getJSONArray("schema"); + + assertEquals(2, schema.length()); + assertEquals("firstname", schema.getJSONObject(0).getString("name")); + assertEquals("age", schema.getJSONObject(1).getString("name")); + } + + // === J. Profile timing similarity to standalone profile endpoint === + + @Test + public void analyzeTimingsInSameOrderOfMagnitudeAsProfile() throws IOException { + String query = "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | stats count() by gender"; + + JSONObject profileResult = executeProfile(query); + JSONObject analyzeResult = executeAnalyze(query); + + double profileTotal = + profileResult.getJSONObject("profile").getJSONObject("summary").getDouble("total_time_ms"); + double analyzeTotal = + analyzeResult.getJSONObject("profile").getJSONObject("summary").getDouble("total_time_ms"); + + // Should be within 5x of each other (generous for CI environments) + assertTrue( + "analyze total (" + analyzeTotal + ") within 5x of profile total (" + profileTotal + ")", + analyzeTotal < profileTotal * 5 && analyzeTotal > profileTotal / 5); + } + + // === K. Pushdown disabled === + + @Test + public void analyzeWithPushdownDisabledShowsNoPushdown() throws IOException { + // Disable pushdown + updateClusterSettings( + new ClusterSetting( + "transient", + org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ENABLED.getKeyValue(), + "false")); + try { + JSONObject result = + executeAnalyze( + "source=" + TEST_INDEX_ACCOUNT + " | where age > 30 | fields firstname, age"); + JSONArray tree = result.getJSONArray("operator_tree"); + + // With pushdown disabled, nothing should be marked as pushed down + // (or it should have multiple nodes since operations stay separate) + if (tree.length() == 1) { + // If still 1 node, it shouldn't be marked pushed_down + assertFalse(tree.getJSONObject(0).optBoolean("is_pushed_down", false)); + } else { + // Multiple nodes means operations weren't merged + assertTrue(tree.length() > 1); + } + } finally { + // Re-enable pushdown + updateClusterSettings( + new ClusterSetting( + "transient", + org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ENABLED.getKeyValue(), + "true")); + } + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml new file mode 100644 index 00000000000..a1087dbe6da --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.analyze.yml @@ -0,0 +1,129 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + - do: + indices.create: + index: ppl_analyze + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + message: + type: keyword + age: + type: integer + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "ppl_analyze", "_id": 1}}' + - '{"message": "hello", "age": 25}' + - '{"index": {"_index": "ppl_analyze", "_id": 2}}' + - '{"message": "world", "age": 35}' + +--- +teardown: + - do: + indices.delete: + index: ppl_analyze + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Analyze returns full response for ppl query": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - match: {query: 'source=ppl_analyze | fields message'} + - is_true: querySegments + - is_true: logicalPlan + - is_true: physicalPlan + - is_true: operator_tree + - is_true: recommendations + - is_true: profile + - gt: {profile.summary.total_time_ms: 0.0} + - gt: {profile.phases.execute.time_ms: 0.0} + - is_true: schema + - is_true: datarows + - match: {total: 2} + - match: {size: 2} + +--- +"Analyze returns operator tree with pushdown info": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | where age > 30 | fields message' + analyze: true + - is_true: operator_tree + - match: {total: 1} + - match: {size: 1} + +--- +"Analyze returns recommendations field": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | where age > 30 | fields message' + analyze: true + - is_true: recommendations + +--- +"Analyze ignored for explain api": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - match: {query: null} + - match: {operator_tree: null} + +--- +"Analyze with non-jdbc format still returns response": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: 'source=ppl_analyze | fields message' + analyze: true + - is_true: query diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml index 882757fd8b7..d8bbb3d5724 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/api/ppl.profile.yml @@ -53,9 +53,18 @@ teardown: - gt: {profile.phases.analyze.time_ms: 0.0} - gt: {profile.phases.optimize.time_ms: 0.0} - gt: {profile.phases.execute.time_ms: 0.0} - - gt: {profile.phases.format.time_ms: 0.0} + - gte: {profile.phases.format.time_ms: 0.0} - gt: {profile.plan.time_ms: 0.0} - match: {profile.plan.rows: 2} + - match: {query: 'source=ppl_profile | fields message'} + - is_true: logicalPlan + - is_true: physicalPlan + - is_true: operator_tree + - is_true: recommendations + - is_true: schema + - is_true: datarows + - match: {total: 2} + - match: {size: 2} --- "Profile ignored for explain api": diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java index bb87bf7fa91..800fcbe1692 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -30,6 +30,7 @@ public class PPLQueryRequestFactory { private static final String DEFAULT_EXPLAIN_MODE = "standard"; private static final String QUERY_PARAMS_PRETTY = "pretty"; private static final String QUERY_PARAMS_PROFILE = "profile"; + private static final String QUERY_PARAMS_ANALYZE = "analyze"; private static final String QUERY_PARAMS_FETCH_SIZE = "fetch_size"; /** @@ -82,9 +83,12 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques try { jsonContent = new JSONObject(content); boolean profileRequested = jsonContent.optBoolean(QUERY_PARAMS_PROFILE, false); + boolean analyzeRequested = jsonContent.optBoolean(QUERY_PARAMS_ANALYZE, false); String queryString = jsonContent.optString(PPL_FIELD_NAME, ""); - boolean enableProfile = - profileRequested && isProfileSupported(restRequest.path(), format, queryString); + // if both profile and analyze are requested, profile overrides analyze + boolean profileSupported = isProfileSupported(restRequest.path(), format, queryString); + boolean enableProfile = profileRequested && profileSupported; + boolean enableAnalyze = analyzeRequested && !profileRequested && profileSupported; // Support fetch_size as a URL parameter if not already in the JSON body if (!jsonContent.has(QUERY_PARAMS_FETCH_SIZE) && restRequest.params().containsKey(QUERY_PARAMS_FETCH_SIZE)) { @@ -104,7 +108,8 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques restRequest.path(), format.getFormatName(), explainMode, - enableProfile); + enableProfile, + enableAnalyze); // set sanitize option if csv format if (format.equals(Format.CSV)) { pplRequest.sanitize(getSanitizeOption(restRequest.params())); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 678ed58f37f..b54c2c49ab9 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -33,6 +33,7 @@ import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.executor.AnalyzeResponse; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.legacy.metrics.MetricName; @@ -211,6 +212,13 @@ protected void doExecute( if (transformedRequest.isExplainRequest()) { pplService.explain( transformedRequest, createExplainResponseListener(transformedRequest, clearingListener)); + /** + * Removing `|| transformedRequest.profile()` from line 200 will separate the `profile` and + * `analyze` endpoints. See PR #5568. + */ + } else if (transformedRequest.analyze() || transformedRequest.profile()) { + pplService.analyze( + transformedRequest, createAnalyzeResponseListener(transformedRequest, clearingListener)); } else { pplService.execute( transformedRequest, @@ -219,6 +227,29 @@ protected void doExecute( } } + private ResponseListener createAnalyzeResponseListener( + PPLQueryRequest request, ActionListener listener) { + return new ResponseListener() { + @Override + public void onResponse(AnalyzeResponse response) { + JsonResponseFormatter formatter = + new JsonResponseFormatter<>(PRETTY) { + @Override + protected Object buildJsonObject(AnalyzeResponse response) { + return response; + } + }; + listener.onResponse( + new TransportPPLQueryResponse(formatter.format(response), formatter.contentType())); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }; + } + /** * TODO: need to extract an interface for both SQL and PPL action handler and move these common * methods to the interface. This is not easy to do now because SQL action handler is still in diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index 4ba1a53d872..68a96a4f924 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -53,6 +53,11 @@ public class TransportPPLQueryRequest extends ActionRequest { @Accessors(fluent = true) private boolean profile = false; + @Setter + @Getter + @Accessors(fluent = true) + private boolean analyze = false; + @Setter @Getter @Accessors(fluent = true) @@ -67,6 +72,7 @@ public TransportPPLQueryRequest(PPLQueryRequest pplQueryRequest) { sanitize = pplQueryRequest.sanitize(); style = pplQueryRequest.style(); profile = pplQueryRequest.profile(); + analyze = pplQueryRequest.analyze(); explainMode = pplQueryRequest.mode().getModeName(); queryId = pplQueryRequest.queryId(); } @@ -83,6 +89,7 @@ public TransportPPLQueryRequest(StreamInput in) throws IOException { sanitize = in.readBoolean(); style = in.readEnum(JsonResponseFormatter.Style.class); profile = in.readBoolean(); + analyze = in.readBoolean(); queryId = in.readOptionalString(); } @@ -116,6 +123,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(sanitize); out.writeEnum(style); out.writeBoolean(profile); + out.writeBoolean(analyze); out.writeOptionalString(queryId); } @@ -172,7 +180,7 @@ public String getDescription() { /** Convert to PPLQueryRequest. */ public PPLQueryRequest toPPLQueryRequest() { PPLQueryRequest pplQueryRequest = - new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile); + new PPLQueryRequest(pplQuery, jsonContent, path, format, explainMode, profile, analyze); pplQueryRequest.sanitize(sanitize); pplQueryRequest.style(style); pplQueryRequest.queryId(queryId); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index 6ad9032432c..7f117e7bb0b 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -8,18 +8,26 @@ import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import static org.opensearch.sql.executor.execution.QueryPlanFactory.NO_CONSUMER_RESPONSE_LISTENER; +import java.util.ArrayList; +import java.util.List; import lombok.extern.log4j.Log4j2; +import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; +import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; +import org.opensearch.sql.executor.AnalyzeResponse; +import org.opensearch.sql.executor.AnalyzeResponse.QuerySegment; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.executor.execution.AbstractPlan; import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.ppl.parser.AstBuilder; import org.opensearch.sql.ppl.parser.AstStatementBuilder; @@ -85,6 +93,93 @@ public void explain(PPLQueryRequest request, ResponseListener l } } + /** + * Analyze the query: produces the AST node and logical plan RelNode. + * + * @param request {@link PPLQueryRequest} + * @param listener {@link ResponseListener} for analyze response + */ + public void analyze(PPLQueryRequest request, ResponseListener listener) { + try { + String queryText = request.getRequest(); + ParseTree cst = parser.parse(queryText); + Statement statement = + cst.accept( + new AstStatementBuilder( + new AstBuilder(queryText, settings), + AstStatementBuilder.StatementBuilderContext.builder() + .isExplain(false) + .fetchSize(request.getFetchSize()) + .highlightConfig(request.getHighlightConfig()) + .format( + request.getFormat() != null && !request.getFormat().isEmpty() + ? org.opensearch.sql.protocol.response.format.Format.ofExplain( + request.getFormat()) + .orElse(null) + : null) + .build())); + + log.info( + "[{}] Incoming request {}", + QueryContext.getRequestId(), + anonymizer.anonymizeStatement(statement)); + + List querySegments = extractQuerySegments(cst, queryText); + UnresolvedPlan unresolvedPlan = ((Query) statement).getPlan(); + queryManager.submit( + queryExecutionFactory.createAnalyzePlan( + queryText, querySegments, unresolvedPlan, PPL_QUERY, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + private List extractQuerySegments(ParseTree cst, String queryText) { + List segments = new ArrayList<>(); + OpenSearchPPLParser.QueryStatementContext queryStmt = findQueryStatement(cst); + if (queryStmt == null) { + return segments; + } + + // First segment: the search/source command (pplCommands) + OpenSearchPPLParser.PplCommandsContext pplCommands = queryStmt.pplCommands(); + if (pplCommands != null) { + segments.add(buildSegment(pplCommands, queryText)); + } + + // Remaining segments: each piped command + for (OpenSearchPPLParser.CommandsContext cmd : queryStmt.commands()) { + segments.add(buildSegment(cmd, queryText)); + } + return segments; + } + + private OpenSearchPPLParser.QueryStatementContext findQueryStatement(ParseTree tree) { + if (tree instanceof OpenSearchPPLParser.QueryStatementContext ctx) { + return ctx; + } + for (int i = 0; i < tree.getChildCount(); i++) { + OpenSearchPPLParser.QueryStatementContext result = findQueryStatement(tree.getChild(i)); + if (result != null) { + return result; + } + } + return null; + } + + private QuerySegment buildSegment(ParserRuleContext ctx, String queryText) { + int start = ctx.getStart().getStartIndex(); + int stop = ctx.getStop().getStopIndex(); + String source = queryText.substring(start, stop + 1); + // For wrapper rules like CommandsContext, drill into the specific child command + ParserRuleContext target = ctx; + if (ctx.getChildCount() == 1 && ctx.getChild(0) instanceof ParserRuleContext child) { + target = child; + } + String nodeType = target.getClass().getSimpleName().replace("Context", ""); + return QuerySegment.builder().nodeType(nodeType).source(source).build(); + } + private AbstractPlan plan( PPLQueryRequest request, ResponseListener queryListener, diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 06c7fe1c38e..0ef1de27fc3 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -52,6 +52,11 @@ public class PPLQueryRequest { @Accessors(fluent = true) private boolean profile = false; + @Setter + @Getter + @Accessors(fluent = true) + private boolean analyze = false; + @Setter @Getter @Accessors(fluent = true) @@ -62,10 +67,9 @@ public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path) { } public PPLQueryRequest(String pplQuery, JSONObject jsonContent, String path, String format) { - this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false); + this(pplQuery, jsonContent, path, format, ExplainMode.STANDARD.getModeName(), false, false); } - /** Constructor of PPLQueryRequest. */ public PPLQueryRequest( String pplQuery, JSONObject jsonContent, @@ -73,12 +77,25 @@ public PPLQueryRequest( String format, String explainMode, boolean profile) { + this(pplQuery, jsonContent, path, format, explainMode, profile, false); + } + + /** Constructor of PPLQueryRequest. */ + public PPLQueryRequest( + String pplQuery, + JSONObject jsonContent, + String path, + String format, + String explainMode, + boolean profile, + boolean analyze) { this.pplQuery = pplQuery; this.jsonContent = jsonContent; this.path = Optional.ofNullable(path).orElse(DEFAULT_PPL_PATH); this.format = format; this.explainMode = explainMode; this.profile = profile; + this.analyze = analyze; } public String getRequest() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java new file mode 100644 index 00000000000..8469348f399 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTrackingTest.java @@ -0,0 +1,266 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.opensearch.sql.executor.QueryType.PPL; + +import java.util.List; +import org.apache.calcite.plan.Contexts; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.apache.calcite.tools.Frameworks; +import org.apache.calcite.tools.RelBuilder; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.statement.Query; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.CalcitePlanContext.NodeIdMapping; +import org.opensearch.sql.calcite.CalciteRelNodeVisitor; +import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.ppl.parser.AstStatementBuilder; + +public class CalcitePPLTrackingTest { + + private final Frameworks.ConfigBuilder config; + private final CalciteRelNodeVisitor planTransformer; + private final Settings settings; + private final DataSourceService dataSourceService; + private final PPLSyntaxParser pplParser = new PPLSyntaxParser(); + + public CalcitePPLTrackingTest() { + this.dataSourceService = mock(DataSourceService.class); + this.planTransformer = new CalciteRelNodeVisitor(dataSourceService); + this.settings = mock(Settings.class); + this.config = + Frameworks.newConfigBuilder() + .defaultSchema( + CalciteAssert.addSchema( + Frameworks.createRootSchema(true), + CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL)) + .programs(); + } + + @Before + public void init() { + doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_ENGINE_ENABLED); + doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES); + doReturn(true).when(settings).getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT); + doReturn(-1).when(settings).getSettingValue(Settings.Key.PPL_SUBSEARCH_MAXOUT); + doReturn(false).when(dataSourceService).dataSourceExists(any()); + } + + private CalcitePlanContext createContext() { + config.context(Contexts.of(RelBuilder.Config.DEFAULT)); + return CalcitePlanContext.create(config.build(), SysLimit.fromSettings(settings), PPL); + } + + private Node plan(String query) { + final AstStatementBuilder builder = + new AstStatementBuilder( + new AstBuilder(query, settings), + AstStatementBuilder.StatementBuilderContext.builder().build()); + return builder.visit(pplParser.parse(query)); + } + + private RelNode getRelNode(String ppl, CalcitePlanContext context) { + Query query = (Query) plan(ppl); + planTransformer.analyze(query.getPlan(), context); + return context.relBuilder.build(); + } + + @Test + public void testTrackingProducesSameLogicalPlanAsNonTracking() { + String ppl = "source=EMP | eval a = 1 | fields EMPNO, a"; + + CalcitePlanContext withoutTracking = createContext(); + RelNode expected = getRelNode(ppl, withoutTracking); + + CalcitePlanContext withTracking = createContext(); + withTracking.setTrackingEnabled(true); + RelNode actual = getRelNode(ppl, withTracking); + + assertEquals(expected.explain().replace("\r\n", "\n"), actual.explain().replace("\r\n", "\n")); + } + + @Test + public void testTrackingDisabledProducesNoMappings() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(false); + getRelNode(ppl, context); + + assertTrue(context.getNodeIdMappings().isEmpty()); + } + + @Test + public void testTrackingEvalRecordsMappings() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + assertFalse(mappings.isEmpty()); + + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Eval mapping", astTypes.contains("Eval")); + } + + @Test + public void testTrackingFilterRecordsMappings() { + String ppl = "source=EMP | where SAL > 1000"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Filter mapping", astTypes.contains("Filter")); + } + + @Test + public void testTrackingSortRecordsMappings() { + String ppl = "source=EMP | sort SAL"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Sort mapping", astTypes.contains("Sort")); + } + + @Test + public void testTrackingMultipleCommandsRecordsMappings() { + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1 | sort SAL"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Filter mapping", astTypes.contains("Filter")); + assertTrue("Should contain Eval mapping", astTypes.contains("Eval")); + assertTrue("Should contain Sort mapping", astTypes.contains("Sort")); + } + + @Test + public void testTrackingMappingsHaveNonEmptyRelNodeIds() { + String ppl = "source=EMP | eval a = 1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + for (NodeIdMapping mapping : context.getNodeIdMappings()) { + assertFalse( + "Mapping for " + mapping.astNodeType() + " should have non-empty RelNode IDs", + mapping.relNodeIds().isEmpty()); + } + } + + @Test + public void testTrackingProjectRecordsMappings() { + String ppl = "source=EMP | fields EMPNO, ENAME"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Project mapping", astTypes.contains("Project")); + } + + @Test + public void testTrackingAggregationRecordsMappings() { + String ppl = "source=EMP | stats count() by DEPTNO"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + assertTrue("Should contain Relation mapping", astTypes.contains("Relation")); + assertTrue("Should contain Aggregation mapping", astTypes.contains("Aggregation")); + } + + @Test + public void testTrackingMultipleCommandsProducesSameLogicalPlan() { + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1 | sort SAL | head 10"; + + CalcitePlanContext withoutTracking = createContext(); + RelNode expected = getRelNode(ppl, withoutTracking); + + CalcitePlanContext withTracking = createContext(); + withTracking.setTrackingEnabled(true); + RelNode actual = getRelNode(ppl, withTracking); + + assertEquals(expected.explain().replace("\r\n", "\n"), actual.explain().replace("\r\n", "\n")); + } + + @Test + public void testVisitChildrenCapturesSubtreeContribution() { + // visitChildren records a child's SUBTREE contribution (all RelNodes produced + // by that child and its descendants). For a multi-command pipeline, the mapping + // for Filter should include RelNodes from its own subtree (Relation + Filter itself). + String ppl = "source=EMP | where SAL > 1000 | eval bonus = SAL * 0.1"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + + // Relation is a leaf — should produce exactly 1 RelNode (the scan) + NodeIdMapping relationMapping = + mappings.stream().filter(m -> m.astNodeType().equals("Relation")).findFirst().orElseThrow(); + assertFalse( + "Relation (leaf) should produce at least one RelNode", + relationMapping.relNodeIds().isEmpty()); + + // Filter's subtree includes Relation beneath it, so visitChildren should + // capture more RelNode IDs for Filter than for Relation alone. + NodeIdMapping filterMapping = + mappings.stream().filter(m -> m.astNodeType().equals("Filter")).findFirst().orElseThrow(); + assertTrue( + "Filter subtree should produce more RelNodes than Relation alone", + filterMapping.relNodeIds().size() > relationMapping.relNodeIds().size()); + } + + @Test + public void testVisitChildrenRecordsAllChildrenSeparately() { + // visitChildren iterates over node.getChild() and records each one. + // For a pipeline with multiple commands, each command gets its own mapping entry. + String ppl = "source=EMP | where SAL > 1000 | sort SAL | head 5"; + CalcitePlanContext context = createContext(); + context.setTrackingEnabled(true); + getRelNode(ppl, context); + + List mappings = context.getNodeIdMappings(); + List astTypes = mappings.stream().map(NodeIdMapping::astNodeType).toList(); + + // Each command in the pipeline should have a separate mapping entry + assertTrue("Should record Relation", astTypes.contains("Relation")); + assertTrue("Should record Filter", astTypes.contains("Filter")); + assertTrue("Should record Sort", astTypes.contains("Sort")); + assertTrue("Should record Head", astTypes.contains("Head")); + } +} From a54acb44094fa6d36982ad3882cd54375325e668 Mon Sep 17 00:00:00 2001 From: opensearch-ci <83309141+opensearch-ci-bot@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:32:56 -0400 Subject: [PATCH 12/78] Remove reverted rest command from 3.8 release notes (#5654) (#5655) The PPL rest command was added (#5599) and fully reverted (#5635) within the same release cycle, so neither should appear in the notes. (cherry picked from commit 7c24851408040d2e3928a1bdfa0e27645a35b742) Signed-off-by: Eric Wei Signed-off-by: opensearch-ci-bot Co-authored-by: Eric Wei --- release-notes/opensearch-sql.release-notes-3.8.0.0.md | 1 - 1 file changed, 1 deletion(-) diff --git a/release-notes/opensearch-sql.release-notes-3.8.0.0.md b/release-notes/opensearch-sql.release-notes-3.8.0.0.md index 61db95af498..8dbff4fd2c0 100644 --- a/release-notes/opensearch-sql.release-notes-3.8.0.0.md +++ b/release-notes/opensearch-sql.release-notes-3.8.0.0.md @@ -49,7 +49,6 @@ Compatible with OpenSearch and OpenSearch Dashboards version 3.8.0 * Honor PPL `fetch_size` on the analytics-engine route ([#5567](https://github.com/opensearch-project/sql/pull/5567)) * Strip analytics-engine-unsupported fields from test data and exclude affected ITs ([#5541](https://github.com/opensearch-project/sql/pull/5541)) * Repair two pre-existing IT failures on main (error type assertion and explain flake) ([#5545](https://github.com/opensearch-project/sql/pull/5545)) -* Revert PPL `rest` command ([#5635](https://github.com/opensearch-project/sql/pull/5635)) ### Infrastructure From e905aec24e12f2f2ba9330ee984be50930983354 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 28 Jul 2026 09:18:57 -0700 Subject: [PATCH 13/78] feat: complex query thread pool (#5628) * feat: slow query thread pool Signed-off-by: Simeon Widdis * handle slow query detection when optimization runs in execution step Signed-off-by: Simeon Widdis * fixes: assorted context propagation issues Signed-off-by: Simeon Widdis * fix remaining integ tests Signed-off-by: Simeon Widdis * add some more thread & security tests Signed-off-by: Simeon Widdis * code self-review, round 1 Signed-off-by: Simeon Widdis * use 2x background threads to account for 2x pools Signed-off-by: Simeon Widdis * rename slow -> complex, add pool indication header Signed-off-by: Simeon Widdis * add a failure log for slow pool requests Signed-off-by: Simeon Widdis * fix profile, add thread pool as part of profile object Signed-off-by: Simeon Widdis * remove leftover build.gradle changes from another branch Signed-off-by: Simeon Widdis * add cancelation polling so ppl cancelation is faster to apply Signed-off-by: Simeon Widdis * Add thread pool profile details to doc Signed-off-by: Simeon Widdis * Move analyze call measurement Signed-off-by: Simeon Widdis * add complex pool IT Signed-off-by: Simeon Widdis * Move calcite context thread copies to dedicated method Signed-off-by: Simeon Widdis * add attach_pid to gitignore Signed-off-by: Simeon Widdis * register timeout handler to complex pool on these requests Signed-off-by: Simeon Widdis * fix units Signed-off-by: Simeon Widdis * remove redundant optimize call from execution engine during execution Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .gitignore | 3 +- .../sql/common/setting/Settings.java | 5 +- .../sql/calcite/CalcitePlanContext.java | 50 +++ .../sql/calcite/utils/CalciteToolsHelper.java | 2 - .../executor/DirectExecutionDispatcher.java | 26 ++ .../sql/executor/ExecutionDispatcher.java | 45 ++ .../opensearch/sql/executor/QueryService.java | 51 ++- .../profile/DefaultProfileContext.java | 4 +- .../sql/monitor/profile/QueryProfile.java | 19 +- .../sql/monitor/profile/QueryProfiling.java | 10 + docs/user/ppl/interfaces/endpoint.md | 4 +- .../sql/calcite/CalciteComplexPoolIT.java | 98 +++++ .../CalcitePPLRelNodeIntegTestCase.java | 4 +- .../sql/security/FGACIndexScanningIT.java | 84 ++++ .../client/OpenSearchNodeClient.java | 16 +- .../executor/OpenSearchExecutionEngine.java | 5 +- .../executor/OpenSearchQueryManager.java | 1 + .../opensearch/executor/ScriptDetector.java | 116 +++++ .../ThreadPoolExecutionDispatcher.java | 174 ++++++++ .../setting/OpenSearchSettings.java | 14 + .../storage/scan/BackgroundSearchScanner.java | 6 +- .../executor/ScriptDetectorTest.java | 212 ++++++++++ .../ThreadPoolExecutionDispatcherTest.java | 399 ++++++++++++++++++ .../org/opensearch/sql/plugin/SQLPlugin.java | 27 +- .../plugin/config/OpenSearchPluginModule.java | 12 +- .../plugin/rest/RestUnifiedQueryAction.java | 37 +- .../transport/TransportPPLQueryAction.java | 8 +- .../rest/RestUnifiedQueryActionTest.java | 3 +- 28 files changed, 1392 insertions(+), 43 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java create mode 100644 core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java diff --git a/.gitignore b/.gitignore index bf9002f999d..dcaf14d2ad7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ src/site-server/node_modules build/ gen/ *.tokens +.attach_pid* # various IDE files .vscode @@ -59,4 +60,4 @@ http-client.env.json !.claude/harness/ .claude/settings.local.json .clinerules -memory-bank \ No newline at end of file +memory-bank diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index bf3e65d8741..d32fd249e02 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -79,7 +79,10 @@ public enum Key { ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL( "plugins.query.executionengine.async_query.external_scheduler.interval"), STREAMING_JOB_HOUSEKEEPER_INTERVAL( - "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval"); + "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval"), + + /** Thread Pool Settings. */ + SQL_COMPLEX_WORKER_POOL_ENABLED("plugins.sql.complex_worker_pool.enabled"); @Getter private final String keyValue; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index b715fdac86d..c7f3bc373ac 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -58,6 +58,11 @@ public class CalcitePlanContext { /** Timewrap series mode: "relative", "short", or "exact". */ public static final ThreadLocal timewrapSeries = new ThreadLocal<>(); + /** + * Thread-local tracking which pool executed this query ("sql-worker" or "sql-complex-worker"). + */ + public static final ThreadLocal executionPool = new ThreadLocal<>(); + /** Thread-local switch that tells whether the current query prefers legacy behavior. */ private static final ThreadLocal legacyPreferredFlag = ThreadLocal.withInitial(() -> true); @@ -245,6 +250,51 @@ public static void clearTimewrapSignals() { stripNullColumns.set(false); timewrapUnitName.set(null); timewrapSeries.set(null); + executionPool.set(null); + } + + /** + * Snapshot of all thread-local state in CalcitePlanContext. Used when dispatching queries to the + * complex worker pool — capture state on the caller thread, restore on the worker thread. + */ + public static class ThreadLocalSnapshot { + final boolean skipEncoding; + final boolean stripNullColumns; + final String timewrapUnitName; + final String timewrapSeries; + final String executionPool; + + private ThreadLocalSnapshot( + boolean skipEncoding, + boolean stripNullColumns, + String timewrapUnitName, + String timewrapSeries, + String executionPool) { + this.skipEncoding = skipEncoding; + this.stripNullColumns = stripNullColumns; + this.timewrapUnitName = timewrapUnitName; + this.timewrapSeries = timewrapSeries; + this.executionPool = executionPool; + } + } + + /** Capture current thread-local state for cross-thread propagation. */ + public static ThreadLocalSnapshot snapshotThreadLocals() { + return new ThreadLocalSnapshot( + skipEncoding.get(), + stripNullColumns.get(), + timewrapUnitName.get(), + timewrapSeries.get(), + executionPool.get()); + } + + /** Restore thread-local state from a snapshot. */ + public static void restoreThreadLocals(ThreadLocalSnapshot snapshot) { + skipEncoding.set(snapshot.skipEncoding); + stripNullColumns.set(snapshot.stripNullColumns); + timewrapUnitName.set(snapshot.timewrapUnitName); + timewrapSeries.set(snapshot.timewrapSeries); + executionPool.set(snapshot.executionPool); } public void pushForeachBindings( diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index 54b9d4ffbaf..682a3ea17c1 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -516,8 +516,6 @@ private static void enrichErrorsForSpecialCases(ErrorReport.Builder report, SQLE public static PreparedStatement run(CalcitePlanContext context, RelNode rel) { ProfileMetric optimizeTime = QueryProfiling.current().getOrCreateMetric(OPTIMIZE); long startTime = System.nanoTime(); - // Optimize the plan by Calcite's HepPlanner before using VolcanoPlanner in prepareStatement. - rel = CalciteToolsHelper.optimize(rel, context); final RelShuttle shuttle = new RelHomogeneousShuttle() { @Override diff --git a/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java b/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java new file mode 100644 index 00000000000..5af110fa7e5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/DirectExecutionDispatcher.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; + +/** + * Default no-op dispatcher that executes inline on the current thread. Used when complex-pool + * routing is disabled or as a fallback. + */ +public class DirectExecutionDispatcher implements ExecutionDispatcher { + + @Override + public void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine) { + engine.execute(plan, context, listener); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java new file mode 100644 index 00000000000..ec7fa193852 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionDispatcher.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.executor; + +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; + +/** + * Dispatches query execution to an appropriate thread pool based on plan characteristics. After + * query analysis and optimization, the dispatcher inspects the plan and routes execution to either + * the fast worker pool (for queries fully pushed to OpenSearch) or the complex worker pool (for + * queries requiring scripts/table scans). + */ +public interface ExecutionDispatcher { + + /** + * Dispatch execution of the given plan via the standard ExecutionEngine. + * + * @param plan the optimized Calcite plan + * @param context the plan context + * @param listener response listener for query results + * @param engine the execution engine to invoke + */ + void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine); + + /** + * Dispatch a task to the appropriate thread pool based on plan characteristics. Use this when the + * execution path differs from the standard ExecutionEngine interface (e.g., analytics engine). + * + * @param plan the optimized Calcite plan used for routing decisions + * @param context the plan context + * @param task the execution task to run + */ + default void dispatchTask(RelNode plan, CalcitePlanContext context, Runnable task) { + task.run(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 123fe7a10fe..858ba0598e6 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -16,7 +16,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; @@ -54,6 +53,7 @@ import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit; import org.opensearch.sql.calcite.plan.rel.LogicalSystemLimit.SystemLimitType; import org.opensearch.sql.calcite.utils.CalciteClassLoaderHelper; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.common.error.QueryProcessingStage; @@ -78,7 +78,6 @@ /** The low level interface of core engine. */ @RequiredArgsConstructor -@AllArgsConstructor @Log4j2 public class QueryService { private final Analyzer analyzer; @@ -86,6 +85,37 @@ public class QueryService { private final Planner planner; private DataSourceService dataSourceService; private Settings settings; + private ExecutionDispatcher executionDispatcher = new DirectExecutionDispatcher(); + + public QueryService( + Analyzer analyzer, + ExecutionEngine executionEngine, + Planner planner, + DataSourceService dataSourceService, + Settings settings) { + this( + analyzer, + executionEngine, + planner, + dataSourceService, + settings, + new DirectExecutionDispatcher()); + } + + public QueryService( + Analyzer analyzer, + ExecutionEngine executionEngine, + Planner planner, + DataSourceService dataSourceService, + Settings settings, + ExecutionDispatcher executionDispatcher) { + this.analyzer = analyzer; + this.executionEngine = executionEngine; + this.planner = planner; + this.dataSourceService = dataSourceService; + this.settings = settings; + this.executionDispatcher = executionDispatcher; + } @Getter(lazy = true) private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService); @@ -199,9 +229,7 @@ public void executeWithCalcite( convertToCalcitePlan(relNode, context), context), "while converting the query to an executable plan"); - analyzeMetric.set(System.nanoTime() - analyzeStart); - - executeCalcitePlan(calcitePlan, context, listener); + executeCalcitePlan(calcitePlan, context, listener, analyzeMetric, analyzeStart); }, QueryService.class); } catch (Throwable t) { @@ -219,11 +247,20 @@ public void executeWithCalcite( private void executeCalcitePlan( RelNode calcitePlan, CalcitePlanContext context, - ResponseListener listener) { + ResponseListener listener, + ProfileMetric analyzeMetric, + long analyzeStart) { try { + // Optimize before dispatch so the dispatcher's ScriptDetector + // sees the post-optimization plan for accurate routing. + RelNode optimizedPlan = CalciteToolsHelper.optimize(calcitePlan, context); + analyzeMetric.set(System.nanoTime() - analyzeStart); + + // Wrap execution with EXECUTING stage tracking — dispatch via + // ExecutionDispatcher which may route to a complex worker pool StageErrorHandler.executeStageVoid( QueryProcessingStage.EXECUTING, - () -> executionEngine.execute(calcitePlan, context, listener), + () -> executionDispatcher.dispatch(optimizedPlan, context, listener, executionEngine), "while running the query"); } catch (RuntimeException e) { ArithmeticException overflow = findArithmeticOverflow(e); diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java index 63327c2d6dd..f3df41356f2 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/DefaultProfileContext.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import org.opensearch.sql.calcite.CalcitePlanContext; /** Default implementation that records profiling metrics. */ public class DefaultProfileContext implements ProfileContext { @@ -63,7 +64,8 @@ public synchronized QueryProfile finish() { double totalMillis = ProfileUtils.roundToMillis(endNanos - startNanos); Object planSnapshot = enginePlan != null ? enginePlan : (planRoot == null ? null : planRoot.snapshot()); - profile = new QueryProfile(totalMillis, snapshot, planSnapshot); + String threadPool = CalcitePlanContext.executionPool.get(); + profile = new QueryProfile(totalMillis, snapshot, planSnapshot, threadPool); return profile; } } diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java index d9d2a785868..71454951557 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfile.java @@ -24,6 +24,9 @@ public final class QueryProfile { /** Execution-engine-specific plan profile: a {@link PlanNode} tree, or a pre-rendered object. */ private final Object plan; + @SerializedName("thread_pool") + private final String threadPool; + /** * Create a new query profile snapshot. * @@ -31,7 +34,7 @@ public final class QueryProfile { * @param phases metric values keyed by {@link MetricName} */ public QueryProfile(double totalTimeMillis, Map phases) { - this(totalTimeMillis, phases, null); + this(totalTimeMillis, phases, null, null); } /** @@ -42,9 +45,23 @@ public QueryProfile(double totalTimeMillis, Map phases) { * @param plan plan tree profiling output */ public QueryProfile(double totalTimeMillis, Map phases, Object plan) { + this(totalTimeMillis, phases, plan, null); + } + + /** + * Create a new query profile snapshot. + * + * @param totalTimeMillis total elapsed milliseconds for the query (rounded to two decimals) + * @param phases metric values keyed by {@link MetricName} + * @param plan plan tree profiling output + * @param threadPool thread pool name that executed the query + */ + public QueryProfile( + double totalTimeMillis, Map phases, Object plan, String threadPool) { this.summary = new Summary(totalTimeMillis); this.phases = buildPhases(phases); this.plan = plan; + this.threadPool = threadPool; } private Map buildPhases(Map phases) { diff --git a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java index 3ef32dac748..900be175050 100644 --- a/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java +++ b/core/src/main/java/org/opensearch/sql/monitor/profile/QueryProfiling.java @@ -57,6 +57,16 @@ public static void clear() { CURRENT.remove(); } + /** + * Set the profiling context for the current thread. Used when propagating context across thread + * boundaries. + * + * @param ctx profiling context to bind + */ + public static void set(ProfileContext ctx) { + CURRENT.set(Objects.requireNonNull(ctx, "ctx")); + } + /** * Run a supplier with the provided profiling context bound to the current thread. * diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 3ef3ccbf37d..6704cc0cd48 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -303,7 +303,8 @@ Expected output (trimmed): "children": [ { "node": "CalciteEnumerableIndexScan", "time_ms": 4.12, "rows": 2 } ] - } + }, + "thread_pool": "sql-worker" } } ``` @@ -315,6 +316,7 @@ Expected output (trimmed): - Plan node names use Calcite physical operator names (for example, `EnumerableCalc` or `CalciteEnumerableIndexScan`). - Plan `time_ms` is inclusive of child operators and represents wall-clock time; overlapping work can make summed plan times exceed `summary.total_time_ms`. - Scan nodes reflect operator wall-clock time; background prefetch can make scan time smaller than total request latency. +- `thread_pool` indicates which thread pool executed the query. Possible values are `sql-worker` (default, pushdown-only queries) and `sql-complex-worker` (queries requiring in-memory evaluation such as scripted fields). ## Highlight diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java new file mode 100644 index 00000000000..b419c8043d3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteComplexPoolIT.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for queries dispatched to the complex worker pool. Verifies that queries + * containing scripts (e.g., parse command) are correctly routed and that profile responses include + * thread_pool metadata. + */ +public class CalciteComplexPoolIT extends PPLIntegTestCase { + + @Override + protected void init() throws Exception { + super.init(); + loadIndex(Index.BANK); + enableCalcite(); + enableComplexPool(); + } + + private void enableComplexPool() throws IOException { + updateClusterSettings( + new ClusterSetting(PERSISTENT, "plugins.sql.complex_worker_pool.enabled", "true")); + } + + @Test + public void testParseCommandDispatchesToComplexPool() throws IOException { + // parse creates script nodes, triggering complex pool dispatch + JSONObject result = + executeQuery( + String.format( + "source=%s | parse address '(?\\\\d+) (?.*)'" + + " | fields number, street | head 1", + TEST_INDEX_BANK)); + + verifyDataRows(result, rows("880", "Holmes Lane")); + } + + @Test + public void testComplexPoolProfileIncludesThreadPool() throws IOException { + // Query with parse to trigger complex pool + String query = + String.format( + "source=%s | parse address '(?\\\\d+)' | fields num | head 1", TEST_INDEX_BANK); + + JSONObject result = executeQueryWithProfile(query); + + assertTrue("Response has profile", result.has("profile")); + JSONObject profile = result.getJSONObject("profile"); + + assertTrue("Profile has thread_pool", profile.has("thread_pool")); + String threadPool = profile.getString("thread_pool"); + assertEquals("Thread pool is sql-complex-worker", "sql-complex-worker", threadPool); + + // Verify profile structure is intact + assertTrue("Profile has summary", profile.has("summary")); + assertTrue("Profile has phases", profile.has("phases")); + } + + @Test + public void testSimpleQueryUsesWorkerPool() throws IOException { + // Query without scripts — should use sql-worker pool + String query = String.format("source=%s | fields account_number | head 1", TEST_INDEX_BANK); + + JSONObject result = executeQueryWithProfile(query); + + assertTrue("Response has profile", result.has("profile")); + JSONObject profile = result.getJSONObject("profile"); + + assertTrue("Profile has thread_pool", profile.has("thread_pool")); + String threadPool = profile.getString("thread_pool"); + assertEquals("Thread pool is sql-worker", "sql-worker", threadPool); + } + + private JSONObject executeQueryWithProfile(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response, true)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java index 9c7d8c90ac3..744b9ec124b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLRelNodeIntegTestCase.java @@ -25,6 +25,7 @@ import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.SysLimit; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.executor.QueryType; @@ -87,7 +88,8 @@ protected RexNode createStringArray(RexBuilder rexBuilder, String... values) { protected void executeRelNodeAndVerify( CalcitePlanContext planContext, RelNode relNode, ResultVerifier verifier) throws SQLException { - try (PreparedStatement statement = OpenSearchRelRunners.run(planContext, relNode)) { + try (PreparedStatement statement = + OpenSearchRelRunners.run(planContext, CalciteToolsHelper.optimize(relNode, planContext))) { ResultSet resultSet = statement.executeQuery(); verifier.verify(resultSet); } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java index 22c93591241..97559b6032c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java @@ -656,4 +656,88 @@ public void testRowLevelSecurity(boolean useCalcite) throws IOException { expectedPublicDocs, totalDocs); } + + /** + * Verifies that document-level security is enforced when queries are dispatched to the complex + * worker pool. Queries containing window functions (eventstats) are routed to sql-complex-worker; + * this test ensures the OpenSearch ThreadContext (which carries DLS filters) is correctly + * propagated across that thread pool boundary. + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRowLevelSecurityEnforcedOnComplexPool(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + String engineLabel = useCalcite ? "V3" : "V2"; + + // eventstats creates a Window node, which ScriptDetector flags as expensive, + // routing the query to the sql-complex-worker pool. + String query = + String.format( + "search source=%s | eventstats count() as total_count by security_level" + + " | stats count() by security_level", + SECURE_LOGS); + JSONObject result = executeQueryAsUser(query, LIMITED_USER); + + var datarows = result.getJSONArray("datarows"); + var schema = result.getJSONArray("schema"); + int levelIdx = -1; + for (int i = 0; i < schema.length(); i++) { + String name = schema.getJSONObject(i).getString("name"); + if ("security_level".equals(name)) { + levelIdx = i; + } + } + assertTrue("Expected security_level in schema", levelIdx >= 0); + + for (int i = 0; i < datarows.length(); i++) { + var row = datarows.getJSONArray(i); + String securityLevel = row.getString(levelIdx); + assertFalse( + String.format( + "[%s] SECURITY VIOLATION on complex pool: limited_user saw '%s' documents. " + + "DLS ThreadContext may not be propagated to sql-complex-worker pool.", + engineLabel, securityLevel), + "confidential".equals(securityLevel) || "internal".equals(securityLevel)); + } + } + + /** + * Verifies that field-level security is enforced when queries are dispatched to the slow worker + * pool. The eventstats command creates a Window node that triggers complex pool dispatch; this + * test ensures the restricted field (ssn) remains invisible. + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testFieldLevelSecurityEnforcedOnSlowPool(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + + // eventstats creates a Window node, which ScriptDetector flags as expensive, + // routing the query to the sql-complex-worker pool. + // manager_user should still NOT see ssn. + String query = + String.format( + "search source=%s | eventstats avg(salary) as avg_salary by department" + + " | fields name, department, salary, avg_salary | head 10", + EMPLOYEE_RECORDS); + JSONObject result = executeQueryAsUser(query, MANAGER_USER); + + var resultSchema = result.getJSONArray("schema"); + boolean hasSSN = false; + boolean hasName = false; + boolean hasAvgSalary = false; + + for (int i = 0; i < resultSchema.length(); i++) { + String fieldName = resultSchema.getJSONObject(i).getString("name"); + if ("ssn".equals(fieldName)) hasSSN = true; + if ("name".equals(fieldName)) hasName = true; + if ("avg_salary".equals(fieldName)) hasAvgSalary = true; + } + + assertTrue("manager_user should see 'name' field on complex pool", hasName); + assertTrue("manager_user should see computed 'avg_salary' field on complex pool", hasAvgSalary); + assertFalse( + "SECURITY VIOLATION on complex pool: manager_user saw 'ssn' field. " + + "FLS ThreadContext may not be propagated to sql-complex-worker pool.", + hasSSN); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b491f38ef80..d9681898f73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -28,14 +28,17 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; +import org.opensearch.core.tasks.TaskId; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexSettings; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; +import org.opensearch.tasks.CancellableTask; import org.opensearch.transport.client.node.NodeClient; /** OpenSearch connection by node client. */ @@ -161,7 +164,18 @@ public Map getIndexMaxResultWindows(String... indexExpression) @Override public OpenSearchResponse search(OpenSearchRequest request) { return request.search( - req -> client.search(req).actionGet(), req -> client.searchScroll(req).actionGet()); + req -> { + applyParentTask(req); + return client.search(req).actionGet(); + }, + req -> client.searchScroll(req).actionGet()); + } + + private void applyParentTask(SearchRequest req) { + CancellableTask task = OpenSearchQueryManager.getCancellableTask(); + if (task != null) { + req.setParentTask(new TaskId(client.getLocalNodeId(), task.getId())); + } } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java index 2e37782b72b..483f2684d61 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java @@ -42,6 +42,7 @@ import org.locationtech.jts.geom.Point; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper; import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelRunners; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.calcite.utils.TimewrapPivot; @@ -262,7 +263,7 @@ public void explain( } })) { // triggers the hook - OpenSearchRelRunners.run(context, rel); + OpenSearchRelRunners.run(context, CalciteToolsHelper.optimize(rel, context)); } if (physicalError.get() != null) { @@ -309,7 +310,7 @@ public void explain( CalcitePlanContext.skipEncoding.set(true); } // triggers the hook - OpenSearchRelRunners.run(context, rel); + OpenSearchRelRunners.run(context, CalciteToolsHelper.optimize(rel, context)); } listener.onResponse( new ExplainResponse( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java index 7aaaaa6655e..c391153fca6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java @@ -32,6 +32,7 @@ public class OpenSearchQueryManager implements QueryManager { private final Settings settings; public static final String SQL_WORKER_THREAD_POOL_NAME = "sql-worker"; + public static final String SQL_COMPLEX_WORKER_THREAD_POOL_NAME = "sql-complex-worker"; public static final String SQL_BACKGROUND_THREAD_POOL_NAME = "sql_background_io"; private static final ThreadLocal cancellableTask = new ThreadLocal<>(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java new file mode 100644 index 00000000000..aafb2f307b3 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ScriptDetector.java @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import java.util.Set; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Window; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexVisitorImpl; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.AggSpec; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; + +/** + * Inspects a Calcite plan tree to determine whether execution will be expensive. Detects: (1) + * user-defined functions (REX_EXTRACT, PARSE, etc.) that become per-document scripts, (2) join + * nodes requiring in-memory merge, and (3) window functions requiring in-memory evaluation. + * + *

Works on both logical plans (before optimization, where PushDownContext is empty) and physical + * plans (after optimization, where PushDownContext tracks scripts). + */ +public final class ScriptDetector { + + private static final Set EXPENSIVE_UDFS = + Set.of("REX_EXTRACT", "REX_EXTRACT_MULTI", "PARSE", "PATTERN_PARSER"); + + private ScriptDetector() {} + + /** + * Returns true if the plan contains patterns indicating expensive execution: UDF calls that + * produce scripts, join nodes, or window functions. + */ + public static boolean hasScripts(RelNode plan) { + boolean[] found = {false}; + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, RelNode parent) { + if (found[0]) { + return; + } + // Physical plan: check PushDownContext for scripts already detected by optimizer + if (node instanceof AbstractCalciteIndexScan scan) { + found[0] = scanHasScripts(scan); + } + // Logical plan: detect join nodes (always require in-memory processing) + if (!found[0] && node instanceof Join) { + found[0] = true; + } + // Logical plan: detect window rel nodes (eventstats, dedup patterns) + if (!found[0] && node instanceof Window) { + found[0] = true; + } + // Logical plan: check projections for UDFs or window expressions + if (!found[0] && node instanceof Project project) { + found[0] = projectHasExpensiveExpressions(project); + } + if (!found[0]) { + super.visit(node, ordinal, parent); + } + } + }.go(plan); + return found[0]; + } + + private static boolean scanHasScripts(AbstractCalciteIndexScan scan) { + PushDownContext ctx = scan.getPushDownContext(); + if (ctx.isScriptPushed()) { + return true; + } + if (ctx.isSortExprPushed()) { + return true; + } + AggSpec aggSpec = ctx.getAggSpec(); + return aggSpec != null && aggSpec.getScriptCount() > 0; + } + + private static boolean projectHasExpensiveExpressions(Project project) { + for (RexNode expr : project.getProjects()) { + if (hasExpensiveRex(expr)) { + return true; + } + } + return false; + } + + private static boolean hasExpensiveRex(RexNode expr) { + boolean[] found = {false}; + expr.accept( + new RexVisitorImpl(true) { + @Override + public Void visitOver(RexOver over) { + found[0] = true; + return null; + } + + @Override + public Void visitCall(RexCall call) { + String name = call.getOperator().getName(); + if (name != null && EXPENSIVE_UDFS.contains(name)) { + found[0] = true; + return null; + } + return super.visitCall(call); + } + }); + return found[0]; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java new file mode 100644 index 00000000000..3de8bb85c2b --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcher.java @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; + +import java.util.Map; +import java.util.function.Consumer; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQueryBase; +import org.apache.calcite.runtime.Hook; +import org.apache.calcite.util.Holder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.ExecutionDispatcher; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.monitor.profile.ProfileContext; +import org.opensearch.sql.monitor.profile.QueryProfiling; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.threadpool.Scheduler.Cancellable; +import org.opensearch.threadpool.ThreadPool; + +/** + * Dispatches query execution to either the fast or complex worker thread pool based on whether the + * plan contains scripts. Plans with scripts require in-memory evaluation and are routed to the + * complex pool so they don't block fast pushdown-only queries. + */ +@RequiredArgsConstructor +public class ThreadPoolExecutionDispatcher implements ExecutionDispatcher { + + private static final Logger LOG = LogManager.getLogger(ThreadPoolExecutionDispatcher.class); + + private final ThreadPool threadPool; + private final Settings settings; + + @Override + public void dispatch( + RelNode plan, + CalcitePlanContext context, + ResponseListener listener, + ExecutionEngine engine) { + dispatchInternal(plan, context, () -> engine.execute(plan, context, listener), listener); + } + + @Override + public void dispatchTask(RelNode plan, CalcitePlanContext context, Runnable task) { + dispatchInternal(plan, context, task, null); + } + + private void dispatchInternal( + RelNode optimizedPlan, + CalcitePlanContext context, + Runnable task, + @Nullable ResponseListener failureListener) { + if (isComplexPoolEnabled() && ScriptDetector.hasScripts(optimizedPlan)) { + LOG.debug("Query plan contains scripts, dispatching to complex worker pool"); + // Capture thread-local state to propagate across thread boundary + Map ctx = ThreadContext.getImmutableContext(); + CancellableTask cancellableTask = OpenSearchQueryManager.getCancellableTask(); + ProfileContext profileContext = QueryProfiling.current(); + CalcitePlanContext.ThreadLocalSnapshot snapshot = CalcitePlanContext.snapshotThreadLocals(); + @Nullable JaninoRelMetadataProvider metadataProvider = + RelMetadataQueryBase.THREAD_PROVIDERS.get(); + long currentTime = Hook.CURRENT_TIME.get(-1L); + TimeValue timeout = settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT); + + threadPool.schedule( + () -> { + final Thread executionThread = Thread.currentThread(); + Cancellable timeoutHandle = + threadPool.schedule( + () -> { + LOG.warn( + "Query execution timed out after {}. Interrupting execution thread.", + timeout); + executionThread.interrupt(); + }, + timeout, + ThreadPool.Names.GENERIC); + Cancellable cancelPoller = scheduleCancellationPoller(cancellableTask, executionThread); + Hook.Closeable hookHandle = null; + try { + // Restore state from caller thread + ThreadContext.putAll(ctx); + OpenSearchQueryManager.setCancellableTask(cancellableTask); + QueryProfiling.set(profileContext); + CalcitePlanContext.restoreThreadLocals(snapshot); + // Override execution pool to indicate complex pool + CalcitePlanContext.executionPool.set(SQL_COMPLEX_WORKER_THREAD_POOL_NAME); + if (metadataProvider != null) { + RelMetadataQueryBase.THREAD_PROVIDERS.set(metadataProvider); + } + if (currentTime >= 0) { + hookHandle = + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(currentTime)); + } + task.run(); + } catch (Exception e) { + LOG.error("Exception during task execution on complex pool", e); + if (failureListener != null) { + failureListener.onFailure(e); + } + } finally { + timeoutHandle.cancel(); + cancelPoller.cancel(); + Thread.interrupted(); + if (hookHandle != null) { + hookHandle.close(); + } + OpenSearchQueryManager.clearCancellableTask(); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + CalcitePlanContext.clearTimewrapSignals(); + QueryProfiling.clear(); + } + }, + new TimeValue(0), + SQL_COMPLEX_WORKER_THREAD_POOL_NAME); + } else { + CalcitePlanContext.executionPool.set(SQL_WORKER_THREAD_POOL_NAME); + task.run(); + } + } + + private static final TimeValue CANCEL_POLL_INTERVAL = new TimeValue(500); + private static final Cancellable NOOP_CANCELLABLE = + new Cancellable() { + @Override + public boolean cancel() { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + }; + + /** + * Polls the cancellable task and interrupts the execution thread when cancelled. This bridges the + * gap where OpenSearchQueryManager's timeout interrupt targets the sql-worker thread but + * execution has moved to the complex-worker thread. + */ + private Cancellable scheduleCancellationPoller( + @Nullable CancellableTask cancellableTask, Thread executionThread) { + if (cancellableTask == null) { + return NOOP_CANCELLABLE; + } + return threadPool.scheduleWithFixedDelay( + () -> { + if (cancellableTask.isCancelled()) { + LOG.debug("Task cancelled, interrupting complex pool execution thread"); + executionThread.interrupt(); + } + }, + CANCEL_POLL_INTERVAL, + ThreadPool.Names.GENERIC); + } + + private boolean isComplexPoolEnabled() { + return settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index b596c7bc47a..ffa0571a9a0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -352,6 +352,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING = + Setting.boolSetting( + Key.SQL_COMPLEX_WORKER_POOL_ENABLED.getKeyValue(), + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + /** Construct OpenSearchSetting. The OpenSearchSetting must be singleton. */ @SuppressWarnings("unchecked") public OpenSearchSettings(ClusterSettings clusterSettings) { @@ -610,6 +617,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.FIELD_TYPE_TOLERANCE, FIELD_TYPE_TOLERANCE_SETTING, new Updater(Key.FIELD_TYPE_TOLERANCE)); + register( + settingBuilder, + clusterSettings, + Key.SQL_COMPLEX_WORKER_POOL_ENABLED, + SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING, + new Updater(Key.SQL_COMPLEX_WORKER_POOL_ENABLED)); defaultSettings = settingBuilder.build(); } @@ -703,6 +716,7 @@ public static List> pluginSettings() { .add(SESSION_INACTIVITY_TIMEOUT_MILLIS_SETTING) .add(STREAMING_JOB_HOUSEKEEPER_INTERVAL_SETTING) .add(FIELD_TYPE_TOLERANCE_SETTING) + .add(SQL_COMPLEX_WORKER_POOL_ENABLED_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java index 65ba189b0d9..3aa347b70fa 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java @@ -15,6 +15,7 @@ import javax.annotation.Nullable; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchSecurityException; +import org.opensearch.core.tasks.TaskCancelledException; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NonFallbackCalciteException; import org.opensearch.sql.monitor.profile.ProfileContext; @@ -118,7 +119,10 @@ private OpenSearchResponse getCurrentResponse(OpenSearchRequest request) { return nextBatchFuture.get(); } catch (OpenSearchSecurityException e) { throw e; - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TaskCancelledException("The task is cancelled."); + } catch (ExecutionException e) { if (e.getCause() instanceof OpenSearchSecurityException) { throw (OpenSearchSecurityException) e.getCause(); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java new file mode 100644 index 00000000000..e44809e40a9 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ScriptDetectorTest.java @@ -0,0 +1,212 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.logical.LogicalWindow; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.sql.SqlOperator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.AggSpec; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ScriptDetectorTest { + + @Test + void returnsFalseForNonScanNode() { + RelNode mockNode = createMockNode(); + assertFalse(ScriptDetector.hasScripts(mockNode)); + } + + @Test + void returnsTrueWhenFilterScriptPushed() { + AbstractCalciteIndexScan scan = createMockScan(true, false, 0); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsTrueWhenAggScriptPresent() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 1); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsTrueWhenSortExprPushed() { + AbstractCalciteIndexScan scan = createMockScan(false, true, 0); + assertTrue(ScriptDetector.hasScripts(scan)); + } + + @Test + void returnsFalseWhenNoScripts() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 0); + assertFalse(ScriptDetector.hasScripts(scan)); + } + + @Test + void detectsScriptsInNestedPlan() { + AbstractCalciteIndexScan scan = createMockScan(false, false, 3); + RelNode parent = createMockNode(scan); + assertTrue(ScriptDetector.hasScripts(parent)); + } + + @Test + void detectsJoinNode() { + LogicalJoin join = mock(LogicalJoin.class); + when(join.getJoinType()).thenReturn(JoinRelType.LEFT); + when(join.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(join).childrenAccept(any(RelVisitor.class)); + assertTrue(ScriptDetector.hasScripts(join)); + } + + @Test + void detectsWindowNode() { + LogicalWindow window = mock(LogicalWindow.class); + when(window.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(window).childrenAccept(any(RelVisitor.class)); + assertTrue(ScriptDetector.hasScripts(window)); + } + + @Test + void detectsExpensiveUdfInProject() { + SqlOperator rexExtractOp = mock(SqlOperator.class); + when(rexExtractOp.getName()).thenReturn("REX_EXTRACT"); + RexCall udfCall = mock(RexCall.class); + when(udfCall.getOperator()).thenReturn(rexExtractOp); + when(udfCall.getOperands()).thenReturn(List.of()); + RelDataType type = mock(RelDataType.class); + when(udfCall.getType()).thenReturn(type); + doAnswer(inv -> inv.>getArgument(0).visitCall(udfCall)) + .when(udfCall) + .accept(any()); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of(udfCall)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertTrue(ScriptDetector.hasScripts(project)); + } + + @Test + void ignoresCheapUdfInProject() throws Exception { + SqlOperator cheapOp = mock(SqlOperator.class); + when(cheapOp.getName()).thenReturn("NOW"); + RexCall cheapCall = mock(RexCall.class); + when(cheapCall.getOperator()).thenReturn(cheapOp); + RelDataType type = mock(RelDataType.class); + when(cheapCall.getType()).thenReturn(type); + // RexVisitorImpl.visitCall accesses call.operands field directly, set it via reflection + java.lang.reflect.Field operandsField = RexCall.class.getDeclaredField("operands"); + operandsField.setAccessible(true); + operandsField.set(cheapCall, com.google.common.collect.ImmutableList.of()); + doAnswer(inv -> inv.>getArgument(0).visitCall(cheapCall)) + .when(cheapCall) + .accept(any()); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of((RexNode) cheapCall)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertFalse(ScriptDetector.hasScripts(project)); + } + + @Test + void detectsRexOverInProject() { + RexOver rexOver = mock(RexOver.class); + SqlOperator op = mock(SqlOperator.class); + when(rexOver.getOperator()).thenReturn(op); + when(rexOver.getOperands()).thenReturn(List.of()); + RelDataType type = mock(RelDataType.class); + when(rexOver.getType()).thenReturn(type); + doAnswer(inv -> inv.>getArgument(0).visitOver(rexOver)) + .when(rexOver) + .accept(any(org.apache.calcite.rex.RexVisitor.class)); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of(rexOver)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertTrue(ScriptDetector.hasScripts(project)); + } + + @Test + void returnsFalseForSimpleFieldProject() { + RexInputRef fieldRef = mock(RexInputRef.class); + RelDataType type = mock(RelDataType.class); + when(fieldRef.getType()).thenReturn(type); + + LogicalProject project = mock(LogicalProject.class); + when(project.getProjects()).thenReturn(List.of((RexNode) fieldRef)); + when(project.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(project).childrenAccept(any(RelVisitor.class)); + + assertFalse(ScriptDetector.hasScripts(project)); + } + + private static RelNode createMockNode(RelNode... children) { + RelNode node = mock(RelNode.class); + List childList = List.of(children); + when(node.getInputs()).thenReturn(childList); + doAnswer( + invocation -> { + RelVisitor visitor = invocation.getArgument(0); + for (int i = 0; i < childList.size(); i++) { + visitor.visit(childList.get(i), i, node); + } + return null; + }) + .when(node) + .childrenAccept(any(RelVisitor.class)); + return node; + } + + private static AbstractCalciteIndexScan createMockScan( + boolean scriptPushed, boolean sortExprPushed, long aggScriptCount) { + AbstractCalciteIndexScan scan = mock(AbstractCalciteIndexScan.class); + + PushDownContext ctx = mock(PushDownContext.class); + when(ctx.isScriptPushed()).thenReturn(scriptPushed); + when(ctx.isSortExprPushed()).thenReturn(sortExprPushed); + + if (aggScriptCount > 0) { + AggSpec aggSpec = mock(AggSpec.class); + when(aggSpec.getScriptCount()).thenReturn(aggScriptCount); + when(ctx.getAggSpec()).thenReturn(aggSpec); + } else { + when(ctx.getAggSpec()).thenReturn(null); + } + + when(scan.getPushDownContext()).thenReturn(ctx); + when(scan.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(scan).childrenAccept(any(RelVisitor.class)); + return scan; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java new file mode 100644 index 00000000000..bf6adbf985e --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/ThreadPoolExecutionDispatcherTest.java @@ -0,0 +1,399 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.executor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelVisitor; +import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; +import org.apache.calcite.rel.metadata.RelMetadataQueryBase; +import org.apache.logging.log4j.ThreadContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; +import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.threadpool.Scheduler.Cancellable; +import org.opensearch.threadpool.Scheduler.ScheduledCancellable; +import org.opensearch.threadpool.ThreadPool; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ThreadPoolExecutionDispatcherTest { + + @Mock private ThreadPool threadPool; + @Mock private Settings settings; + @Mock private CalcitePlanContext context; + @Mock private ResponseListener listener; + @Mock private ExecutionEngine engine; + + private ThreadPoolExecutionDispatcher dispatcher; + + @BeforeEach + void setUp() { + dispatcher = new ThreadPoolExecutionDispatcher(threadPool, settings); + // Mock schedule calls to return non-null cancellables (for both outer dispatch and inner + // timeout) + when(threadPool.schedule(any(Runnable.class), any(TimeValue.class), any())) + .thenReturn(mock(ScheduledCancellable.class)); + when(threadPool.scheduleWithFixedDelay(any(Runnable.class), any(TimeValue.class), any())) + .thenReturn(mock(Cancellable.class)); + } + + @AfterEach + void tearDown() { + ThreadContext.clearAll(); + OpenSearchQueryManager.clearCancellableTask(); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + CalcitePlanContext.clearTimewrapSignals(); + } + + @Test + void executesInlineWhenNoScripts() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + RelNode plan = createMockNode(); + + dispatcher.dispatch(plan, context, listener, engine); + + verify(engine).execute(plan, context, listener); + verify(threadPool, never()).schedule(any(), any(TimeValue.class), any()); + } + + @Test + void dispatchesToSlowPoolWhenScriptsDetected() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + + dispatcher.dispatch(scan, context, listener, engine); + + verify(threadPool) + .schedule( + any(Runnable.class), eq(new TimeValue(0)), eq(SQL_COMPLEX_WORKER_THREAD_POOL_NAME)); + verify(engine, never()).execute(any(RelNode.class), any(), any(ResponseListener.class)); + } + + @Test + void executesInlineWhenSlowPoolDisabled() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(false); + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + + dispatcher.dispatch(scan, context, listener, engine); + + verify(engine).execute(scan, context, listener); + verify(threadPool, never()).schedule(any(), any(TimeValue.class), any()); + } + + @Test + void scheduledRunnableCallsEngine() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + verify(engine).execute(scan, context, listener); + } + + @Test + void propagatesCancellableTaskToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + + AtomicReference taskOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate running on a different thread — clear the ThreadLocal first + OpenSearchQueryManager.clearCancellableTask(); + task.run(); + taskOnSlowPool.set(OpenSearchQueryManager.getCancellableTask()); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + // During execution, the task should have been available + // (we check via a side-channel since the finally block clears it) + verify(engine).execute(scan, context, listener); + // After execution, it should be cleaned up + assertNull( + OpenSearchQueryManager.getCancellableTask(), + "CancellableTask should be cleared after execution"); + } + + @Test + void propagatesLog4jThreadContextToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + ThreadContext.put("request.id", "test-123"); + ThreadContext.put("user", "admin"); + + AtomicReference requestIdOnSlowPool = new AtomicReference<>(); + AtomicReference userOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate a different thread — clear MDC + ThreadContext.clearAll(); + task.run(); + requestIdOnSlowPool.set(ThreadContext.get("request.id")); + userOnSlowPool.set(ThreadContext.get("user")); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertEquals("test-123", requestIdOnSlowPool.get()); + assertEquals("admin", userOnSlowPool.get()); + } + + @Test + void propagatesMetadataProviderToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + JaninoRelMetadataProvider provider = mock(JaninoRelMetadataProvider.class); + RelMetadataQueryBase.THREAD_PROVIDERS.set(provider); + + AtomicReference providerOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + RelMetadataQueryBase.THREAD_PROVIDERS.remove(); + task.run(); + providerOnSlowPool.set(RelMetadataQueryBase.THREAD_PROVIDERS.get()); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + // After finally block, metadata provider should be cleaned up + assertNull( + RelMetadataQueryBase.THREAD_PROVIDERS.get(), + "Metadata provider should be cleaned up after execution"); + } + + @Test + void propagatesTimewrapSignalsToSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CalcitePlanContext.stripNullColumns.set(true); + CalcitePlanContext.timewrapUnitName.set("HOUR"); + CalcitePlanContext.timewrapSeries.set("timestamp"); + + AtomicReference stripOnSlowPool = new AtomicReference<>(); + AtomicReference unitOnSlowPool = new AtomicReference<>(); + AtomicReference seriesOnSlowPool = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Clear thread-locals to simulate new thread + CalcitePlanContext.clearTimewrapSignals(); + CalcitePlanContext.stripNullColumns.set(false); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + // Capture the values during engine.execute + doAnswer( + invocation -> { + stripOnSlowPool.set(CalcitePlanContext.stripNullColumns.get()); + unitOnSlowPool.set(CalcitePlanContext.timewrapUnitName.get()); + seriesOnSlowPool.set(CalcitePlanContext.timewrapSeries.get()); + return null; + }) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertEquals(true, stripOnSlowPool.get()); + assertEquals("HOUR", unitOnSlowPool.get()); + assertEquals("timestamp", seriesOnSlowPool.get()); + } + + @Test + void forwardsExceptionToListenerOnSlowPool() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + RuntimeException error = new RuntimeException("execution failed"); + doThrow(error).when(engine).execute(any(RelNode.class), any(), any(ResponseListener.class)); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + verify(listener).onFailure(error); + } + + @Test + void cleansUpThreadLocalsAfterException() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + CalcitePlanContext.timewrapUnitName.set("DAY"); + RelMetadataQueryBase.THREAD_PROVIDERS.set(mock(JaninoRelMetadataProvider.class)); + + doThrow(new RuntimeException("boom")) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertNull(OpenSearchQueryManager.getCancellableTask()); + assertNull(RelMetadataQueryBase.THREAD_PROVIDERS.get()); + // timewrapSignals cleared via clearTimewrapSignals() + assertNull(CalcitePlanContext.timewrapUnitName.get()); + } + + @Test + void cancellableTaskAvailableDuringExecution() { + when(settings.getSettingValue(Settings.Key.SQL_COMPLEX_WORKER_POOL_ENABLED)) + .thenReturn(true); + when(settings.getSettingValue(Settings.Key.PPL_QUERY_TIMEOUT)) + .thenReturn(new TimeValue(60000)); + CancellableTask mockTask = mock(CancellableTask.class); + OpenSearchQueryManager.setCancellableTask(mockTask); + + AtomicReference taskDuringExecution = new AtomicReference<>(); + doAnswer( + invocation -> { + Runnable task = invocation.getArgument(0); + // Simulate running on a different thread + OpenSearchQueryManager.clearCancellableTask(); + task.run(); + return mock(ScheduledCancellable.class); + }) + .when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any()); + + doAnswer( + invocation -> { + taskDuringExecution.set(OpenSearchQueryManager.getCancellableTask()); + return null; + }) + .when(engine) + .execute(any(RelNode.class), any(), any(ResponseListener.class)); + + AbstractCalciteIndexScan scan = createMockScanWithScripts(); + dispatcher.dispatch(scan, context, listener, engine); + + assertNotNull( + taskDuringExecution.get(), "CancellableTask should be available during execution"); + assertEquals(mockTask, taskDuringExecution.get()); + } + + private static RelNode createMockNode(RelNode... children) { + RelNode node = mock(RelNode.class); + List childList = List.of(children); + when(node.getInputs()).thenReturn(childList); + doAnswer( + invocation -> { + RelVisitor visitor = invocation.getArgument(0); + for (int i = 0; i < childList.size(); i++) { + visitor.visit(childList.get(i), i, node); + } + return null; + }) + .when(node) + .childrenAccept(any(RelVisitor.class)); + return node; + } + + private static AbstractCalciteIndexScan createMockScanWithScripts() { + AbstractCalciteIndexScan scan = mock(AbstractCalciteIndexScan.class); + PushDownContext ctx = mock(PushDownContext.class); + when(ctx.isScriptPushed()).thenReturn(true); + when(ctx.isSortExprPushed()).thenReturn(false); + when(ctx.getAggSpec()).thenReturn(null); + when(scan.getPushDownContext()).thenReturn(ctx); + when(scan.getInputs()).thenReturn(List.of()); + doAnswer(invocation -> null).when(scan).childrenAccept(any(RelVisitor.class)); + return scan; + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index e1278aa75e8..214c442755f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.datasource.model.DataSourceMetadata.defaultOpenSearchDataSourceMetadata; import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_BACKGROUND_THREAD_POOL_NAME; +import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_COMPLEX_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.opensearch.executor.OpenSearchQueryManager.SQL_WORKER_THREAD_POOL_NAME; import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_REQUEST_BUFFER_INDEX_NAME; @@ -233,7 +234,13 @@ private BiFunction createSqlAnalyticsRout } cached[0] = new RestUnifiedQueryAction( - client, clusterService, executor, contextProvider, pluginSettings); + client, + clusterService, + executor, + contextProvider, + pluginSettings, + new org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher( + client.threadPool(), pluginSettings)); } return cached[0]; }; @@ -446,10 +453,12 @@ public ScheduledJobParser getJobParser() { @Override public List> getExecutorBuilders(Settings settings) { - // The worker pool is the primary pool where most of the work is done. The background thread - // pool is a separate queue for asynchronous requests to other nodes. We keep them separate to - // prevent deadlocks during async fetches on small node counts. Tasks in the background pool - // should do no work except I/O to other services. + // The worker pool is the primary pool where most of the work is done. The complex-worker pool + // handles queries that require scripts (table scans that can't be pushed to Lucene) so they + // don't starve fast queries. The background thread pool is a separate queue for asynchronous + // requests to other nodes. We keep them separate to prevent deadlocks during async fetches on + // small node counts. Tasks in the background pool should do no work except I/O to other + // services. return List.of( new FixedExecutorBuilder( settings, @@ -457,11 +466,17 @@ public List> getExecutorBuilders(Settings settings) { OpenSearchExecutors.allocatedProcessors(settings), 1000, "thread_pool." + SQL_WORKER_THREAD_POOL_NAME), + new FixedExecutorBuilder( + settings, + SQL_COMPLEX_WORKER_THREAD_POOL_NAME, + OpenSearchExecutors.allocatedProcessors(settings), + 1000, + "thread_pool." + SQL_COMPLEX_WORKER_THREAD_POOL_NAME), new FixedExecutorBuilder( settings, SQL_BACKGROUND_THREAD_POOL_NAME, settings.getAsInt( - "thread_pool.search.size", OpenSearchExecutors.allocatedProcessors(settings)), + "thread_pool.search.size", 2 * OpenSearchExecutors.allocatedProcessors(settings)), 1000, "thread_pool." + SQL_BACKGROUND_THREAD_POOL_NAME)); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java index 057c88c9a02..816f1071310 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -15,6 +15,7 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.DelegatingExecutionEngine; +import org.opensearch.sql.executor.ExecutionDispatcher; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryService; @@ -26,6 +27,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; +import org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher; import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; @@ -114,13 +116,19 @@ public SQLService sqlService( /** {@link QueryPlanFactory}. */ @Provides public QueryPlanFactory queryPlanFactory( - DataSourceService dataSourceService, ExecutionEngine executionEngine, Settings settings) { + DataSourceService dataSourceService, + ExecutionEngine executionEngine, + Settings settings, + NodeClient nodeClient) { Analyzer analyzer = new Analyzer( new ExpressionAnalyzer(functionRepository), dataSourceService, functionRepository); Planner planner = new Planner(LogicalPlanOptimizer.create()); + ExecutionDispatcher executionDispatcher = + new ThreadPoolExecutionDispatcher(nodeClient.threadPool(), settings); QueryService queryService = - new QueryService(analyzer, executionEngine, planner, dataSourceService, settings); + new QueryService( + analyzer, executionEngine, planner, dataSourceService, settings, executionDispatcher); return new QueryPlanFactory(queryService); } } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index f8214096e41..26c71060c2e 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -69,18 +69,21 @@ public class RestUnifiedQueryAction { private final ClusterService clusterService; private final org.opensearch.analytics.EngineContextProvider contextProvider; private final org.opensearch.sql.common.setting.Settings pluginSettings; + private final org.opensearch.sql.executor.ExecutionDispatcher executionDispatcher; public RestUnifiedQueryAction( NodeClient client, ClusterService clusterService, QueryPlanExecutor> planExecutor, org.opensearch.analytics.EngineContextProvider contextProvider, - org.opensearch.sql.common.setting.Settings pluginSettings) { + org.opensearch.sql.common.setting.Settings pluginSettings, + org.opensearch.sql.executor.ExecutionDispatcher executionDispatcher) { this.client = client; this.clusterService = clusterService; this.analyticsEngine = new AnalyticsExecutionEngine(planExecutor); this.contextProvider = contextProvider; this.pluginSettings = pluginSettings; + this.executionDispatcher = executionDispatcher; } /** @@ -229,19 +232,25 @@ private void doExecute( // string, so apply the equivalent top-level limit here before the system cap. plan = addFetchSizeLimit(plan, planContext, fetchSize); plan = addQuerySizeLimit(plan, planContext); - if (profiling) { - analyticsEngine.executeWithProfile( - plan, - planContext, - queryCtx, - createQueryListener(queryType, profileCtx, closingListener)); - } else { - analyticsEngine.execute( - plan, - planContext, - queryCtx, - createQueryListener(queryType, profileCtx, closingListener)); - } + plan = + org.opensearch.sql.calcite.utils.CalciteToolsHelper.optimize( + plan, planContext); + RelNode finalPlan = plan; + Runnable executeTask = + profiling + ? () -> + analyticsEngine.executeWithProfile( + finalPlan, + planContext, + queryCtx, + createQueryListener(queryType, profileCtx, closingListener)) + : () -> + analyticsEngine.execute( + finalPlan, + planContext, + queryCtx, + createQueryListener(queryType, profileCtx, closingListener)); + executionDispatcher.dispatchTask(finalPlan, planContext, executeTask); } catch (Exception e) { closingListener.onFailure(e); } finally { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index b54c2c49ab9..772f1ec123f 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -136,7 +136,13 @@ private void buildUnifiedQueryHandlerIfReady() { if (executor != null && contextProvider != null) { this.unifiedQueryHandler = new RestUnifiedQueryAction( - clientRef, clusterServiceRef, executor, contextProvider, pluginSettingsRef); + clientRef, + clusterServiceRef, + executor, + contextProvider, + pluginSettingsRef, + new org.opensearch.sql.opensearch.executor.ThreadPoolExecutionDispatcher( + clientRef.threadPool(), pluginSettingsRef)); } } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 111597bb587..de1ce111558 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -54,7 +54,8 @@ public void setUp() { clusterService, executor, mock(EngineContextProvider.class), - mock(org.opensearch.sql.common.setting.Settings.class)); + mock(org.opensearch.sql.common.setting.Settings.class), + new org.opensearch.sql.executor.DirectExecutionDispatcher()); } @Test From de95ffbe9c15b450676016574f009b0fa09afdd7 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:05:05 -0700 Subject: [PATCH 14/78] Detect integral SUM overflow and fix BIGINT AVG (#5612) --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 6 +- .../udf/udaf/BigintAvgAggFunction.java | 31 +++ .../udf/udaf/CheckedLongSumAggFunction.java | 22 ++ .../utils/UserDefinedFunctionUtils.java | 25 +- .../function/PPLBuiltinOperators.java | 18 +- .../expression/function/PPLFuncImpTable.java | 38 ++- .../udf/udaf/BigintAvgAggFunctionTest.java | 44 ++++ .../udaf/CheckedLongSumAggFunctionTest.java | 42 ++++ docs/user/ppl/functions/expressions.md | 10 +- .../remote/CalcitePPLAggregationIT.java | 175 ++++++++++++- .../opensearch/sql/ppl/StatsCommandIT.java | 5 - .../calcite/chart_null_str.yaml | 8 +- .../calcite/clickbench/q10.yaml | 4 +- .../expectedOutput/calcite/clickbench/q3.yaml | 4 +- .../calcite/clickbench/q30.yaml | 4 +- .../calcite/clickbench/q31.yaml | 4 +- .../calcite/clickbench/q32.yaml | 4 +- .../calcite/clickbench/q33.yaml | 4 +- .../calcite/explain_agg_sort_on_measure2.yaml | 4 +- .../calcite/explain_agg_sort_on_measure4.yaml | 4 +- .../explain_agg_sort_on_measure_complex1.yaml | 4 +- .../explain_agg_sort_on_measure_complex2.yaml | 4 +- ...t_on_measure_multi_buckets_not_pushed.yaml | 4 +- .../calcite/explain_agg_with_script.yaml | 4 +- .../explain_agg_with_sum_enhancement.yaml | 4 +- .../calcite/explain_bin_minspan.json | 7 +- ...gg_with_sort_on_one_measure_not_push1.yaml | 4 +- ...gg_with_sort_on_one_measure_not_push2.yaml | 4 +- .../calcite/explain_streamstats_global.yaml | 4 +- ...xplain_streamstats_global_null_bucket.yaml | 4 +- .../calcite/explain_streamstats_reset.yaml | 23 +- ...explain_streamstats_reset_null_bucket.yaml | 25 +- .../agg_case_composite_cannot_push.yaml | 4 +- .../agg_composite2_range_count_push.yaml | 4 +- ...agg_composite2_range_range_count_push.yaml | 4 +- .../agg_composite_range_metric_push.yaml | 4 +- .../agg_range_count_push.yaml | 2 +- .../agg_range_metric_complex_push.yaml | 4 +- .../agg_range_metric_push.yaml | 2 +- .../agg_range_range_metric_push.yaml | 6 +- .../chart_multiple_group_keys.yaml | 8 +- .../calcite_no_pushdown/chart_null_str.yaml | 8 +- .../chart_single_group_key.yaml | 7 +- .../calcite_no_pushdown/chart_with_limit.yaml | 4 +- .../explain_agg_with_script.yaml | 6 +- .../explain_agg_with_sum_enhancement.yaml | 4 +- .../explain_bin_minspan.json | 7 +- .../explain_filter_agg_push.yaml | 4 +- .../calcite_no_pushdown/explain_output.yaml | 4 +- .../explain_sort_agg_push.json | 2 +- .../explain_sort_then_agg_push.json | 2 +- .../explain_streamstats_global.yaml | 4 +- ...xplain_streamstats_global_null_bucket.yaml | 4 +- .../explain_streamstats_reset.yaml | 23 +- ...explain_streamstats_reset_null_bucket.yaml | 25 +- .../rest-api-spec/test/issues/5164_agg.yml | 230 ++++++++++++++++++ .../opensearch/request/AggregateAnalyzer.java | 17 ++ .../response/agg/CheckedLongSumParser.java | 47 ++++ .../request/AggregateAnalyzerTest.java | 60 +++++ .../agg/CheckedLongSumParserTest.java | 70 ++++++ .../calcite/OpenSearchSparkSqlDialect.java | 3 +- .../sql/ppl/calcite/CalcitePPLJoinTest.java | 2 +- .../ppl/calcite/CalcitePPLTimewrapTest.java | 5 +- 63 files changed, 966 insertions(+), 157 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java create mode 100644 core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 3320391c0d2..ca0c524b4a1 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -286,7 +286,7 @@ SELECT department, SUM(age) AS total FROM catalog.employees GROUP BY department .assertPlan( """ LogicalProject(department=[$0], total=[$1]) - LogicalAggregate(group=[{0}], SUM(age)=[SUM($1)]) + LogicalAggregate(group=[{0}], SUM(age)=[CHECKED_LONG_SUM($1)]) LogicalProject(department=[$3], age=[$2]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -366,7 +366,7 @@ SELECT department, SUM(age) FILTER(WHERE age > 30) FROM catalog.employees """) .assertPlan( """ - LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[SUM($1) FILTER $2]) + LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[CHECKED_LONG_SUM($1) FILTER $2]) LogicalProject(department=[$3], age=[$2], $f3=[>($2, 30)]) LogicalTableScan(table=[[catalog, employees]]) """); @@ -487,7 +487,7 @@ SELECT name, SUM(age) OVER(PARTITION BY department ORDER BY age) FROM catalog.em """) .assertPlan( """ - LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) + LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[CHECKED_LONG_SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) LogicalTableScan(table=[[catalog, employees]]) """); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java new file mode 100644 index 00000000000..4b33b9a0ffa --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunction.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT average aggregate that accumulates in double to avoid an intermediate long overflow. */ +public class BigintAvgAggFunction { + + public static Accumulator init() { + return new Accumulator(); + } + + public static Accumulator add(Accumulator accumulator, Long value) { + if (value != null) { + accumulator.sum += value; + accumulator.count++; + } + return accumulator; + } + + public static Double result(Accumulator accumulator) { + return accumulator.count == 0 ? null : accumulator.sum / accumulator.count; + } + + public static class Accumulator { + private double sum; + private long count; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java new file mode 100644 index 00000000000..7e146eacedb --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunction.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +/** BIGINT sum aggregate that throws when its running long accumulator overflows. */ +public class CheckedLongSumAggFunction { + + public static long init() { + return 0L; + } + + public static long add(long accumulator, long value) { + return Math.addExact(accumulator, value); + } + + public static long result(long accumulator) { + return accumulator; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index f619d966cc8..0b365fc98e6 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -94,9 +94,32 @@ public static SqlUserDefinedAggFunction createUserDefinedAggFunction( String functionName, SqlReturnTypeInference returnType, @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction(udafClass, functionName, returnType, operandMetadata); + } + + /** Creates an aggregate function from a class following Calcite's reflective UDAF convention. */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { + return createReflectiveAggFunction( + udafClass, functionName, SqlKind.OTHER_FUNCTION, returnType, operandMetadata); + } + + /** + * Creates an aggregate function whose kind remains visible to planner rules while execution uses + * the supplied reflective UDAF. + */ + public static SqlUserDefinedAggFunction createReflectiveAggFunction( + Class udafClass, + String functionName, + SqlKind kind, + SqlReturnTypeInference returnType, + @Nullable UDFOperandMetadata operandMetadata) { return new SqlUserDefinedAggFunction( new SqlIdentifier(functionName, SqlParserPos.ZERO), - SqlKind.OTHER_FUNCTION, + kind, returnType, null, operandMetadata, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 2a670af3fee..812b94967f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptExprMethodWithPropertiesToUDF; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.adaptMathFunctionToUDF; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createReflectiveAggFunction; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.createUserDefinedAggFunction; import com.google.common.base.Suppliers; @@ -29,6 +30,8 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.calcite.udf.udaf.BigintAvgAggFunction; +import org.opensearch.sql.calcite.udf.udaf.CheckedLongSumAggFunction; import org.opensearch.sql.calcite.udf.udaf.DistinctCountApproxLogicalAggFunction; import org.opensearch.sql.calcite.udf.udaf.FirstAggFunction; import org.opensearch.sql.calcite.udf.udaf.LastAggFunction; @@ -449,7 +452,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); public static final SqlOperator TONUMBER = new ToNumberFunction().toUDF("TONUMBER"); public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); - // PPL Convert command functions public static final SqlOperator AUTO = new AutoConvertFunction().toUDF("AUTO"); public static final SqlOperator NUM = new NumConvertFunction().toUDF("NUM"); @@ -488,6 +490,20 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { new NullableSqlAvgAggFunction(SqlKind.VAR_POP); public static final SqlAggFunction VAR_SAMP_NULLABLE = new NullableSqlAvgAggFunction(SqlKind.VAR_SAMP); + public static final SqlAggFunction CHECKED_LONG_SUM = + createReflectiveAggFunction( + CheckedLongSumAggFunction.class, + "CHECKED_LONG_SUM", + SqlKind.SUM, + ReturnTypes.BIGINT_FORCE_NULLABLE, + PPLOperandTypes.NUMERIC); + public static final SqlAggFunction BIGINT_AVG = + createReflectiveAggFunction( + BigintAvgAggFunction.class, + "AVG", + SqlKind.AVG, + ReturnTypes.DOUBLE_NULLABLE, + PPLOperandTypes.NUMERIC); public static final SqlAggFunction TAKE = createUserDefinedAggFunction( TakeAggFunction.class, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 64f29906829..a4f05dd675e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -283,6 +283,7 @@ import java.util.Optional; import java.util.StringJoiner; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; @@ -1595,7 +1596,14 @@ void register( } void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFunction) { - SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(aggFunction); + registerOperator(functionName, aggFunction, field -> aggFunction); + } + + void registerOperator( + BuiltinFunctionName functionName, + SqlAggFunction typeCheckerSource, + Function aggFunctionSelector) { + SqlOperandTypeChecker innerTypeChecker = extractTypeCheckerFromUDF(typeCheckerSource); PPLTypeChecker typeChecker = wrapSqlOperandTypeChecker(innerTypeChecker, functionName.name(), true); AggHandler handler = @@ -1603,15 +1611,30 @@ void registerOperator(BuiltinFunctionName functionName, SqlAggFunction aggFuncti List newArgList = argList.stream().map(PlanUtils::derefMapCall).collect(Collectors.toList()); return UserDefinedFunctionUtils.makeAggregateCall( - aggFunction, List.of(field), newArgList, ctx.relBuilder); + aggFunctionSelector.apply(field), List.of(field), newArgList, ctx.relBuilder); }; register(functionName, handler, typeChecker); } + /** Registers checked integral sums while retaining standard SUM behavior for other types. */ + void registerSumOperator() { + registerOperator( + SUM, + SqlStdOperatorTable.SUM, + field -> + isIntegral(field.getType().getSqlTypeName()) + ? PPLBuiltinOperators.CHECKED_LONG_SUM + : SqlStdOperatorTable.SUM); + } + + private static boolean isIntegral(SqlTypeName typeName) { + return SqlTypeName.INT_TYPES.contains(typeName); + } + void populate() { registerOperator(MAX, SqlStdOperatorTable.MAX); registerOperator(MIN, SqlStdOperatorTable.MIN); - registerOperator(SUM, SqlStdOperatorTable.SUM); + registerSumOperator(); registerOperator(VARSAMP, PPLBuiltinOperators.VAR_SAMP_NULLABLE); registerOperator(VARPOP, PPLBuiltinOperators.VAR_POP_NULLABLE); registerOperator(STDDEV_SAMP, PPLBuiltinOperators.STDDEV_SAMP_NULLABLE); @@ -1630,7 +1653,14 @@ void populate() { register( AVG, - (distinct, field, argList, ctx) -> ctx.relBuilder.avg(distinct, null, field), + (distinct, field, argList, ctx) -> { + if (field.getType().getSqlTypeName() == SqlTypeName.BIGINT) { + return ctx.relBuilder + .aggregateCall(PPLBuiltinOperators.BIGINT_AVG, field) + .distinct(distinct); + } + return ctx.relBuilder.avg(distinct, null, field); + }, wrapSqlOperandTypeChecker( SqlStdOperatorTable.AVG.getOperandTypeChecker(), AVG.name(), false)); diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java new file mode 100644 index 00000000000..ab70f912211 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/BigintAvgAggFunctionTest.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class BigintAvgAggFunctionTest { + + @Test + void retainsAvgKindForPushdownRules() { + assertEquals(SqlKind.AVG, PPLBuiltinOperators.BIGINT_AVG.getKind()); + } + + @Test + void averagesWithoutLongOverflow() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + accumulator = BigintAvgAggFunction.add(accumulator, Long.MAX_VALUE); + + assertEquals((double) Long.MAX_VALUE, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void ignoresNulls() { + BigintAvgAggFunction.Accumulator accumulator = BigintAvgAggFunction.init(); + accumulator = BigintAvgAggFunction.add(accumulator, null); + accumulator = BigintAvgAggFunction.add(accumulator, 10L); + + assertEquals(10D, BigintAvgAggFunction.result(accumulator)); + } + + @Test + void returnsNullForEmptyInput() { + assertNull(BigintAvgAggFunction.result(BigintAvgAggFunction.init())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java new file mode 100644 index 00000000000..0128cb4aa67 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/udf/udaf/CheckedLongSumAggFunctionTest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.calcite.sql.SqlKind; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +class CheckedLongSumAggFunctionTest { + + @Test + void retainsSumKindForPlannerRules() { + assertEquals(SqlKind.SUM, PPLBuiltinOperators.CHECKED_LONG_SUM.getKind()); + } + + @Test + void sumsExactly() { + long accumulator = CheckedLongSumAggFunction.init(); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L << 62); + accumulator = CheckedLongSumAggFunction.add(accumulator, 1L); + + assertEquals((1L << 62) + 1L, CheckedLongSumAggFunction.result(accumulator)); + } + + @Test + void throwsOnPositiveOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MAX_VALUE, 1L)); + } + + @Test + void throwsOnNegativeOverflow() { + assertThrows( + ArithmeticException.class, () -> CheckedLongSumAggFunction.add(Long.MIN_VALUE, -1L)); + } +} diff --git a/docs/user/ppl/functions/expressions.md b/docs/user/ppl/functions/expressions.md index 427a0334b58..14d9606a0c2 100644 --- a/docs/user/ppl/functions/expressions.md +++ b/docs/user/ppl/functions/expressions.md @@ -13,7 +13,14 @@ Arithmetic expressions are formed by combining numeric literals and binary arith ### Overflow behavior -Integer and long arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. For example, `eval x = int_field + 1` where `int_field` is `2147483647` (integer max) returns an error rather than `-2147483648`. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. +Long (`BIGINT`) arithmetic operations (`+`, `-`, `*`) in `eval` expressions detect overflow and return an error instead of silently wrapping. Narrower integer operands are widened before arithmetic, so crossing the 32-bit integer boundary does not overflow. Floating-point (`float`, `double`) arithmetic follows IEEE 754 and does not produce overflow errors. + +The accumulator used by `stats sum(integral_field)` depends on whether Calcite pushdown is enabled: + +- With `plugins.calcite.pushdown.enabled=true` (the default), OpenSearch uses its native double-based `sum`, then the result is checked and narrowed to `BIGINT`. Large in-range sums can lose low-order precision. A small overflow that rounds to a signed `BIGINT` boundary can also be indistinguishable from an in-range sum and may saturate at the boundary instead of returning an error. +- With `plugins.calcite.pushdown.enabled=false`, Calcite uses an exact `BIGINT` accumulator and `Math.addExact` for every addition. It returns an error as soon as the running sum exceeds the `BIGINT` range. + +For example, summing `4611686018427387904` (`2^62`) and `1` returns the exact `4611686018427387905` without pushdown. With pushdown enabled, the double accumulator cannot represent the low-order `1`, so the result is `4611686018427387904`. ### Precedence @@ -189,4 +196,3 @@ fetched rows / total rows = 2/2 | 28 | +-----+ ``` - \ No newline at end of file diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index a2ab93b6599..f043d9b3afc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -20,12 +20,16 @@ import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; +import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; +import static org.opensearch.sql.util.TestUtils.isIndexExist; +import static org.opensearch.sql.util.TestUtils.performRequest; import java.io.IOException; import java.util.Arrays; import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -93,6 +97,173 @@ public void testSumAvg() throws IOException { verifyDataRows(actual, rows(186973)); } + @Test + public void testSumAllIntegralTypes() throws IOException { + String stats = + "stats sum(byte_number), sum(short_number), sum(integer_number), sum(long_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + JSONObject actual = executeQuery(query); + verifySchema( + actual, + schema("sum(byte_number)", "bigint"), + schema("sum(short_number)", "bigint"), + schema("sum(integer_number)", "bigint"), + schema("sum(long_number)", "bigint")); + verifyDataRows(actual, rows(4L, 3L, 2L, 1L)); + + String explain = explainQueryToString(query); + assertAllIntegralSumsAreChecked(explain); + + // HEAD prevents aggregation pushdown while preserving each field's integral input type. + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + verifyDataRows(executeQuery(fallbackQuery), rows(4L, 3L, 2L, 1L)); + + String fallbackExplain = explainQueryToString(fallbackQuery); + assertTrue(fallbackExplain.contains("EnumerableAggregate")); + assertAllIntegralSumsAreChecked(fallbackExplain); + } + + private static void assertAllIntegralSumsAreChecked(String explain) { + assertTrue(explain.contains("sum(byte_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(short_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(integer_number)=[CHECKED_LONG_SUM(")); + assertTrue(explain.contains("sum(long_number)=[CHECKED_LONG_SUM(")); + } + + @Test + public void testSumAvgLongOverflow() throws IOException { + String overflowIndex = "test_sum_long_overflow"; + createLongIndex(overflowIndex, Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE); + String inRangeIndex = "test_sum_long_in_range"; + createLongIndex(inRangeIndex, 1000000000000L, 2000000000000L, 3000000000000L); + String boundaryIndex = "test_sum_long_boundary"; + createLongIndex(boundaryIndex, Long.MAX_VALUE); + String exactIndex = "test_sum_long_exact"; + createLongIndex(exactIndex, 4611686018427387904L, 1L); + + // SUM overflows the BIGINT range (3 * (2^63 - 1)); surfaced as a client error rather than + // silently wrapping to a negative value. + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + + // HEAD forces enumerable execution even when global pushdown is enabled. + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + // AVG is averaged in DOUBLE, so it holds the true average (the shared value) without wrapping. + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows(9.223372036854776e18)); + + JSONObject fallbackAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v)", overflowIndex)); + verifySchema(fallbackAvg, schema("avg(v)", "double")); + verifyDataRows(fallbackAvg, rows(9.223372036854776e18)); + + JSONObject expressionAvg = + executeQuery(String.format("source=%s | head 3 | stats avg(v + 0)", overflowIndex)); + verifySchema(expressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(expressionAvg, rows(9.223372036854776e18)); + + String pushedExpressionQuery = String.format("source=%s | stats avg(v + 0)", overflowIndex); + JSONObject pushedExpressionAvg = executeQuery(pushedExpressionQuery); + verifySchema(pushedExpressionAvg, schema("avg(v + 0)", "double")); + verifyDataRows(pushedExpressionAvg, rows(9.223372036854776e18)); + if (!isPushdownDisabled() && !isAnalyticsParquetIndicesEnabled()) { + assertTrue(explainQueryToString(pushedExpressionQuery).contains("AGGREGATION->")); + } + + // A sum well within the BIGINT range returns the exact value with no error. + JSONObject inRange = executeQuery(String.format("source=%s | stats sum(v)", inRangeIndex)); + verifySchema(inRange, schema("sum(v)", "bigint")); + verifyDataRows(inRange, rows(6000000000000L)); + + // A single Long.MAX_VALUE is a valid (non-overflowing) sum and must not error. + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(9223372036854775807L)); + + // Native sum pushdown uses double and loses the low-order bit; the fallback and analytics + // backends retain it. + JSONObject exact = executeQuery(String.format("source=%s | stats sum(v)", exactIndex)); + verifySchema(exact, schema("sum(v)", "bigint")); + long expectedExact = + (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) + ? 4611686018427387905L + : 4611686018427387904L; + verifyDataRows(exact, rows(expectedExact)); + + // HEAD prevents pushdown, so the checked long accumulator retains the low-order bit. + JSONObject exactFallback = + executeQuery(String.format("source=%s | head 2 | stats sum(v)", exactIndex)); + verifySchema(exactFallback, schema("sum(v)", "bigint")); + verifyDataRows(exactFallback, rows(4611686018427387905L)); + } + + @Test + public void testNegativeLongSumOverflowAndBoundary() throws IOException { + String overflowIndex = "test_sum_long_negative_overflow"; + createLongIndex(overflowIndex, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + String boundaryIndex = "test_sum_long_negative_boundary"; + createLongIndex(boundaryIndex, Long.MIN_VALUE); + + assertSumOverflow(String.format("source=%s | stats sum(v)", overflowIndex)); + assertSumOverflow(String.format("source=%s | head 3 | stats sum(v)", overflowIndex)); + + JSONObject boundary = executeQuery(String.format("source=%s | stats sum(v)", boundaryIndex)); + verifySchema(boundary, schema("sum(v)", "bigint")); + verifyDataRows(boundary, rows(Long.MIN_VALUE)); + + JSONObject avg = executeQuery(String.format("source=%s | stats avg(v)", overflowIndex)); + verifySchema(avg, schema("avg(v)", "double")); + verifyDataRows(avg, rows((double) Long.MIN_VALUE)); + } + + @Test + public void testFallbackLongSumRejectsIntermediateOverflow() throws IOException { + String index = "test_sum_long_intermediate_overflow"; + createLongIndex(index, Long.MAX_VALUE, 1L, -1L); + + // The final mathematical result fits, but Math.addExact rejects the intermediate MAX + 1. + assertSumOverflow(String.format("source=%s | sort - v | head 3 | stats sum(v)", index)); + } + + @Test + public void testFloatingSumsDoNotUseCheckedLongAccumulator() throws IOException { + String stats = + "stats sum(double_number), sum(float_number)," + + " sum(half_float_number), sum(scaled_float_number)"; + String query = String.format("source=%s | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + String fallbackQuery = + String.format("source=%s | head 1 | %s", TEST_INDEX_DATATYPE_NUMERIC, stats); + + assertEquals(1, executeQuery(query).getInt("total")); + assertFalse(explainQueryToString(query).contains("CHECKED_LONG_SUM")); + assertEquals(1, executeQuery(fallbackQuery).getInt("total")); + assertFalse(explainQueryToString(fallbackQuery).contains("CHECKED_LONG_SUM")); + } + + private void createLongIndex(String index, long... values) throws IOException { + if (isIndexExist(client(), index)) { + return; + } + + createIndexByRestClient( + client(), index, "{\"mappings\":{\"properties\":{\"v\":{\"type\":\"long\"}}}}"); + StringBuilder body = new StringBuilder(); + for (long value : values) { + body.append("{\"index\":{}}\n").append("{\"v\":").append(value).append("}\n"); + } + Request bulk = new Request("POST", "/" + index + "/_bulk?refresh=true"); + bulk.setJsonEntity(body.toString()); + performRequest(client(), bulk); + } + + private void assertSumOverflow(String query) throws IOException { + Throwable error = assertThrowsWithReplace(RuntimeException.class, () -> executeQuery(query)); + verifyErrorMessageContains(error, "verflow"); + } + @Test public void testAsExistedField() throws IOException { JSONObject actual = @@ -993,9 +1164,7 @@ public void testSumGroupByNullValue() throws IOException { String.format( "source=%s | stats sum(balance) as a by age", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); - // SUM of an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 instead - // (a known pushdown quirk); the analytics-engine backend (DataFusion) follows the spec like - // Calcite-no-pushdown and returns null. See testSumNull and #3408. + // Native sum returns 0 for an all-null bucket; fallback and analytics backends return null. Object emptySum = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows( response, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 7417fd112ec..4fd8aa7f852 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -512,11 +512,6 @@ public void testSumWithNull() throws IOException { "source=%s | where age = 36 | stats sum(balance)", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("sum(balance)", null, "bigint")); - // TODO: Fix -- temporary workaround for the pushdown issue: - // The current pushdown implementation will return 0 for sum when getting null values as input. - // Returning null should be the expected behavior. - // The analytics-engine backend (DataFusion) follows the SQL spec like Calcite-no-pushdown — - // SUM of all-null is null, not 0. Integer expectedValue = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows(response, rows(expectedValue)); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index 726eeedc429..f9196e3b597 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -25,15 +25,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], expr#11=[IS NOT NULL($t4)], age=[$t4], avg(balance)=[$t10], $condition=[$t11]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], expr#4=[IS NOT NULL($t3)], age=[$t3], avg(balance)=[$t2], $condition=[$t4]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml index f900b2ccbec..4e024b0c12d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q10.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(sum(AdvEngineID)=[$1], c=[$2], avg(ResolutionWidth)=[$3], dc(UserID)=[$4], RegionID=[$0]) - LogicalAggregate(group=[{0}], sum(AdvEngineID)=[SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0}], sum(AdvEngineID)=[CHECKED_LONG_SUM($1)], c=[COUNT()], avg(ResolutionWidth)=[AVG($2)], dc(UserID)=[COUNT(DISTINCT $3)]) LogicalProject(RegionID=[$68], AdvEngineID=[$19], ResolutionWidth=[$80], UserID=[$84]) LogicalFilter(condition=[IS NOT NULL($68)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(AdvEngineID)=CHECKED_LONG_SUM($0),c=COUNT(),avg(ResolutionWidth)=AVG($2),dc(UserID)=COUNT(DISTINCT $3)), PROJECT->[sum(AdvEngineID), c, avg(ResolutionWidth), dc(UserID), RegionID], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"RegionID":{"terms":{"field":"RegionID","size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"dc(UserID)":{"cardinality":{"field":"UserID"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml index ef93b63ee80..24ddecd3881 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q3.yaml @@ -1,8 +1,8 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(AdvEngineID)=[SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) + LogicalAggregate(group=[{}], sum(AdvEngineID)=[CHECKED_LONG_SUM($0)], count()=[COUNT()], avg(ResolutionWidth)=[AVG($1)]) LogicalProject(AdvEngineID=[$19], ResolutionWidth=[$80]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},sum(AdvEngineID)=CHECKED_LONG_SUM($0),count()=COUNT(),avg(ResolutionWidth)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"sum(AdvEngineID)":{"sum":{"field":"AdvEngineID"}},"count()":{"value_count":{"field":"_index"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml index d50a9ec47ce..189ab6349e1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q30.yaml @@ -1,11 +1,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + LogicalAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) LogicalProject(ResolutionWidth=[$80], $f90=[+(CAST($80):BIGINT, 1)], $f91=[+(CAST($80):BIGINT, 2)], $f92=[+(CAST($80):BIGINT, 3)], $f93=[+(CAST($80):BIGINT, 4)], $f94=[+(CAST($80):BIGINT, 5)], $f95=[+(CAST($80):BIGINT, 6)], $f96=[+(CAST($80):BIGINT, 7)], $f97=[+(CAST($80):BIGINT, 8)], $f98=[+(CAST($80):BIGINT, 9)], $f99=[+(CAST($80):BIGINT, 10)], $f100=[+(CAST($80):BIGINT, 11)], $f101=[+(CAST($80):BIGINT, 12)], $f102=[+(CAST($80):BIGINT, 13)], $f103=[+(CAST($80):BIGINT, 14)], $f104=[+(CAST($80):BIGINT, 15)], $f105=[+(CAST($80):BIGINT, 16)], $f106=[+(CAST($80):BIGINT, 17)], $f107=[+(CAST($80):BIGINT, 18)], $f108=[+(CAST($80):BIGINT, 19)], $f109=[+(CAST($80):BIGINT, 20)], $f110=[+(CAST($80):BIGINT, 21)], $f111=[+(CAST($80):BIGINT, 22)], $f112=[+(CAST($80):BIGINT, 23)], $f113=[+(CAST($80):BIGINT, 24)], $f114=[+(CAST($80):BIGINT, 25)], $f115=[+(CAST($80):BIGINT, 26)], $f116=[+(CAST($80):BIGINT, 27)], $f117=[+(CAST($80):BIGINT, 28)], $f118=[+(CAST($80):BIGINT, 29)], $f119=[+(CAST($80):BIGINT, 30)], $f120=[+(CAST($80):BIGINT, 31)], $f121=[+(CAST($80):BIGINT, 32)], $f122=[+(CAST($80):BIGINT, 33)], $f123=[+(CAST($80):BIGINT, 34)], $f124=[+(CAST($80):BIGINT, 35)], $f125=[+(CAST($80):BIGINT, 36)], $f126=[+(CAST($80):BIGINT, 37)], $f127=[+(CAST($80):BIGINT, 38)], $f128=[+(CAST($80):BIGINT, 39)], $f129=[+(CAST($80):BIGINT, 40)], $f130=[+(CAST($80):BIGINT, 41)], $f131=[+(CAST($80):BIGINT, 42)], $f132=[+(CAST($80):BIGINT, 43)], $f133=[+(CAST($80):BIGINT, 44)], $f134=[+(CAST($80):BIGINT, 45)], $f135=[+(CAST($80):BIGINT, 46)], $f136=[+(CAST($80):BIGINT, 47)], $f137=[+(CAST($80):BIGINT, 48)], $f138=[+(CAST($80):BIGINT, 49)], $f139=[+(CAST($80):BIGINT, 50)], $f140=[+(CAST($80):BIGINT, 51)], $f141=[+(CAST($80):BIGINT, 52)], $f142=[+(CAST($80):BIGINT, 53)], $f143=[+(CAST($80):BIGINT, 54)], $f144=[+(CAST($80):BIGINT, 55)], $f145=[+(CAST($80):BIGINT, 56)], $f146=[+(CAST($80):BIGINT, 57)], $f147=[+(CAST($80):BIGINT, 58)], $f148=[+(CAST($80):BIGINT, 59)], $f149=[+(CAST($80):BIGINT, 60)], $f150=[+(CAST($80):BIGINT, 61)], $f151=[+(CAST($80):BIGINT, 62)], $f152=[+(CAST($80):BIGINT, 63)], $f153=[+(CAST($80):BIGINT, 64)], $f154=[+(CAST($80):BIGINT, 65)], $f155=[+(CAST($80):BIGINT, 66)], $f156=[+(CAST($80):BIGINT, 67)], $f157=[+(CAST($80):BIGINT, 68)], $f158=[+(CAST($80):BIGINT, 69)], $f159=[+(CAST($80):BIGINT, 70)], $f160=[+(CAST($80):BIGINT, 71)], $f161=[+(CAST($80):BIGINT, 72)], $f162=[+(CAST($80):BIGINT, 73)], $f163=[+(CAST($80):BIGINT, 74)], $f164=[+(CAST($80):BIGINT, 75)], $f165=[+(CAST($80):BIGINT, 76)], $f166=[+(CAST($80):BIGINT, 77)], $f167=[+(CAST($80):BIGINT, 78)], $f168=[+(CAST($80):BIGINT, 79)], $f169=[+(CAST($80):BIGINT, 80)], $f170=[+(CAST($80):BIGINT, 81)], $f171=[+(CAST($80):BIGINT, 82)], $f172=[+(CAST($80):BIGINT, 83)], $f173=[+(CAST($80):BIGINT, 84)], $f174=[+(CAST($80):BIGINT, 85)], $f175=[+(CAST($80):BIGINT, 86)], $f176=[+(CAST($80):BIGINT, 87)], $f177=[+(CAST($80):BIGINT, 88)], $f178=[+(CAST($80):BIGINT, 89)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[SUM($0)], sum(ResolutionWidth+1)=[SUM($1)], sum(ResolutionWidth+2)=[SUM($2)], sum(ResolutionWidth+3)=[SUM($3)], sum(ResolutionWidth+4)=[SUM($4)], sum(ResolutionWidth+5)=[SUM($5)], sum(ResolutionWidth+6)=[SUM($6)], sum(ResolutionWidth+7)=[SUM($7)], sum(ResolutionWidth+8)=[SUM($8)], sum(ResolutionWidth+9)=[SUM($9)], sum(ResolutionWidth+10)=[SUM($10)], sum(ResolutionWidth+11)=[SUM($11)], sum(ResolutionWidth+12)=[SUM($12)], sum(ResolutionWidth+13)=[SUM($13)], sum(ResolutionWidth+14)=[SUM($14)], sum(ResolutionWidth+15)=[SUM($15)], sum(ResolutionWidth+16)=[SUM($16)], sum(ResolutionWidth+17)=[SUM($17)], sum(ResolutionWidth+18)=[SUM($18)], sum(ResolutionWidth+19)=[SUM($19)], sum(ResolutionWidth+20)=[SUM($20)], sum(ResolutionWidth+21)=[SUM($21)], sum(ResolutionWidth+22)=[SUM($22)], sum(ResolutionWidth+23)=[SUM($23)], sum(ResolutionWidth+24)=[SUM($24)], sum(ResolutionWidth+25)=[SUM($25)], sum(ResolutionWidth+26)=[SUM($26)], sum(ResolutionWidth+27)=[SUM($27)], sum(ResolutionWidth+28)=[SUM($28)], sum(ResolutionWidth+29)=[SUM($29)], sum(ResolutionWidth+30)=[SUM($30)], sum(ResolutionWidth+31)=[SUM($31)], sum(ResolutionWidth+32)=[SUM($32)], sum(ResolutionWidth+33)=[SUM($33)], sum(ResolutionWidth+34)=[SUM($34)], sum(ResolutionWidth+35)=[SUM($35)], sum(ResolutionWidth+36)=[SUM($36)], sum(ResolutionWidth+37)=[SUM($37)], sum(ResolutionWidth+38)=[SUM($38)], sum(ResolutionWidth+39)=[SUM($39)], sum(ResolutionWidth+40)=[SUM($40)], sum(ResolutionWidth+41)=[SUM($41)], sum(ResolutionWidth+42)=[SUM($42)], sum(ResolutionWidth+43)=[SUM($43)], sum(ResolutionWidth+44)=[SUM($44)], sum(ResolutionWidth+45)=[SUM($45)], sum(ResolutionWidth+46)=[SUM($46)], sum(ResolutionWidth+47)=[SUM($47)], sum(ResolutionWidth+48)=[SUM($48)], sum(ResolutionWidth+49)=[SUM($49)], sum(ResolutionWidth+50)=[SUM($50)], sum(ResolutionWidth+51)=[SUM($51)], sum(ResolutionWidth+52)=[SUM($52)], sum(ResolutionWidth+53)=[SUM($53)], sum(ResolutionWidth+54)=[SUM($54)], sum(ResolutionWidth+55)=[SUM($55)], sum(ResolutionWidth+56)=[SUM($56)], sum(ResolutionWidth+57)=[SUM($57)], sum(ResolutionWidth+58)=[SUM($58)], sum(ResolutionWidth+59)=[SUM($59)], sum(ResolutionWidth+60)=[SUM($60)], sum(ResolutionWidth+61)=[SUM($61)], sum(ResolutionWidth+62)=[SUM($62)], sum(ResolutionWidth+63)=[SUM($63)], sum(ResolutionWidth+64)=[SUM($64)], sum(ResolutionWidth+65)=[SUM($65)], sum(ResolutionWidth+66)=[SUM($66)], sum(ResolutionWidth+67)=[SUM($67)], sum(ResolutionWidth+68)=[SUM($68)], sum(ResolutionWidth+69)=[SUM($69)], sum(ResolutionWidth+70)=[SUM($70)], sum(ResolutionWidth+71)=[SUM($71)], sum(ResolutionWidth+72)=[SUM($72)], sum(ResolutionWidth+73)=[SUM($73)], sum(ResolutionWidth+74)=[SUM($74)], sum(ResolutionWidth+75)=[SUM($75)], sum(ResolutionWidth+76)=[SUM($76)], sum(ResolutionWidth+77)=[SUM($77)], sum(ResolutionWidth+78)=[SUM($78)], sum(ResolutionWidth+79)=[SUM($79)], sum(ResolutionWidth+80)=[SUM($80)], sum(ResolutionWidth+81)=[SUM($81)], sum(ResolutionWidth+82)=[SUM($82)], sum(ResolutionWidth+83)=[SUM($83)], sum(ResolutionWidth+84)=[SUM($84)], sum(ResolutionWidth+85)=[SUM($85)], sum(ResolutionWidth+86)=[SUM($86)], sum(ResolutionWidth+87)=[SUM($87)], sum(ResolutionWidth+88)=[SUM($88)], sum(ResolutionWidth+89)=[SUM($89)]) + EnumerableAggregate(group=[{}], sum(ResolutionWidth)=[CHECKED_LONG_SUM($0)], sum(ResolutionWidth+1)=[CHECKED_LONG_SUM($1)], sum(ResolutionWidth+2)=[CHECKED_LONG_SUM($2)], sum(ResolutionWidth+3)=[CHECKED_LONG_SUM($3)], sum(ResolutionWidth+4)=[CHECKED_LONG_SUM($4)], sum(ResolutionWidth+5)=[CHECKED_LONG_SUM($5)], sum(ResolutionWidth+6)=[CHECKED_LONG_SUM($6)], sum(ResolutionWidth+7)=[CHECKED_LONG_SUM($7)], sum(ResolutionWidth+8)=[CHECKED_LONG_SUM($8)], sum(ResolutionWidth+9)=[CHECKED_LONG_SUM($9)], sum(ResolutionWidth+10)=[CHECKED_LONG_SUM($10)], sum(ResolutionWidth+11)=[CHECKED_LONG_SUM($11)], sum(ResolutionWidth+12)=[CHECKED_LONG_SUM($12)], sum(ResolutionWidth+13)=[CHECKED_LONG_SUM($13)], sum(ResolutionWidth+14)=[CHECKED_LONG_SUM($14)], sum(ResolutionWidth+15)=[CHECKED_LONG_SUM($15)], sum(ResolutionWidth+16)=[CHECKED_LONG_SUM($16)], sum(ResolutionWidth+17)=[CHECKED_LONG_SUM($17)], sum(ResolutionWidth+18)=[CHECKED_LONG_SUM($18)], sum(ResolutionWidth+19)=[CHECKED_LONG_SUM($19)], sum(ResolutionWidth+20)=[CHECKED_LONG_SUM($20)], sum(ResolutionWidth+21)=[CHECKED_LONG_SUM($21)], sum(ResolutionWidth+22)=[CHECKED_LONG_SUM($22)], sum(ResolutionWidth+23)=[CHECKED_LONG_SUM($23)], sum(ResolutionWidth+24)=[CHECKED_LONG_SUM($24)], sum(ResolutionWidth+25)=[CHECKED_LONG_SUM($25)], sum(ResolutionWidth+26)=[CHECKED_LONG_SUM($26)], sum(ResolutionWidth+27)=[CHECKED_LONG_SUM($27)], sum(ResolutionWidth+28)=[CHECKED_LONG_SUM($28)], sum(ResolutionWidth+29)=[CHECKED_LONG_SUM($29)], sum(ResolutionWidth+30)=[CHECKED_LONG_SUM($30)], sum(ResolutionWidth+31)=[CHECKED_LONG_SUM($31)], sum(ResolutionWidth+32)=[CHECKED_LONG_SUM($32)], sum(ResolutionWidth+33)=[CHECKED_LONG_SUM($33)], sum(ResolutionWidth+34)=[CHECKED_LONG_SUM($34)], sum(ResolutionWidth+35)=[CHECKED_LONG_SUM($35)], sum(ResolutionWidth+36)=[CHECKED_LONG_SUM($36)], sum(ResolutionWidth+37)=[CHECKED_LONG_SUM($37)], sum(ResolutionWidth+38)=[CHECKED_LONG_SUM($38)], sum(ResolutionWidth+39)=[CHECKED_LONG_SUM($39)], sum(ResolutionWidth+40)=[CHECKED_LONG_SUM($40)], sum(ResolutionWidth+41)=[CHECKED_LONG_SUM($41)], sum(ResolutionWidth+42)=[CHECKED_LONG_SUM($42)], sum(ResolutionWidth+43)=[CHECKED_LONG_SUM($43)], sum(ResolutionWidth+44)=[CHECKED_LONG_SUM($44)], sum(ResolutionWidth+45)=[CHECKED_LONG_SUM($45)], sum(ResolutionWidth+46)=[CHECKED_LONG_SUM($46)], sum(ResolutionWidth+47)=[CHECKED_LONG_SUM($47)], sum(ResolutionWidth+48)=[CHECKED_LONG_SUM($48)], sum(ResolutionWidth+49)=[CHECKED_LONG_SUM($49)], sum(ResolutionWidth+50)=[CHECKED_LONG_SUM($50)], sum(ResolutionWidth+51)=[CHECKED_LONG_SUM($51)], sum(ResolutionWidth+52)=[CHECKED_LONG_SUM($52)], sum(ResolutionWidth+53)=[CHECKED_LONG_SUM($53)], sum(ResolutionWidth+54)=[CHECKED_LONG_SUM($54)], sum(ResolutionWidth+55)=[CHECKED_LONG_SUM($55)], sum(ResolutionWidth+56)=[CHECKED_LONG_SUM($56)], sum(ResolutionWidth+57)=[CHECKED_LONG_SUM($57)], sum(ResolutionWidth+58)=[CHECKED_LONG_SUM($58)], sum(ResolutionWidth+59)=[CHECKED_LONG_SUM($59)], sum(ResolutionWidth+60)=[CHECKED_LONG_SUM($60)], sum(ResolutionWidth+61)=[CHECKED_LONG_SUM($61)], sum(ResolutionWidth+62)=[CHECKED_LONG_SUM($62)], sum(ResolutionWidth+63)=[CHECKED_LONG_SUM($63)], sum(ResolutionWidth+64)=[CHECKED_LONG_SUM($64)], sum(ResolutionWidth+65)=[CHECKED_LONG_SUM($65)], sum(ResolutionWidth+66)=[CHECKED_LONG_SUM($66)], sum(ResolutionWidth+67)=[CHECKED_LONG_SUM($67)], sum(ResolutionWidth+68)=[CHECKED_LONG_SUM($68)], sum(ResolutionWidth+69)=[CHECKED_LONG_SUM($69)], sum(ResolutionWidth+70)=[CHECKED_LONG_SUM($70)], sum(ResolutionWidth+71)=[CHECKED_LONG_SUM($71)], sum(ResolutionWidth+72)=[CHECKED_LONG_SUM($72)], sum(ResolutionWidth+73)=[CHECKED_LONG_SUM($73)], sum(ResolutionWidth+74)=[CHECKED_LONG_SUM($74)], sum(ResolutionWidth+75)=[CHECKED_LONG_SUM($75)], sum(ResolutionWidth+76)=[CHECKED_LONG_SUM($76)], sum(ResolutionWidth+77)=[CHECKED_LONG_SUM($77)], sum(ResolutionWidth+78)=[CHECKED_LONG_SUM($78)], sum(ResolutionWidth+79)=[CHECKED_LONG_SUM($79)], sum(ResolutionWidth+80)=[CHECKED_LONG_SUM($80)], sum(ResolutionWidth+81)=[CHECKED_LONG_SUM($81)], sum(ResolutionWidth+82)=[CHECKED_LONG_SUM($82)], sum(ResolutionWidth+83)=[CHECKED_LONG_SUM($83)], sum(ResolutionWidth+84)=[CHECKED_LONG_SUM($84)], sum(ResolutionWidth+85)=[CHECKED_LONG_SUM($85)], sum(ResolutionWidth+86)=[CHECKED_LONG_SUM($86)], sum(ResolutionWidth+87)=[CHECKED_LONG_SUM($87)], sum(ResolutionWidth+88)=[CHECKED_LONG_SUM($88)], sum(ResolutionWidth+89)=[CHECKED_LONG_SUM($89)]) EnumerableCalc(expr#0=[{inputs}], expr#1=[CAST($t0):BIGINT], expr#2=[1:BIGINT], expr#3=[+($t1, $t2)], expr#4=[2:BIGINT], expr#5=[+($t1, $t4)], expr#6=[3:BIGINT], expr#7=[+($t1, $t6)], expr#8=[4:BIGINT], expr#9=[+($t1, $t8)], expr#10=[5:BIGINT], expr#11=[+($t1, $t10)], expr#12=[6:BIGINT], expr#13=[+($t1, $t12)], expr#14=[7:BIGINT], expr#15=[+($t1, $t14)], expr#16=[8:BIGINT], expr#17=[+($t1, $t16)], expr#18=[9:BIGINT], expr#19=[+($t1, $t18)], expr#20=[10:BIGINT], expr#21=[+($t1, $t20)], expr#22=[11:BIGINT], expr#23=[+($t1, $t22)], expr#24=[12:BIGINT], expr#25=[+($t1, $t24)], expr#26=[13:BIGINT], expr#27=[+($t1, $t26)], expr#28=[14:BIGINT], expr#29=[+($t1, $t28)], expr#30=[15:BIGINT], expr#31=[+($t1, $t30)], expr#32=[16:BIGINT], expr#33=[+($t1, $t32)], expr#34=[17:BIGINT], expr#35=[+($t1, $t34)], expr#36=[18:BIGINT], expr#37=[+($t1, $t36)], expr#38=[19:BIGINT], expr#39=[+($t1, $t38)], expr#40=[20:BIGINT], expr#41=[+($t1, $t40)], expr#42=[21:BIGINT], expr#43=[+($t1, $t42)], expr#44=[22:BIGINT], expr#45=[+($t1, $t44)], expr#46=[23:BIGINT], expr#47=[+($t1, $t46)], expr#48=[24:BIGINT], expr#49=[+($t1, $t48)], expr#50=[25:BIGINT], expr#51=[+($t1, $t50)], expr#52=[26:BIGINT], expr#53=[+($t1, $t52)], expr#54=[27:BIGINT], expr#55=[+($t1, $t54)], expr#56=[28:BIGINT], expr#57=[+($t1, $t56)], expr#58=[29:BIGINT], expr#59=[+($t1, $t58)], expr#60=[30:BIGINT], expr#61=[+($t1, $t60)], expr#62=[31:BIGINT], expr#63=[+($t1, $t62)], expr#64=[32:BIGINT], expr#65=[+($t1, $t64)], expr#66=[33:BIGINT], expr#67=[+($t1, $t66)], expr#68=[34:BIGINT], expr#69=[+($t1, $t68)], expr#70=[35:BIGINT], expr#71=[+($t1, $t70)], expr#72=[36:BIGINT], expr#73=[+($t1, $t72)], expr#74=[37:BIGINT], expr#75=[+($t1, $t74)], expr#76=[38:BIGINT], expr#77=[+($t1, $t76)], expr#78=[39:BIGINT], expr#79=[+($t1, $t78)], expr#80=[40:BIGINT], expr#81=[+($t1, $t80)], expr#82=[41:BIGINT], expr#83=[+($t1, $t82)], expr#84=[42:BIGINT], expr#85=[+($t1, $t84)], expr#86=[43:BIGINT], expr#87=[+($t1, $t86)], expr#88=[44:BIGINT], expr#89=[+($t1, $t88)], expr#90=[45:BIGINT], expr#91=[+($t1, $t90)], expr#92=[46:BIGINT], expr#93=[+($t1, $t92)], expr#94=[47:BIGINT], expr#95=[+($t1, $t94)], expr#96=[48:BIGINT], expr#97=[+($t1, $t96)], expr#98=[49:BIGINT], expr#99=[+($t1, $t98)], expr#100=[50:BIGINT], expr#101=[+($t1, $t100)], expr#102=[51:BIGINT], expr#103=[+($t1, $t102)], expr#104=[52:BIGINT], expr#105=[+($t1, $t104)], expr#106=[53:BIGINT], expr#107=[+($t1, $t106)], expr#108=[54:BIGINT], expr#109=[+($t1, $t108)], expr#110=[55:BIGINT], expr#111=[+($t1, $t110)], expr#112=[56:BIGINT], expr#113=[+($t1, $t112)], expr#114=[57:BIGINT], expr#115=[+($t1, $t114)], expr#116=[58:BIGINT], expr#117=[+($t1, $t116)], expr#118=[59:BIGINT], expr#119=[+($t1, $t118)], expr#120=[60:BIGINT], expr#121=[+($t1, $t120)], expr#122=[61:BIGINT], expr#123=[+($t1, $t122)], expr#124=[62:BIGINT], expr#125=[+($t1, $t124)], expr#126=[63:BIGINT], expr#127=[+($t1, $t126)], expr#128=[64:BIGINT], expr#129=[+($t1, $t128)], expr#130=[65:BIGINT], expr#131=[+($t1, $t130)], expr#132=[66:BIGINT], expr#133=[+($t1, $t132)], expr#134=[67:BIGINT], expr#135=[+($t1, $t134)], expr#136=[68:BIGINT], expr#137=[+($t1, $t136)], expr#138=[69:BIGINT], expr#139=[+($t1, $t138)], expr#140=[70:BIGINT], expr#141=[+($t1, $t140)], expr#142=[71:BIGINT], expr#143=[+($t1, $t142)], expr#144=[72:BIGINT], expr#145=[+($t1, $t144)], expr#146=[73:BIGINT], expr#147=[+($t1, $t146)], expr#148=[74:BIGINT], expr#149=[+($t1, $t148)], expr#150=[75:BIGINT], expr#151=[+($t1, $t150)], expr#152=[76:BIGINT], expr#153=[+($t1, $t152)], expr#154=[77:BIGINT], expr#155=[+($t1, $t154)], expr#156=[78:BIGINT], expr#157=[+($t1, $t156)], expr#158=[79:BIGINT], expr#159=[+($t1, $t158)], expr#160=[80:BIGINT], expr#161=[+($t1, $t160)], expr#162=[81:BIGINT], expr#163=[+($t1, $t162)], expr#164=[82:BIGINT], expr#165=[+($t1, $t164)], expr#166=[83:BIGINT], expr#167=[+($t1, $t166)], expr#168=[84:BIGINT], expr#169=[+($t1, $t168)], expr#170=[85:BIGINT], expr#171=[+($t1, $t170)], expr#172=[86:BIGINT], expr#173=[+($t1, $t172)], expr#174=[87:BIGINT], expr#175=[+($t1, $t174)], expr#176=[88:BIGINT], expr#177=[+($t1, $t176)], expr#178=[89:BIGINT], expr#179=[+($t1, $t178)], ResolutionWidth=[$t0], $f90=[$t3], $f91=[$t5], $f92=[$t7], $f93=[$t9], $f94=[$t11], $f95=[$t13], $f96=[$t15], $f97=[$t17], $f98=[$t19], $f99=[$t21], $f100=[$t23], $f101=[$t25], $f102=[$t27], $f103=[$t29], $f104=[$t31], $f105=[$t33], $f106=[$t35], $f107=[$t37], $f108=[$t39], $f109=[$t41], $f110=[$t43], $f111=[$t45], $f112=[$t47], $f113=[$t49], $f114=[$t51], $f115=[$t53], $f116=[$t55], $f117=[$t57], $f118=[$t59], $f119=[$t61], $f120=[$t63], $f121=[$t65], $f122=[$t67], $f123=[$t69], $f124=[$t71], $f125=[$t73], $f126=[$t75], $f127=[$t77], $f128=[$t79], $f129=[$t81], $f130=[$t83], $f131=[$t85], $f132=[$t87], $f133=[$t89], $f134=[$t91], $f135=[$t93], $f136=[$t95], $f137=[$t97], $f138=[$t99], $f139=[$t101], $f140=[$t103], $f141=[$t105], $f142=[$t107], $f143=[$t109], $f144=[$t111], $f145=[$t113], $f146=[$t115], $f147=[$t117], $f148=[$t119], $f149=[$t121], $f150=[$t123], $f151=[$t125], $f152=[$t127], $f153=[$t129], $f154=[$t131], $f155=[$t133], $f156=[$t135], $f157=[$t137], $f158=[$t139], $f159=[$t141], $f160=[$t143], $f161=[$t145], $f162=[$t147], $f163=[$t149], $f164=[$t151], $f165=[$t153], $f166=[$t155], $f167=[$t157], $f168=[$t159], $f169=[$t161], $f170=[$t163], $f171=[$t165], $f172=[$t167], $f173=[$t169], $f174=[$t171], $f175=[$t173], $f176=[$t175], $f177=[$t177], $f178=[$t179]) CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[ResolutionWidth]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["ResolutionWidth"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml index bf40fe857ed..12fc0646da6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q31.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], SearchEngineID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(SearchEngineID=[$65], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($65), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($0, ''), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), SearchEngineID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"SearchEngineID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"SearchEngineID|ClientIP":{"multi_terms":{"terms":[{"field":"SearchEngineID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml index 81236b33d51..9cdd38482bc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q32.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) LogicalFilter(condition=[<>($63, '')]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[FILTER->AND(<>($1, ''), IS NOT NULL($0), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 3},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($2),avg(ResolutionWidth)=AVG($4)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"SearchPhrase","boost":1.0}}],"must_not":[{"term":{"SearchPhrase":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"WatchID","boost":1.0}},{"exists":{"field":"ClientIP","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml index ccda84ba38a..a64c682196a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q33.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10]) LogicalProject(c=[$2], sum(IsRefresh)=[$3], avg(ResolutionWidth)=[$4], WatchID=[$0], ClientIP=[$1]) - LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[SUM($2)], avg(ResolutionWidth)=[AVG($3)]) + LogicalAggregate(group=[{0, 1}], c=[COUNT()], sum(IsRefresh)=[CHECKED_LONG_SUM($2)], avg(ResolutionWidth)=[AVG($3)]) LogicalProject(WatchID=[$41], ClientIP=[$76], IsRefresh=[$72], ResolutionWidth=[$80]) LogicalFilter(condition=[AND(IS NOT NULL($41), IS NOT NULL($76))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),sum(IsRefresh)=CHECKED_LONG_SUM($1),avg(ResolutionWidth)=AVG($3)), PROJECT->[c, sum(IsRefresh), avg(ResolutionWidth), WatchID, ClientIP], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"WatchID|ClientIP":{"multi_terms":{"terms":[{"field":"WatchID"},{"field":"ClientIP"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(IsRefresh)":{"sum":{"field":"IsRefresh"}},"avg(ResolutionWidth)":{"avg":{"field":"ResolutionWidth"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml index 9c41efa9139..48debf64773 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure2.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum=[$1], state=[$0]) - LogicalAggregate(group=[{0}], sum=[SUM($1)]) + LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum=CHECKED_LONG_SUM($0)), PROJECT->[sum, state], SORT_AGG_METRICS->[0 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"sum":"desc"},{"_key":"asc"}]},"aggregations":{"sum":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml index f2105ce0d3c..5dad5e945b4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure4.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], span(age,5)=[$0]) - LogicalAggregate(group=[{1}], sum(balance)=[SUM($0)]) + LogicalAggregate(group=[{1}], sum(balance)=[CHECKED_LONG_SUM($0)]) LogicalProject(balance=[$7], span(age,5)=[SPAN($10, 5, null:NULL)]) LogicalFilter(condition=[IS NOT NULL($10)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},sum(balance)=CHECKED_LONG_SUM($0)), PROJECT->[sum(balance), span(age,5)], SORT_AGG_METRICS->[0 DESC LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"span(age,5)":{"histogram":{"field":"age","interval":5.0,"offset":0.0,"order":[{"sum(balance)":"desc"},{"_key":"asc"}],"keyed":false,"min_doc_count":1},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml index cd0355241fe..4e87192da1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex1.yaml @@ -3,9 +3,9 @@ calcite: LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$1], c=[$2], dc(employer)=[$3], state=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], c=[COUNT()], dc(employer)=[COUNT(DISTINCT $2)]) LogicalProject(state=[$7], balance=[$3], employer=[$6]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={2},sum(balance)=CHECKED_LONG_SUM($0),c=COUNT(),dc(employer)=COUNT(DISTINCT $1)), PROJECT->[sum(balance), c, dc(employer), state], SORT_AGG_METRICS->[1 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"state":{"terms":{"field":"state.keyword","size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"c":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"dc(employer)":{"cardinality":{"field":"employer.keyword"}},"c":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 59cd137ca59..3e3a45b6386 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$2], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$2], dir0=[DESC-nulls-last]) LogicalProject(sum(balance)=[$2], count()=[$3], d=[$4], gender=[$0], new_state=[$1]) - LogicalAggregate(group=[{0, 1}], sum(balance)=[SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) + LogicalAggregate(group=[{0, 1}], sum(balance)=[CHECKED_LONG_SUM($2)], count()=[COUNT()], d=[COUNT(DISTINCT $3)]) LogicalProject(gender=[$4], new_state=[$17], balance=[$3], employer=[$6]) LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($17))]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum(balance)=CHECKED_LONG_SUM($2),count()=COUNT(),d=COUNT(DISTINCT $3)), PROJECT->[sum(balance), count(), d, gender, new_state], SORT_AGG_METRICS->[2 DESC LAST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml index 68cb12a49dd..c4c572ce9e5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_buckets_not_pushed.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], dir0=[ASC-nulls-first]) LogicalProject(c=[$2], s=[$3], span(age,5)=[$1], state=[$0]) - LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0, 2}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3], span(age,5)=[SPAN($8, 5, null:NULL)]) LogicalFilter(condition=[AND(IS NOT NULL($8), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},c=COUNT(),s=CHECKED_LONG_SUM($1)), PROJECT->[c, s, span(age,5), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}},{"span(age,5)":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":5.0}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index bc65d5c4c29..a1d7c439d9e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum_SUM=CHECKED_LONG_SUM($1),sum_COUNT=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum_SUM":{"sum":{"field":"balance"}},"sum_COUNT":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index 1d664d5cd43..f6402d44d63 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -2,9 +2,9 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=CHECKED_LONG_SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=CHECKED_LONG_SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json index 064aa294a2d..f265a37f292 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml index 3767a38b3a9..2fb92e7c652 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push1.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$2], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml index 520f729b7f9..b5192ed01ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multiple_agg_with_sort_on_one_measure_not_push2.yaml @@ -3,10 +3,10 @@ calcite: LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first]) LogicalProject(c=[$1], s=[$2], state=[$0]) - LogicalAggregate(group=[{0}], c=[COUNT()], s=[SUM($1)]) + LogicalAggregate(group=[{0}], c=[COUNT()], s=[CHECKED_LONG_SUM($1)]) LogicalProject(state=[$7], balance=[$3]) LogicalFilter(condition=[IS NOT NULL($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], fetch=[10000]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},c=COUNT(),s=CHECKED_LONG_SUM($0)), PROJECT->[c, s, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"s":{"sum":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml index 0478b24369c..9955ef801c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml index a1cf6ae00e9..902af30e1d7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_global_null_bucket.yaml @@ -10,9 +10,9 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) CalciteEnumerableTopK(sort0=[$17], dir0=[ASC], fetch=[10000]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml index 324960f28dd..d0f762c426d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset.yaml @@ -22,17 +22,16 @@ calcite: EnumerableCalc(expr#0..11=[{inputs}], expr#12=[34], expr#13=[>($t8, $t12)], expr#14=[1], expr#15=[0], expr#16=[CASE($t13, $t14, $t15)], expr#17=[25], expr#18=[<($t8, $t17)], expr#19=[CASE($t18, $t14, $t15)], expr#20=[IS NULL($t4)], proj#0..11=[{exprs}], __reset_before_flag__=[$t16], __reset_after_flag__=[$t19], $14=[$t20]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], expr#11=[IS NULL($t0)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10], $4=[$t11]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml index 42b50e7eb5f..213eef91aa2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_streamstats_reset_null_bucket.yaml @@ -24,17 +24,16 @@ calcite: EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], sort1=[$1], sort2=[$2], dir0=[ASC], dir1=[ASC], dir2=[ASC]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], gender=[$t0], __stream_seq__=[$t2], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[34], expr#4=[>($t1, $t3)], expr#5=[1], expr#6=[0], expr#7=[CASE($t4, $t5, $t6)], expr#8=[25], expr#9=[<($t1, $t8)], expr#10=[CASE($t9, $t5, $t6)], proj#0..2=[{exprs}], __reset_before_flag__=[$t7], __reset_after_flag__=[$t10]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[gender, age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml index 059caa2e2d2..79252ca2148 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_case_composite_cannot_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], state=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], state=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=[CASE($t20, $t21, $t11)], age_range=[$t22], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml index 43e27cd2d5d..b3ec99de2cf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t4, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t3)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t4)], avg(balance)=[$t11], count()=[$t5], age_range=[$t0], state=[$t1], gender=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)], count()=[COUNT()]) + EnumerableCalc(expr#0..4=[{inputs}], avg(balance)=[$t3], count()=[$t4], age_range=[$t0], state=[$t1], gender=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg(balance)=[AVG($3)], count()=[COUNT()]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], age_range=[$t23], state=[$t9], gender=[$t4], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml index 6dfa7cd65a3..e49dee2d350 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite2_range_range_count_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], avg_balance=[$t10], age_range=[$t0], balance_range=[$t1], state=[$t2]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)]) + EnumerableCalc(expr#0..3=[{inputs}], avg_balance=[$t3], age_range=[$t0], balance_range=[$t1], state=[$t2]) + EnumerableAggregate(group=[{0, 1, 2}], avg_balance=[AVG($3)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[35], expr#20=[<($t10, $t19)], expr#21=['u35':VARCHAR], expr#22=['a35':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], expr#24=[20000], expr#25=[<($t7, $t24)], expr#26=['medium':VARCHAR], expr#27=['high':VARCHAR], expr#28=[CASE($t25, $t26, $t27)], age_range=[$t23], balance_range=[$t28], state=[$t9], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml index 41ed8ba61fc..1dd00811008 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_composite_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg(balance)=[$t9], state=[$t0], age_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg(balance)=[$t2], state=[$t0], age_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg(balance)=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=['a30':VARCHAR], expr#23=[CASE($t20, $t21, $t22)], state=[$t9], age_range=[$t23], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml index 67ad0f0fd07..1ae9205fa10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_count_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(age)=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[30..40)]], expr#23=[SEARCH($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml index 10ead7ad449..14664cb5df6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_complex_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], age_range=[$t0]) - EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], age_range=[$t0]) + EnumerableAggregate(group=[{0}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[Sarg[[35..40), [80..+∞)]], expr#23=[SEARCH($t10, $t22)], expr#24=['30-40 or >=80':VARCHAR], expr#25=[null:NULL], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], balance=[$t7]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml index a81e208bdbf..6a33abfd5df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_metric_push.yaml @@ -10,4 +10,4 @@ calcite: EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg_age=[$t8], age_range=[$t0]) EnumerableAggregate(group=[{0}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], age_range=[$t26], age=[$t10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml index 404726f6083..1aee0d0ced4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/agg_range_range_metric_push.yaml @@ -7,7 +7,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_balance=[$t9], age_range=[$t0], balance_range=[$t1]) - EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_balance=[$t2], age_range=[$t0], balance_range=[$t1]) + EnumerableAggregate(group=[{0, 1}], avg_balance=[AVG($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[30], expr#20=[<($t10, $t19)], expr#21=['u30':VARCHAR], expr#22=[40], expr#23=[<($t10, $t22)], expr#24=['u40':VARCHAR], expr#25=['u100':VARCHAR], expr#26=[CASE($t20, $t21, $t23, $t24, $t25)], expr#27=[20000], expr#28=[<($t7, $t27)], expr#29=['medium':VARCHAR], expr#30=['high':VARCHAR], expr#31=[CASE($t28, $t29, $t30)], age_range=[$t26], balance_range=[$t31], balance=[$t7]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml index fe925e0a80a..1d1b9bfec33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_multiple_group_keys.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{4, 10}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 10}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[SAFE_CAST($t10)], expr#22=[IS NOT NULL($t21)], expr#23=[AND($t19, $t20, $t22)], proj#0..18=[{exprs}], $condition=[$t23]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml index beb3275a6c6..1876916cb25 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_null_str.yaml @@ -26,15 +26,15 @@ calcite: EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['nil'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], gender=[$t0], age=[$t10], avg(balance)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], gender=[$t0], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[AND($t16, $t17)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[SAFE_CAST($t1)], expr#5=[0], expr#6=[=($t3, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t2)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t3)], age=[$t4], avg(balance)=[$t10]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], age=[$t3], avg(balance)=[$t2]) + EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) EnumerableCalc(expr#0..12=[{inputs}], expr#13=[10], expr#14=[null:NULL], expr#15=[SPAN($t5, $t13, $t14)], expr#16=[IS NOT NULL($t4)], expr#17=[IS NOT NULL($t3)], expr#18=[SAFE_CAST($t15)], expr#19=[IS NOT NULL($t18)], expr#20=[AND($t16, $t17, $t19)], gender=[$t4], balance=[$t3], age0=[$t15], $condition=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml index 8224f075819..d6fd5118aa5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_single_group_key.yaml @@ -9,7 +9,6 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], gender=[$t0], avg(balance)=[$t8]) - EnumerableAggregate(group=[{4}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) - EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableAggregate(group=[{4}], avg(balance)=[AVG($7)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t4)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml index 16aa3871687..52fb3848a8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/chart_with_limit.yaml @@ -9,7 +9,7 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$0], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], state=[$t1], gender=[$t0], avg(balance)=[$t9]) - EnumerableAggregate(group=[{4, 9}], agg#0=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], state=[$t1], gender=[$t0], avg(balance)=[$t2]) + EnumerableAggregate(group=[{4, 9}], avg(balance)=[AVG($7)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t9)], expr#20=[IS NOT NULL($t7)], expr#21=[AND($t19, $t20)], proj#0..18=[{exprs}], $condition=[$t21]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml index 1db12fc013f..a5d14f85dcf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.yaml @@ -2,11 +2,11 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum=[$2], len=[$0], gender=[$1]) - LogicalAggregate(group=[{0, 1}], sum=[SUM($2)]) + LogicalAggregate(group=[{0, 1}], sum=[CHECKED_LONG_SUM($2)]) LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[100:BIGINT], expr#8=[*($t2, $t7)], expr#9=[+($t6, $t8)], expr#10=[CHAR_LENGTH($t0)], sum=[$t9], len=[$t10], gender=[$t0]) - EnumerableAggregate(group=[{4}], sum_SUM=[$SUM0($7)], agg#1=[COUNT($7)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100:BIGINT], expr#4=[*($t2, $t3)], expr#5=[+($t1, $t4)], expr#6=[CHAR_LENGTH($t0)], sum=[$t5], len=[$t6], gender=[$t0]) + EnumerableAggregate(group=[{4}], sum_SUM=[CHECKED_LONG_SUM($7)], sum_COUNT=[COUNT($7)]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml index 655e16839ed..0a06b733276 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -2,12 +2,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) - LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)=[CHECKED_LONG_SUM($2)], sum(balance - 100)=[CHECKED_LONG_SUM($3)], sum(balance * 100)=[CHECKED_LONG_SUM($4)], sum(balance / 100)=[CHECKED_LONG_SUM($5)]) LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100:BIGINT], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) + EnumerableAggregate(group=[{0}], sum(balance)=[CHECKED_LONG_SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[CHECKED_LONG_SUM($2)]) EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json index a31d2acfc61..0b9e873d52c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_minspan.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[MINSPAN_BUCKET($8, 5.0E0:DOUBLE, -(MAX($8) OVER (), MIN($8) OVER ()), MAX($8) OVER ())])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[5.0E0:DOUBLE], expr#14=[-($t11, $t12)], expr#15=[MINSPAN_BUCKET($t8, $t13, $t14, $t11)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t15])\n EnumerableLimit(fetch=[5])\n EnumerableWindow(window#0=[window(aggs [MAX($8), MIN($8)])])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml index ac3728eacb9..a283b195883 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -8,7 +8,7 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml index f781995261c..5acfa39c392 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -19,7 +19,7 @@ calcite: EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) EnumerableWindow(window#0=[window(partition {1} order by [1 ASC-nulls-first] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t11)], state=[$t1], age2=[$t11], $condition=[$t12]) - EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t1], age2=[$t4], $condition=[$t5]) + EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json index 028a56cb020..126f559f4d0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], avg_age=[AVG($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json index 9d64b554b18..1be834a39ba 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[0], expr#4=[=($t2, $t3)], expr#5=[null:BIGINT], expr#6=[CASE($t4, $t5, $t1)], expr#7=[CAST($t6):DOUBLE], expr#8=[/($t7, $t2)], avg(balance)=[$t8], state=[$t0])\n EnumerableAggregate(group=[{7}], agg#0=[$SUM0($3)], agg#1=[COUNT($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], state=[$t0])\n EnumerableAggregate(group=[{7}], avg(balance)=[AVG($3)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml index 0bf9a2c50ce..f901475ef8d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableHashJoin(condition=[AND(IS NOT DISTINCT FROM($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml index d72bf7b429f..2816bf68f20 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_global_null_bucket.yaml @@ -10,10 +10,10 @@ calcite: LogicalProject(__r_seq__=[ROW_NUMBER() OVER ()], __r_gender__=[$4], __r_age__=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..19=[{inputs}], expr#20=[0], expr#21=[=($t19, $t20)], expr#22=[null:BIGINT], expr#23=[CASE($t21, $t22, $t18)], expr#24=[CAST($t23):DOUBLE], expr#25=[/($t24, $t19)], proj#0..10=[{exprs}], avg_age=[$t25]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..10=[{exprs}], avg_age=[$t18]) EnumerableLimit(fetch=[10000]) EnumerableSort(sort0=[$17], dir0=[ASC]) - EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], agg#0=[$SUM0($20)], agg#1=[COUNT($20)]) + EnumerableAggregate(group=[{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}], avg_age=[AVG($20)]) EnumerableMergeJoin(condition=[AND(=($4, $19), >=($18, -($17, 1)), <=($18, $17))], joinType=[left]) EnumerableSort(sort0=[$4], dir0=[ASC]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml index 3ec98ba9382..b131d24ba2c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $14=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[=($t5, $t6)], expr#8=[null:BIGINT], expr#9=[CASE($t7, $t8, $t4)], expr#10=[CAST($t9):DOUBLE], expr#11=[/($t10, $t5)], proj#0..3=[{exprs}], avg_age=[$t11]) - EnumerableAggregate(group=[{0, 1, 2, 3}], agg#0=[$SUM0($5)], agg#1=[COUNT($5)]) - EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2, 3}]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2, 3}], avg_age=[AVG($5)]) + EnumerableHashJoin(condition=[AND(=($2, $7), <($6, $1), OR(=($4, $0), AND(IS NULL($4), $3)))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2, 3}]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..1=[{exprs}], __seg_id__=[$t9], $f16=[$t4]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], expr#26=[IS NULL($t4)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25], $4=[$t26]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml index 40fb4087001..e0ceaba3192 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_streamstats_reset_null_bucket.yaml @@ -23,17 +23,16 @@ calcite: EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], proj#0..10=[{exprs}], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..4=[{inputs}], expr#5=[0], expr#6=[=($t4, $t5)], expr#7=[null:BIGINT], expr#8=[CASE($t6, $t7, $t3)], expr#9=[CAST($t8):DOUBLE], expr#10=[/($t9, $t4)], proj#0..2=[{exprs}], avg_age=[$t10]) - EnumerableAggregate(group=[{0, 1, 2}], agg#0=[$SUM0($4)], agg#1=[COUNT($4)]) - EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) - EnumerableAggregate(group=[{0, 1, 2}]) - EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) - EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) - EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableAggregate(group=[{0, 1, 2}], avg_age=[AVG($4)]) + EnumerableHashJoin(condition=[AND(=($2, $6), =($0, $3), <($5, $1))], joinType=[inner]) + EnumerableAggregate(group=[{0, 1, 2}]) + EnumerableCalc(expr#0..5=[{inputs}], expr#6=[0], expr#7=[COALESCE($t5, $t6)], expr#8=[+($t4, $t7)], proj#0..1=[{exprs}], __seg_id__=[$t8]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($2)])], window#1=[window(rows between UNBOUNDED PRECEDING and $4 PRECEDING aggs [$SUM0($3)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + EnumerableCalc(expr#0..6=[{inputs}], expr#7=[0], expr#8=[COALESCE($t6, $t7)], expr#9=[+($t5, $t8)], proj#0..2=[{exprs}], __seg_id__=[$t9]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [$SUM0($3)])], window#1=[window(rows between UNBOUNDED PRECEDING and $5 PRECEDING aggs [$SUM0($4)])], constants=[[1]]) + EnumerableCalc(expr#0..17=[{inputs}], expr#18=[34], expr#19=[>($t8, $t18)], expr#20=[1], expr#21=[0], expr#22=[CASE($t19, $t20, $t21)], expr#23=[25], expr#24=[<($t8, $t23)], expr#25=[CASE($t24, $t20, $t21)], gender=[$t4], age=[$t8], __stream_seq__=[$t17], __reset_before_flag__=[$t22], __reset_after_flag__=[$t25]) + EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml new file mode 100644 index 00000000000..d638ed4495f --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5164_agg.yml @@ -0,0 +1,230 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5164 +# SUM/AVG over a BIGINT (long) column near 2^63 must not silently wrap to a negative value. +# +# The enumerable SUM accumulator for a long argument is a plain long, so a running sum past 2^63 +# wraps to a negative result (e.g. SUM(long) returned -5594372458244005145). AVG reduces to +# SUM(field)/COUNT(field), so its intermediate long SUM wraps the same way (AVG returned a negative +# average). SUM(long) now uses checked BIGINT accumulation with Math.addExact in the Calcite +# fallback. The pushdown path uses OpenSearch's native double-based sum and checks its final value +# before narrowing to BIGINT. AVG(long) is averaged in DOUBLE, which holds the true average without +# wrapping. + +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: test_agg_overflow + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_overflow + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - '{"index": {}}' + - '{"long_field": 9223372036854775807}' + - do: + indices.create: + index: test_agg_in_range + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_in_range + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 1000000000000}' + - '{"index": {}}' + - '{"long_field": 2000000000000}' + - '{"index": {}}' + - '{"long_field": 3000000000000}' + - do: + indices.create: + index: test_agg_exact + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + bulk: + index: test_agg_exact + refresh: true + body: + - '{"index": {}}' + - '{"long_field": 4611686018427387904}' + - '{"index": {}}' + - '{"long_field": 1}' + - do: + indices.create: + index: test_agg_reduce_left + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_left + refresh: true + body: + long_field: 9223372036854775807 + - do: + indices.create: + index: test_agg_reduce_right + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + long_field: + type: long + - do: + index: + index: test_agg_reduce_right + refresh: true + body: + long_field: 4096 + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + - do: + indices.delete: + index: test_agg_overflow + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_in_range + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_exact + ignore_unavailable: true + - do: + indices.delete: + index: test_agg_reduce_* + ignore_unavailable: true + +--- +"SUM of large longs overflowing BIGINT throws error": + - skip: + features: + - headers + # 3 * (2^63 - 1) far exceeds the BIGINT range; historically this wrapped to a negative value. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"Pushed SUM whose final value exceeds BIGINT throws error": + - skip: + features: + - headers + # Each one-shard index has an in-range partial. The final result is far enough beyond 2^63 to be + # distinguishable from Long.MAX_VALUE after native double accumulation. + - do: + catch: bad_request + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_reduce_* | stats sum(long_field) + - match: { "$body": "/[Oo]verflow/" } + +--- +"AVG of large longs does not overflow or error": + - skip: + features: + - headers + # The average of three identical values is that value; averaging in DOUBLE avoids the wrap that + # a long intermediate SUM would cause. 9223372036854775807 is returned as a double (9.223...E18). + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_overflow | stats avg(long_field) + - match: { total: 1 } + - match: { datarows: [[9.223372036854776e18]] } + +--- +"SUM of longs within BIGINT range does not error": + - skip: + features: + - headers + # 1e12 + 2e12 + 3e12 = 6e12, well within long range; must return the exact sum, no error. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_in_range | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[6000000000000]] } + +--- +"Pushed SUM of large in-range longs uses native double precision": + - skip: + features: + - headers + # 2^62 + 1 fits in BIGINT but cannot be represented exactly by a double accumulator. + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387904]] } + +--- +"Fallback SUM of large in-range longs retains low-order precision": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test_agg_exact | head 2 | stats sum(long_field) + - match: { total: 1 } + - match: { datarows: [[4611686018427387905]] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 775b0278683..5a336a2e500 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -89,11 +89,13 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.ScriptQueryExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -477,6 +479,10 @@ private static Pair createRegularAggregation( AggregateBuilderHelper helper, List dedupSortKeys) { + if (aggCall.getAggregation() == PPLBuiltinOperators.CHECKED_LONG_SUM) { + return createCheckedLongSumAggregation(args, aggName, helper); + } + return switch (aggCall.getAggregation().kind) { case AVG -> Pair.of( @@ -651,6 +657,17 @@ yield switch (functionName) { }; } + private static Pair createCheckedLongSumAggregation( + List> args, String aggName, AggregateBuilderHelper helper) { + if (args.size() != 1) { + throw new AggregateAnalyzerException("CHECKED_LONG_SUM requires exactly one argument"); + } + + return Pair.of( + helper.build(args.getFirst().getKey(), AggregationBuilders.sum(aggName)), + new CheckedLongSumParser(aggName)); + } + private static boolean supportsMaxMinAggregation(ExprType fieldType) { ExprType coreType = (fieldType instanceof OpenSearchDataType) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java new file mode 100644 index 00000000000..7ea18298b13 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParser.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +/** + * Narrows OpenSearch's double-based native sum to the BIGINT type declared by CHECKED_LONG_SUM. + * + *

OpenSearch may already have lost low-order precision before this parser receives the result. A + * double at the positive BIGINT boundary is also ambiguous because {@code Long.MAX_VALUE} rounds to + * {@code 2^63}; Java's narrowing conversion saturates that value to {@code Long.MAX_VALUE}. + */ +@EqualsAndHashCode +@RequiredArgsConstructor +public class CheckedLongSumParser implements MetricParser { + + private static final double TWO_POW_63 = 0x1p63; + + @Getter private final String name; + + @Override + public List> parse(Aggregation aggregation) { + double value = ((NumericMetricsAggregation.SingleValue) aggregation).value(); + Long narrowed = Double.isNaN(value) ? null : narrow(value); + return Collections.singletonList( + new HashMap<>(Collections.singletonMap(aggregation.getName(), narrowed))); + } + + static long narrow(double value) { + if (!Double.isFinite(value) || value > TWO_POW_63 || value < -TWO_POW_63) { + throw new ArithmeticException("BIGINT overflow in SUM"); + } + return (long) value; + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 4779332abac..0995a60adeb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -41,12 +41,14 @@ import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.request.AggregateAnalyzer.ExpressionNotAnalyzableException; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CheckedLongSumParser; import org.opensearch.sql.opensearch.response.agg.FilterParser; import org.opensearch.sql.opensearch.response.agg.MetricParserHelper; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; @@ -176,6 +178,64 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { }); } + @Test + void analyze_checkedLongSum() throws ExpressionNotAnalyzableException { + AggregateCall checkedLongSumCall = + AggregateCall.create( + PPLBuiltinOperators.CHECKED_LONG_SUM, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(0), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "checked_sum"); + Aggregate aggregate = createMockAggregate(List.of(checkedLongSumCall), ImmutableBitSet.of()); + Project project = createMockProject(List.of(0)); + AggregateAnalyzer.AggregateBuilderHelper helper = + new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); + + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, project, List.of("checked_sum"), helper); + + SumAggregationBuilder builder = + assertInstanceOf(SumAggregationBuilder.class, result.getLeft().getFirst()); + assertEquals("a", builder.field()); + NoBucketAggregationParser parser = + assertInstanceOf(NoBucketAggregationParser.class, result.getRight()); + assertInstanceOf( + CheckedLongSumParser.class, + parser.getMetricsParser().getMetricParserMap().get("checked_sum")); + } + + @Test + void analyze_checkedLongSumExpressionUsesNativeScriptedSum() + throws ExpressionNotAnalyzableException { + buildAggregation("checked_sum") + .withAggCall( + b -> + b.aggregateCall( + PPLBuiltinOperators.CHECKED_LONG_SUM, + b.call(SqlStdOperatorTable.PLUS, b.field("a"), b.literal(1))) + .as("checked_sum")) + .expectDslTemplate("[{\"checked_sum\":{\"sum\":{\"script\":*}}}]") + .expectResponseParser( + new MetricParserHelper(List.of(new CheckedLongSumParser("checked_sum")))) + .verify(); + } + + @Test + void analyze_bigintAvgUsesNativeField() throws ExpressionNotAnalyzableException { + buildAggregation("avg") + .withAggCall(b -> b.aggregateCall(PPLBuiltinOperators.BIGINT_AVG, b.field("a")).as("avg")) + .expectDslQuery("[{\"avg\":{\"avg\":{\"field\":\"a\"}}}]") + .expectResponseParser(new MetricParserHelper(List.of(new SingleValueParser("avg")))) + .verify(); + } + @Test void analyze_aggCall_extended() throws ExpressionNotAnalyzableException { AggregateCall varSampCall = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java new file mode 100644 index 00000000000..8a5c940ada6 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/agg/CheckedLongSumParserTest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.search.aggregations.metrics.NumericMetricsAggregation; + +class CheckedLongSumParserTest { + + private final CheckedLongSumParser parser = new CheckedLongSumParser("sum"); + + @Test + void narrowsNativeSumToLong() { + assertEquals(42L, value(parser.parse(aggregation(42d)))); + } + + @Test + void preservesNativeDoublePrecisionBehavior() { + double rounded = (double) ((1L << 62) + 1L); + assertEquals(1L << 62, value(parser.parse(aggregation(rounded)))); + } + + @Test + void saturatesAmbiguousPositiveBoundary() { + assertEquals(Long.MAX_VALUE, value(parser.parse(aggregation((double) Long.MAX_VALUE)))); + } + + @Test + void rejectsValueClearlyOutsideLongRange() { + assertThrows(ArithmeticException.class, () -> parser.parse(aggregation(Math.nextUp(0x1p63)))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Math.nextDown(-0x1p63)))); + } + + @Test + void rejectsInfiniteValue() { + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.POSITIVE_INFINITY))); + assertThrows( + ArithmeticException.class, () -> parser.parse(aggregation(Double.NEGATIVE_INFINITY))); + } + + @Test + void convertsNanToNull() { + assertNull(value(parser.parse(aggregation(Double.NaN)))); + } + + private static NumericMetricsAggregation.SingleValue aggregation(double value) { + NumericMetricsAggregation.SingleValue aggregation = + mock(NumericMetricsAggregation.SingleValue.class); + when(aggregation.getName()).thenReturn("sum"); + when(aggregation.value()).thenReturn(value); + return aggregation; + } + + private static Object value(List> rows) { + return rows.getFirst().get("sum"); + } +} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java index 2d044da58e6..c25e4655bf8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/calcite/OpenSearchSparkSqlDialect.java @@ -25,7 +25,8 @@ public class OpenSearchSparkSqlDialect extends SparkSqlDialect { ImmutableMap.of( "ARG_MIN", "MIN_BY", "ARG_MAX", "MAX_BY", - "SAFE_CAST", "TRY_CAST"); + "SAFE_CAST", "TRY_CAST", + "CHECKED_LONG_SUM", "SUM"); private static final Map CALL_SEPARATOR = ImmutableMap.of("SAFE_CAST", "AS"); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index 415acc5558b..850cefd7308 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -482,7 +482,7 @@ public void testJoinWithRelationSubquery() { RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(sum=[$1], JOB=[$0])\n" - + " LogicalAggregate(group=[{0}], sum=[SUM($1)])\n" + + " LogicalAggregate(group=[{0}], sum=[CHECKED_LONG_SUM($1)])\n" + " LogicalProject(JOB=[$2], MGR=[$3])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java index 66027839f8e..63cc0572453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTimewrapTest.java @@ -86,7 +86,7 @@ public void testTimewrapDayProducesUnpivotedPlan() { + ":BIGINT NOT NULL) OVER (), CAST(UNIX_TIMESTAMP($0)):BIGINT NOT NULL), 86400), 1)])\n" + " LogicalSort(sort0=[$0], dir0=[ASC])\n" + " LogicalProject(@timestamp=[$0], sum(value)=[$1])\n" - + " LogicalAggregate(group=[{1}], sum(value)=[SUM($0)])\n" + + " LogicalAggregate(group=[{1}], sum(value)=[CHECKED_LONG_SUM($0)])\n" + " LogicalProject(value=[$1], @timestamp0=[SPAN($0, 6, 'h')])\n" + " LogicalFilter(condition=[AND(>=($0, TIMESTAMP('2024-07-01" + " 00:00:00':VARCHAR)), <=($0, TIMESTAMP('2024-07-03 18:00:00':VARCHAR)), IS NOT" @@ -109,7 +109,8 @@ public void testTimewrapDaySparkSql() { + " `__base_offset__`, ((MAX(CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) OVER (RANGE" + " BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) -" + " CAST(UNIX_TIMESTAMP(`@timestamp`) AS BIGINT)) / 86400 + 1 `__period__`\n" - + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`, SUM(`value`) `sum(value)`\n" + + "FROM (SELECT SPAN(`@timestamp`, 6, 'h') `@timestamp`," + + " SUM(`value`) `sum(value)`\n" + "FROM `scott`.`events`\n" + "WHERE `@timestamp` >= TIMESTAMP('2024-07-01 00:00:00') AND `@timestamp` <=" + " TIMESTAMP('2024-07-03 18:00:00') AND `value` IS NOT NULL\n" From d5a182d3a62587d5d46272a8630efb783dadba35 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 30 Jul 2026 20:03:14 -0700 Subject: [PATCH 15/78] Push down aggregation on text field without .keyword sub-field (#5646) * Push down aggregation on text field without .keyword sub-field For a text-typed group key or metric argument with no .keyword sub-field, NamedFieldExpression.getReferenceForTermQuery() returned null and CompositeValuesSourceBuilder/ValueCountAggregationBuilder rejected the null field, so pushDownAggregate silently fell back to a full _source scan and client-side aggregation. Route those bare RexInputRefs through a Calcite script that reads the value from _source, matching TermQuery/LikeQuery/RexStandardizer for filter and script fields. Composite terms buckets and metric aggregations that accept a script (notably count(FIELD)) now push down. Signed-off-by: Peng Huo Signed-off-by: Peng Huo * Apply spotless formatting Signed-off-by: Peng Huo Signed-off-by: Peng Huo * Update explain plan pins for text-field aggregation pushdown Four CalciteExplainIT plans encoded the pre-fix behavior where dedup / chart / timechart on a text field with no .keyword sub-field silently fell back to a client-side aggregation over an unbounded scan. With the AggregateAnalyzer fix those queries now push down as composite terms (script over _source), so the pinned physical plans are stale. - Rename testDedupTextTypeNotPushdown -> testDedupTextTypePushdown and update explain_dedup_text_type_push.yaml to the composite terms + top_hits DSL. - Refresh chart_null_str.yaml (chart limit=10 ... over gender by age span=10) to the composite terms(script) + histogram plan. - Refresh explain_timechart.yaml and explain_timechart_count.yaml (timechart span=1m ... by host) to the composite terms(script) + date_histogram plan. Add DedupCommandIT.testDedupOnTextField to verify behavioral equivalence: the result set for `source=bank | dedup email` matches the fixture's set of distinct emails, running both under the V2 path (base class) and Calcite pushdown path (CalciteDedupCommandIT). Signed-off-by: Peng Huo Signed-off-by: Peng Huo * Verify dedup row values in testDedupOnTextField Widen the assertion beyond the dedup key to also verify the associated projected columns per row (firstname, balance), so the top_hits round-trip in the pushed-down dedup DSL is checked end-to-end. Signed-off-by: Peng Huo Signed-off-by: Peng Huo * Add timechart avg-by-text-host result IT Assert row-level results for `source=events | timechart span=1m avg(cpu_usage) by host` on the events fixture, where `host` is a text field with no .keyword sub-field. Golden values were collected on upstream/main (unpushed) before applying the fix, so the assertion pins behavioral equivalence between the V2 client-side plan and the pushed composite terms(script)+date_histogram plan. Signed-off-by: Peng Huo Signed-off-by: Peng Huo * Trim comment on testTimechartAvgByTextHost Signed-off-by: Peng Huo Signed-off-by: Peng Huo --------- Signed-off-by: Peng Huo Signed-off-by: Peng Huo --- .../sql/calcite/remote/CalciteExplainIT.java | 6 +- .../remote/CalciteTimechartCommandIT.java | 20 ++ .../opensearch/sql/ppl/DedupCommandIT.java | 21 ++ .../calcite/chart_null_str.yaml | 10 +- .../explain_dedup_text_type_no_push.yaml | 13 - .../calcite/explain_dedup_text_type_push.yaml | 10 + .../calcite/explain_timechart.yaml | 15 +- .../calcite/explain_timechart_count.yaml | 9 +- .../test/issues/text_agg_pushdown.yml | 282 ++++++++++++++++++ .../opensearch/request/AggregateAnalyzer.java | 10 +- .../request/AggregateAnalyzerTest.java | 111 +++++-- 11 files changed, 441 insertions(+), 66 deletions(-) delete mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 04dc2b0e74b..d4850951eab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2366,9 +2366,11 @@ public void testDedupWithExpr() throws IOException { } @Test - public void testDedupTextTypeNotPushdown() throws IOException { + public void testDedupTextTypePushdown() throws IOException { + // A text field with no .keyword sub-field is aggregated by reading its value from _source via + // a Calcite script; dedup therefore pushes down as a composite terms + top_hits aggregation. enabledOnlyWhenPushdownIsEnabled(); - String expected = loadExpectedPlan("explain_dedup_text_type_no_push.yaml"); + String expected = loadExpectedPlan("explain_dedup_text_type_push.yaml"); assertYamlEqualsIgnoreId( expected, explainQueryYaml(String.format("source=%s | dedup email", TEST_INDEX_BANK))); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java index 73396ab31b9..7bd32bb2af2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTimechartCommandIT.java @@ -85,6 +85,26 @@ public void testTimechartWithCustomTimeField() throws IOException { verifyDataRows(result, rows("2017-01-01 00:00:00", 2), rows("2018-01-01 00:00:00", 5)); } + @Test + public void testTimechartAvgByTextHost() throws IOException { + // `host` is mapped as text with no .keyword sub-field, so this timechart pushes down via the + // text-field aggregation path (composite terms(script over _source) + date_histogram, with + // avg(cpu_usage) as a numeric metric on the composite). + JSONObject result = executeQuery("source=events | timechart span=1m avg(cpu_usage) by host"); + verifySchema( + result, + schema("@timestamp", "timestamp"), + schema("host", "string"), + schema("avg(cpu_usage)", "double")); + verifyDataRows( + result, + rows("2024-07-01 00:00:00", "web-01", 45.2), + rows("2024-07-01 00:01:00", "web-02", 38.7), + rows("2024-07-01 00:02:00", "web-01", 55.3), + rows("2024-07-01 00:03:00", "db-01", 42.1), + rows("2024-07-01 00:04:00", "web-02", 41.8)); + } + @Test public void testTimechartWithMinuteSpanNoGroupBy() throws IOException { JSONObject result = executeQuery("source=events | timechart span=1m avg(cpu_usage)"); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java index 45ecf02af8f..1507764b020 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java @@ -58,6 +58,27 @@ public void testAllowMoreDuplicates() throws IOException { verifyDataRows(result, rows(true), rows(true), rows(false), rows(false)); } + @Test + public void testDedupOnTextField() throws IOException { + // `email` is mapped as text with no .keyword sub-field, so dedup runs via the text-field + // aggregation pushdown path (composite terms + top_hits with the field read from _source). + // Assert not just the dedup key set but also the associated projected columns per row, so + // the top_hits round-trip is exercised end-to-end. + JSONObject result = + executeQuery( + String.format( + "source=%s | dedup email | fields email, firstname, balance", TEST_INDEX_BANK)); + verifyDataRows( + result, + rows("amberduke@pyrami.com", "Amber JOHnny", 39225), + rows("hattiebond@netagy.com", "Hattie", 5686), + rows("nanettebates@quility.com", "Nanette", 32838), + rows("daleadams@boink.com", "Dale", 4180), + rows("elinorratliff@scentric.com", "Elinor", 16418), + rows("virginiaayala@filodyne.com", "Virginia", 40540), + rows("dillardmcpherson@quailcom.com", "Dillard", 48086)); + } + @Test public void testKeepEmptyDedup() throws IOException { JSONObject result = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml index f9196e3b597..9776f5dafc1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_null_str.yaml @@ -26,14 +26,10 @@ calcite: EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) EnumerableSort(sort0=[$1], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], gender=[$t0], age=[$t3], avg(balance)=[$t2]) - EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["gender"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], age=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[SAFE_CAST($t1)], expr#4=[IS NOT NULL($t3)], age=[$t3], avg(balance)=[$t2], $condition=[$t4]) - EnumerableAggregate(group=[{0, 2}], avg(balance)=[AVG($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[10], expr#4=[null:NULL], expr#5=[SPAN($t2, $t3, $t4)], gender=[$t1], balance=[$t0], age0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[PROJECT->[balance, gender, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","gender","age"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[SAFE_CAST($t0)], expr#3=[IS NOT NULL($t2)], age=[$t2], avg(balance)=[$t1], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank_with_null_values]], PushDownContext=[[FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(balance)=AVG($1)), PROJECT->[age0, avg(balance)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["gender"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"age0":{"histogram":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc","interval":10.0}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml deleted file mode 100644 index e7f26a14a96..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_no_push.yaml +++ /dev/null @@ -1,13 +0,0 @@ -calcite: - logical: | - LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) - LogicalFilter(condition=[<=($19, 1)]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $11)]) - LogicalFilter(condition=[IS NOT NULL($11)]) - CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) - physical: | - EnumerableLimit(fetch=[10000]) - EnumerableCalc(expr#0..13=[{inputs}], expr#14=[1], expr#15=[<=($t13, $t14)], proj#0..12=[{exprs}], $condition=[$t15]) - EnumerableWindow(window#0=[window(partition {11} rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->IS NOT NULL($11)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"email","boost":1.0}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml new file mode 100644 index 00000000000..11aa4be7da2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_text_type_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($19, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $11)]) + LogicalFilter(condition=[IS NOT NULL($11)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"email":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["email"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email"},{"field":"account_number"},{"field":"firstname"},{"field":"address"},{"field":"birthdate"},{"field":"gender"},{"field":"city"},{"field":"lastname"},{"field":"balance"},{"field":"employer"},{"field":"state"},{"field":"age"},{"field":"male"}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml index 0818c18eabb..bbb9c602016 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -19,21 +19,14 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($1), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], avg(cpu_usage)=[$t8]) + CalciteEnumerableTopK(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000]) EnumerableAggregate(group=[{0, 1}], agg#0=[$SUM0($2)], agg#1=[COUNT($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t0, $t3, $t4)], host=[$t1], cpu_usage=[$t2], @timestamp0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp","host","cpu_usage"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(cpu_usage)=AVG($1)), PROJECT->[@timestamp0, host, avg(cpu_usage)], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"last","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableAggregate(group=[{0}], __grand_total__=[SUM($1)]) - EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], avg(cpu_usage)=[$t8]) - EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) - EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], @timestamp0=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->AND(IS NOT NULL($0), IS NOT NULL($2)), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(cpu_usage)=AVG($1)), PROJECT->[host, avg(cpu_usage)]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"bool":{"must":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"cpu_usage","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml index f26cb9e5822..a8f49b59ee4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -23,13 +23,8 @@ calcite: EnumerableAggregate(group=[{0, 1}], count()=[$SUM0($2)]) EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NULL($t1)], expr#6=['NULL'], expr#7=[10], expr#8=[<=($t4, $t7)], expr#9=['OTHER'], expr#10=[CASE($t5, $t6, $t8, $t1, $t9)], @timestamp=[$t0], host=[$t10], count()=[$t2]) EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) - EnumerableSort(sort0=[$1], dir0=[ASC]) - EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], count()=[$t2]) - EnumerableAggregate(group=[{0, 1}], count()=[COUNT()]) - EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t0, $t2, $t3)], host=[$t1], @timestamp0=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"_source":{"includes":["@timestamp","host"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[@timestamp0, host, count()], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"last","order":"asc"}}},{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) EnumerableSort(sort0=[$0], dir0=[ASC]) EnumerableCalc(expr#0..2=[{inputs}], host=[$t0], _row_number_chart_=[$t2]) EnumerableWindow(window#0=[window(order by [1 DESC-nulls-last] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) - EnumerableAggregate(group=[{0}], __grand_total__=[COUNT()]) - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($0), PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["host","@timestamp"]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[FILTER->IS NOT NULL($0), FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},__grand_total__=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"filter":[{"exists":{"field":"@timestamp","boost":1.0}},{"exists":{"field":"host","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"host":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQAaXsKICAiZHluYW1pY1BhcmFtIjogMCwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["host"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml new file mode 100644 index 00000000000..f13798a7729 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/text_agg_pushdown.yml @@ -0,0 +1,282 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + + # Primary index — appId is text-only, no keyword sub-field. Before the fix this + # scenario silently degraded to a full _source scan with client-side aggregation. + - do: + indices.create: + index: logs_text_only + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + appId: + type: text + level: + type: keyword + + - do: + bulk: + index: logs_text_only + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"appId": "app-1", "level": "info"}' + - '{"index": {"_id": "2"}}' + - '{"appId": "app-1", "level": "warn"}' + - '{"index": {"_id": "3"}}' + - '{"appId": "app-2", "level": "info"}' + - '{"index": {"_id": "4"}}' + - '{"appId": "app-3", "level": "info"}' + - '{"index": {"_id": "5"}}' + - '{"appId": "app-1", "level": "info"}' + + # Companion index — appId is text with a keyword sub-field. Sits under the + # same logs_text* pattern so multi-index queries also exercise the + # conflicting-mapping case where the merged type loses its keyword sub-field. + - do: + indices.create: + index: logs_text_with_keyword + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + appId: + type: text + fields: + keyword: + type: keyword + ignore_above: 256 + + - do: + bulk: + index: logs_text_with_keyword + refresh: true + body: + - '{"index": {}}' + - '{"appId": "app-1"}' + - '{"index": {}}' + - '{"appId": "app-1"}' + - '{"index": {}}' + - '{"appId": "app-2"}' + +--- +teardown: + - do: + indices.delete: + index: logs_text_only,logs_text_with_keyword + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + +--- +"top command on a text field returns correct counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | top 10 appId + - match: { total: 3 } + - match: + schema: + - { name: appId, type: string } + - { name: count, type: bigint } + - match: + datarows: + - [ "app-1", 3 ] + - [ "app-2", 1 ] + - [ "app-3", 1 ] + +--- +"stats by text field returns correct counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | stats count() by appId + - match: { total: 3 } + - match: + schema: + - { name: "count()", type: bigint } + - { name: appId, type: string } + - match: + datarows: + - [ 3, "app-1" ] + - [ 1, "app-2" ] + - [ 1, "app-3" ] + +--- +"top on text field pushes down as composite terms with _source script": + # AGGREGATION pushdown means the physical plan carries a composite_buckets + # aggregation whose terms source uses a script (not a field), because appId + # is a text field with no .keyword sub-field. Presence of both AGGREGATION + # in PushDownContext and the composite/script bucket confirms the DSL is + # pushed to OpenSearch instead of falling back to an unbounded scan. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | top 10 appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"stats by text field pushes down as composite terms with _source script": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"count(text_field) on text-only pushes down as scripted value_count": + # count(FIELD) reaches the metric path with a bare RexInputRef (Calcite does not + # insert a numeric cast around COUNT). For a text field without .keyword, the + # metric aggregation must still push down by using a script value source that + # reads the field from _source, not fall back to an unbounded scan. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_only | stats count(appId) + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/\"count\\(appId\\)\":\\{\"value_count\":\\{\"script\"/" } + +--- +"count(text_field) on text-only returns correct count": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text_only | stats count(appId) + - match: { total: 1 } + - match: + schema: + - { name: "count(appId)", type: bigint } + - match: + datarows: + - [ 5 ] + +--- +"text field with .keyword sub-field pushes down using the sub-field": + # A text field that carries `.keyword` must be rewritten to `.keyword` + # in the DSL, not routed through the script path. Baseline that guarantees the + # fix did NOT alter behavior for keyword-backed fields. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text_with_keyword | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"field\":\"appId\\.keyword\"/" } + +--- +"multi-index stats across text-only + text+keyword pushes down via _source script": + # `source=logs_text*` spans a text-only and a text+keyword index. The + # merged `appId` field loses its .keyword sub-field (because one member has + # none). Before the fix this fell back to a full _source scan and client-side + # aggregation; after the fix the composite terms bucket uses the _source- + # reading script for the merged field type. + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl.explain: + body: + query: source=logs_text* | stats count() by appId + - match: { "calcite.physical": "/AGGREGATION/" } + - match: { "calcite.physical": "/composite_buckets/" } + - match: { "calcite.physical": "/\"appId\":\\{\"terms\":\\{\"script\"/" } + +--- +"multi-index stats returns correct merged counts": + # logs_text_only: app-1=3, app-2=1, app-3=1 + # logs_text_with_keyword: app-1=2, app-2=1 + # merged: app-1=5, app-2=2, app-3=1 + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text* | stats count() by appId + - match: { total: 3 } + - match: + schema: + - { name: "count()", type: bigint } + - { name: appId, type: string } + - match: + datarows: + - [ 5, "app-1" ] + - [ 2, "app-2" ] + - [ 1, "app-3" ] + +--- +"multi-index top returns correct merged counts": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=logs_text* | top 10 appId + - match: { total: 3 } + - match: + schema: + - { name: appId, type: string } + - { name: count, type: bigint } + - match: + datarows: + - [ "app-1", 5 ] + - [ "app-2", 2 ] + - [ "app-3", 1 ] diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 5a336a2e500..6aba75cbcde 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -160,7 +160,13 @@ > T build(RexNode node, T sourceBuilde T build(RexNode node, Function fieldBuilder, Function scriptBuilder) { if (node == null) return fieldBuilder.apply(METADATA_FIELD); else if (node instanceof RexInputRef ref) { - return fieldBuilder.apply(inferNamedField(node).getReferenceForTermQuery()); + String fieldRef = inferNamedField(node).getReferenceForTermQuery(); + // Text field with no .keyword sub-field is not aggregatable directly. Fall back to a + // Calcite script that reads the value from _source. + if (fieldRef == null) { + return scriptBuilder.apply(inferScript(node).getScript()); + } + return fieldBuilder.apply(fieldRef); } else if (node instanceof RexCall || node instanceof RexLiteral) { return scriptBuilder.apply(inferScript(node).getScript()); } @@ -177,7 +183,7 @@ NamedFieldExpression inferNamedField(RexNode node) { } ScriptQueryExpression inferScript(RexNode node) { - if (node instanceof RexCall || node instanceof RexLiteral) { + if (node instanceof RexCall || node instanceof RexLiteral || node instanceof RexInputRef) { return new ScriptQueryExpression( node, rowType, fieldTypes, cluster, Collections.emptyMap()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 0995a60adeb..521ac7f109f 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -361,33 +360,81 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { } @Test - void analyze_aggCall_TextWithoutKeyword() { + void analyze_aggCall_TextWithoutKeyword_countPushesDownAsScript() + throws ExpressionNotAnalyzableException { + // count(FIELD) on a text field with no .keyword sub-field must not fall back to an + // unbounded client-side scan. The metric aggregation is built with a script value source + // that reads the value from _source, matching the pattern used by TermQuery/LikeQuery. + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + SchemaPlus root = Frameworks.createRootSchema(true); + root.add( + "test", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory tf) { + return rowType; + } + }); + RelBuilder rb = + RelBuilder.create(Frameworks.newConfigBuilder().defaultSchema(root).build()).scan("test"); + AggregateCall aggCall = AggregateCall.create( - SqlStdOperatorTable.SUM, + SqlStdOperatorTable.COUNT, false, false, false, ImmutableList.of(), - ImmutableList.of(0), + // arg #2 in the row type is `c` — text without .keyword sub-field + ImmutableList.of(2), -1, null, RelCollations.EMPTY, - typeFactory.createSqlType(SqlTypeName.INTEGER), - "sum"); - Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of()); - Project project = createMockProject(List.of(2)); + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + + RelNode rel = rb.aggregate(rb.groupKey(), List.of(aggCall)).build(); + Aggregate aggregate = (Aggregate) rel; + AggregateAnalyzer.AggregateBuilderHelper helper = - new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); - ExpressionNotAnalyzableException exception = - assertThrows( - ExpressionNotAnalyzableException.class, - () -> AggregateAnalyzer.analyze(aggregate, project, List.of("sum"), helper)); - assertEquals("[field] must not be null: [sum]", exception.getCause().getMessage()); + new AggregateAnalyzer.AggregateBuilderHelper( + rowType, fieldTypes, aggregate.getCluster(), true, BUCKET_SIZE); + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, null, List.of("cnt"), helper); + + String dsl = result.getLeft().toString(); + // The value_count metric must use a script value source (not a raw field) on `c`. + assertTrue( + dsl.contains("\"cnt\":{\"value_count\":{\"script\":{"), + "expected value_count metric on 'c' to use a script, but got: " + dsl); + assertTrue( + dsl.contains("\"lang\":\"opensearch_compounded_script\""), + "expected compounded script lang, but got: " + dsl); + assertTrue( + dsl.contains("\"DIGESTS\":[\"c\"]"), + "expected script to reference field 'c' via DIGESTS, but got: " + dsl); } @Test - void analyze_groupBy_TextWithoutKeyword() { + void analyze_groupBy_TextWithoutKeyword() throws ExpressionNotAnalyzableException { + // Grouping by a text field with no .keyword sub-field must not fall back to an unbounded + // client-side scan. Instead, the composite terms bucket is built with a script value source + // that reads the field from _source, matching the pattern used by TermQuery/LikeQuery. + Hook.CURRENT_TIME.addThread((Consumer>) h -> h.set(0L)); + + SchemaPlus root = Frameworks.createRootSchema(true); + root.add( + "test", + new AbstractTable() { + @Override + public RelDataType getRowType(RelDataTypeFactory tf) { + return rowType; + } + }); + RelBuilder rb = + RelBuilder.create(Frameworks.newConfigBuilder().defaultSchema(root).build()).scan("test"); + AggregateCall aggCall = AggregateCall.create( SqlStdOperatorTable.COUNT, @@ -401,16 +448,32 @@ void analyze_groupBy_TextWithoutKeyword() { RelCollations.EMPTY, typeFactory.createSqlType(SqlTypeName.INTEGER), "cnt"); - List outputFields = List.of("c", "cnt"); - Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of(0)); - Project project = createMockProject(List.of(2)); + + RelNode rel = rb.aggregate(rb.groupKey(ImmutableBitSet.of(2)), List.of(aggCall)).build(); + Aggregate aggregate = (Aggregate) rel; + Project project = null; + AggregateAnalyzer.AggregateBuilderHelper helper = - new AggregateAnalyzer.AggregateBuilderHelper(rowType, fieldTypes, null, true, BUCKET_SIZE); - ExpressionNotAnalyzableException exception = - assertThrows( - ExpressionNotAnalyzableException.class, - () -> AggregateAnalyzer.analyze(aggregate, project, outputFields, helper)); - assertEquals("[field] must not be null", exception.getCause().getMessage()); + new AggregateAnalyzer.AggregateBuilderHelper( + rowType, fieldTypes, aggregate.getCluster(), true, BUCKET_SIZE); + Pair, OpenSearchAggregationResponseParser> result = + AggregateAnalyzer.analyze(aggregate, project, List.of("c", "cnt"), helper); + + String dsl = result.getLeft().toString(); + // Composite bucket for `c` must use a script value source (not a raw field). + assertTrue( + dsl.contains( + "\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" + + "{\"c\":{\"terms\":{\"script\":{"), + "expected composite terms bucket on 'c' to use a script, but got: " + dsl); + // The script must be tagged as a Calcite compounded script and reference the `c` field. + assertTrue( + dsl.contains("\"lang\":\"opensearch_compounded_script\""), + "expected compounded script lang, but got: " + dsl); + assertTrue( + dsl.contains("\"DIGESTS\":[\"c\"]"), + "expected script to reference field 'c' via DIGESTS, but got: " + dsl); + assertInstanceOf(BucketAggregationParser.class, result.getRight()); } @Test From 2852724fc063eda9dac91cf99a51137cdab8762c Mon Sep 17 00:00:00 2001 From: opensearch-ci <83309141+opensearch-ci-bot@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:41:32 -0400 Subject: [PATCH 16/78] Add ci.opensearch.org/m2/ mirror for plugin resolution (sql) (#5664) (#5666) * Add ci.opensearch.org/m2/ mirror for plugin resolution (sql) * Address order issues --------- (cherry picked from commit fe20ba69e3bb201183d2462bce37bf520cc3aa92) Signed-off-by: shreyah963 Signed-off-by: Peter Zhu Signed-off-by: opensearch-ci-bot Co-authored-by: Shreya Bhatta Co-authored-by: Peter Zhu --- settings.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.gradle b/settings.gradle index 7fc93fe1725..5f83c4f5a87 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,6 +6,7 @@ pluginManagement { repositories { maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } gradlePluginPortal() mavenCentral() } From 79d532df923271937cc7316c04d74666380dc0ae Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sat, 1 Aug 2026 13:10:47 -0400 Subject: [PATCH 17/78] Add ci.opensearch.org/m2/ mirror to buildscript and project repos (sql) (#5667) Signed-off-by: Peter Zhu --- build.gradle | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 9047a9c3feb..7f32a194739 100644 --- a/build.gradle +++ b/build.gradle @@ -68,6 +68,7 @@ buildscript { repositories { mavenLocal() maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } mavenCentral() maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } } @@ -93,13 +94,14 @@ apply plugin: 'opensearch.java-agent' // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() - maven { url "https://ci.opensearch.org/maven2/" } - mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available maven { url 'https://jitpack.io' content { includeGroup "com.github.babbel" } } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } + maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } + mavenCentral() } spotless { @@ -175,14 +177,15 @@ allprojects { subprojects { repositories { mavenLocal() - maven { url "https://ci.opensearch.org/maven2/" } - mavenCentral() maven { url 'https://jitpack.io' content { includeGroup "com.github.babbel" } } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" } maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" } + maven { url "https://ci.opensearch.org/maven2/" } + maven { url "https://ci.opensearch.org/m2/" } + mavenCentral() } // Publish internal modules as Maven artifacts for external use, such as by opensearch-spark and opensearch-cli. From 8ca47ac58f5b06efbb359471826b0ca821b3da41 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Tue, 4 Aug 2026 09:11:39 -0700 Subject: [PATCH 18/78] Add a generic, extensible rest-endpoint provider SPI (#5656) * [Feature] Add PPL `rest` command (Calcite system row source) Add a leading `rest ` command that exposes a curated, read-only, fixed-schema set of in-cluster management endpoints as a PPL table. --------- Signed-off-by: Louis Chu --- .../sql/common/setting/Settings.java | 1 + .../opensearch/sql/ast/tree/RestRelation.java | 25 +++ .../sql/utils/SystemIndexUtils.java | 108 ++++++++++ docs/category.json | 1 + docs/user/ppl/cmd/rest.md | 63 ++++++ docs/user/ppl/index.md | 1 + doctest/build.gradle | 2 + integ-test/build.gradle | 6 + .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 9 + .../sql/calcite/remote/CalcitePPLRestIT.java | 112 +++++++++++ .../sql/plugin/AnalyticsEngineCompatIT.java | 28 +++ .../sql/ppl/NewAddedCommandsIT.java | 13 ++ .../sql/security/RestCommandSecurityIT.java | 90 +++++++++ .../opensearch/sql/sql/VectorSearchIT.java | 1 + opensearch/build.gradle | 1 + .../rules/EnumerableCatalogScanRule.java | 63 ++++++ .../rules/EnumerableSystemIndexScanRule.java | 50 ----- .../planner/rules/OpenSearchIndexRules.java | 6 +- .../setting/OpenSearchSettings.java | 17 +- .../storage/OpenSearchStorageEngine.java | 32 ++- .../storage/rest/CoreEndpointsProvider.java | 64 ++++++ .../storage/rest/RestCatalogSource.java | 57 ++++++ .../storage/rest/RestEndpointRegistry.java | 176 ++++++++++++++++ .../rest/RestEndpointRegistryHolder.java | 32 +++ .../opensearch/storage/rest/RestRequest.java | 55 +++++ ...n.java => AbstractCalciteCatalogScan.java} | 12 +- ...java => CalciteEnumerableCatalogScan.java} | 23 ++- ...an.java => CalciteLogicalCatalogScan.java} | 20 +- .../system/CalciteScannableCatalogScan.java | 29 +++ .../storage/system/CatalogSource.java | 39 ++++ ....java => OpenSearchCatalogEnumerator.java} | 19 +- .../system/OpenSearchCatalogTable.java | 67 +++++++ .../storage/system/OpenSearchSystemIndex.java | 105 ---------- .../system/SystemIndexCatalogSource.java | 70 +++++++ .../setting/OpenSearchSettingsTest.java | 9 + .../storage/OpenSearchStorageEngineTest.java | 85 +++++++- .../rest/CoreEndpointsProviderTest.java | 60 ++++++ .../storage/rest/RestCatalogSourceTest.java | 143 +++++++++++++ .../rest/RestEndpointExtensibilityTest.java | 95 +++++++++ .../rest/RestEndpointRegistryTest.java | 188 ++++++++++++++++++ ...t.java => OpenSearchCatalogTableTest.java} | 25 ++- .../org/opensearch/sql/plugin/SQLPlugin.java | 18 ++ .../plugin/rest/RestUnifiedQueryAction.java | 9 +- .../rest/RestUnifiedQueryActionTest.java | 6 + ppl-rest-spi/README.md | 171 ++++++++++++++++ ppl-rest-spi/build.gradle | 106 ++++++++++ .../org/opensearch/sql/spi/rest/ArgSpec.java | 83 ++++++++ .../sql/spi/rest/RestEndpointContext.java | 38 ++++ .../sql/spi/rest/RestEndpointDefinition.java | 74 +++++++ .../sql/spi/rest/RestEndpointHandler.java | 28 +++ .../sql/spi/rest/RestEndpointProvider.java | 21 ++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 14 ++ .../opensearch/sql/ppl/parser/AstBuilder.java | 30 +++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 19 ++ .../sql/ppl/calcite/CalcitePPLRestTest.java | 67 +++++++ .../sql/ppl/parser/AstBuilderTest.java | 28 +++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 12 ++ settings.gradle | 1 + 60 files changed, 2521 insertions(+), 209 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java create mode 100644 docs/user/ppl/cmd/rest.md create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{AbstractCalciteSystemIndexScan.java => AbstractCalciteCatalogScan.java} (68%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteEnumerableSystemIndexScan.java => CalciteEnumerableCatalogScan.java} (80%) rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{CalciteLogicalSystemIndexScan.java => CalciteLogicalCatalogScan.java} (59%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java rename opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexEnumerator.java => OpenSearchCatalogEnumerator.java} (76%) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java delete mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java rename opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/{OpenSearchSystemIndexTest.java => OpenSearchCatalogTableTest.java} (76%) create mode 100644 ppl-rest-spi/README.md create mode 100644 ppl-rest-spi/build.gradle create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java create mode 100644 ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index d32fd249e02..5ab82f6b6c9 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,6 +36,7 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), + PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java new file mode 100644 index 00000000000..d0a0e34d3c9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import java.util.Collections; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, + * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that + * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source + * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. + */ +@ToString +@EqualsAndHashCode(callSuper = false) +public class RestRelation extends Relation { + public RestRelation(UnresolvedExpression tableName) { + super(Collections.singletonList(tableName)); + } +} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index c9fa35d7068..059bb8238d0 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,6 +5,10 @@ package org.opensearch.sql.utils; +import java.nio.charset.StandardCharsets; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -34,6 +38,110 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } + /** + * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} + * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a + * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution + * the same way the uppercase system-mapping names do. + */ + private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; + + private static final String REST_SOURCE_PREFIX = "REST"; + + /** True if the resolved table name is a {@code rest} source token. */ + public static boolean isRestSource(String indexName) { + return indexName.endsWith(REST_SOURCE_SUFFIX); + } + + /** + * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link + * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. + * The endpoint/args have already been allow-list-validated before this is called. + */ + public static String restTable(RestSpec spec) { + StringBuilder sb = new StringBuilder(); + sb.append("endpoint=").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append('\n').append("count=").append(spec.getCount()); + } + if (spec.getTimeout() != null) { + sb.append('\n').append("timeout=").append(spec.getTimeout()); + } + if (spec.getArgs() != null) { + for (Map.Entry e : spec.getArgs().entrySet()) { + sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); + } + } + return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; + } + + /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ + public static RestSpec decodeRestSpec(String indexName) { + // Validate the token shape before slicing it: callers gate on isRestSource today, but a + // public decoder must not assume its precondition, otherwise a malformed token would throw an + // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. + if (!isRestSource(indexName)) { + throw new IllegalArgumentException("not a valid rest source token: " + indexName); + } + String body = + indexName.substring( + REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); + String decoded = fromHex(body); + String endpoint = null; + Integer count = null; + String timeout = null; + LinkedHashMap args = new LinkedHashMap<>(); + for (String line : decoded.split("\n")) { + if (line.isEmpty()) { + continue; + } + int eq = line.indexOf('='); + if (eq < 0) { + continue; + } + String k = line.substring(0, eq); + String v = line.substring(eq + 1); + if (k.equals("endpoint")) { + endpoint = v; + } else if (k.equals("count")) { + count = Integer.parseInt(v); + } else if (k.equals("timeout")) { + timeout = v; + } else if (k.startsWith("arg.")) { + args.put(k.substring("arg.".length()), v); + } + } + if (endpoint == null) { + throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); + } + return new RestSpec(endpoint, args, count, timeout); + } + + private static String toHex(String s) { + return HexFormat.of().formatHex(s.getBytes(StandardCharsets.UTF_8)); + } + + private static String fromHex(String h) { + if (h.length() % 2 != 0) { + throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); + } + return new String(HexFormat.of().parseHex(h), StandardCharsets.UTF_8); + } + + /** + * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional + * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) + * share it without a cross-module dependency. + */ + @Getter + @RequiredArgsConstructor + public static class RestSpec { + private final String endpoint; + private final Map args; + private final Integer count; + private final String timeout; + } + /** * Compose system mapping table. * diff --git a/docs/category.json b/docs/category.json index 7296b089650..a8665b82eac 100644 --- a/docs/category.json +++ b/docs/category.json @@ -35,6 +35,7 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", + "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md new file mode 100644 index 00000000000..6860a619b97 --- /dev/null +++ b/docs/user/ppl/cmd/rest.md @@ -0,0 +1,63 @@ +# rest + +The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. + +> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. + +The `rest` command is a generic, extensible framework: a plugin contributes additional read-only endpoints through the `RestEndpointProvider` extension point without changing the grammar. This first version ships a single built-in endpoint, `/_cluster/health`. Additional endpoints (for example `/_cat/nodes`, `/_cat/shards`, `/_cluster/state`, `/_cluster/settings`) can be added in follow-ups, with any response redaction handled inside the provider's own handler. + +## Enabling the command + +`/_cluster/health` is **enabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to `["/_cluster/health"]`. Any other endpoint is rejected until a deployment adds it to the allow-list (a node-level setting, applied at node startup and not changeable at runtime): + +```yaml +plugins.ppl.rest.allowed_endpoints: ["/_cluster/health"] +``` + +Every endpoint must be listed explicitly by name; there is no wildcard, so a newly installed or upgraded provider is never enabled without an explicit allow-list change. Set an empty list to disable the command entirely. + +## Syntax + +```syntax +rest [count=] [= ...] +``` + +## Parameters + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | +| `count=` | Optional | Caps the number of emitted rows. | +| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`). | + +## Allow-list + +`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. + +| Endpoint | Output columns | Accepted args | +| --- | --- | --- | +| `/_cluster/health` | `response` (string): the full cluster-health response as JSON. Extract fields with `json_extract` or the `spath` command (see the example below). | `local` | + +## Example: Reading fields from the response + +`/_cluster/health` returns the full health response in a single `response` column as JSON. Extract the fields you need with `json_extract` (or the `spath` command): + +```ppl ignore +| rest '/_cluster/health' +| eval status = json_extract(response, 'status'), + number_of_nodes = json_extract(response, 'number_of_nodes') +| fields status, number_of_nodes +``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++--------+-----------------+ +| status | number_of_nodes | +|--------+-----------------| +| green | 1 | ++--------+-----------------+ +``` + +Because the whole response is available, a query can read any field it exposes (for example `active_shards`, `active_primary_shards`, `unassigned_shards`) without the endpoint pre-declaring a column for it. The extracted columns then compose with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan, for example `| rest '/_cluster/health' | spath input=response path=status output=status | where status = 'green'`. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 647c4568446..3e5a08d990d 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -75,6 +75,7 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | +| [rest command](cmd/rest.md) | 3.9 | experimental (since 3.9) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/doctest/build.gradle b/doctest/build.gradle index cce64170f56..45f4b768aef 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,6 +205,8 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' + // Only /_cluster/health is registered; pin the allow-list to it so the rest.md examples run. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1435d1d499d..29627ff7b60 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,6 +387,8 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" + // Only /_cluster/health is registered; pin the allow-list to it for the rest ITs. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } yamlRestTest { testDistribution = 'archive' @@ -405,6 +407,8 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" + // Only /_cluster/health is registered; pin the allow-list to it for RestCommandSecurityIT. + setting 'plugins.ppl.rest.allowed_endpoints', '/_cluster/health' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -419,6 +423,8 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" + // Composite-default cluster: PPL queries route to the analytics engine unless excluded. + setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 6d0c88bf773..e7c024e49fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -59,6 +59,7 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, + CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index d4850951eab..d3589fcf138 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -66,6 +66,15 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. + @Test + public void explainRestCommand() throws IOException { + String result = explainQueryToString("| rest '/_cluster/health' | fields response"); + Assert.assertTrue( + "Expected a rest scan node in the explain output, got: " + result, + result.contains("CatalogScan")); + } + @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java new file mode 100644 index 00000000000..ae16f839e9c --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java @@ -0,0 +1,112 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for the {@code rest} leading command on the Calcite path. This first version + * ships a single built-in endpoint, {@code /_cluster/health} (a deterministic single-row endpoint + * that carries no network identifiers and needs no redaction). These tests exercise it end to end + * and verify the allow-list and per-arg gates. + */ +public class CalcitePPLRestIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void testRestClusterHealthSchema() throws IOException { + JSONObject result = executeQuery("| rest '/_cluster/health' | fields response"); + verifySchema(result, schema("response", "string")); + } + + @Test + public void testRestClusterHealthResponseIsValidJson() throws IOException { + JSONObject result = + executeQuery("| rest '/_cluster/health' | eval ok = json_valid(response) | fields ok"); + verifyDataRows(result, rows(true)); + } + + @Test + public void testRestClusterHealthJsonExtract() throws IOException { + JSONObject result = + executeQuery( + "| rest '/_cluster/health' | eval status = json_extract(response, 'status')" + + " | fields status"); + verifySchema(result, schema("status", "string")); + } + + @Test + public void testRestClusterHealthSpath() throws IOException { + JSONObject result = + executeQuery( + "| rest '/_cluster/health' | spath input=response path=status output=status" + + " | where status = 'green' or status = 'yellow' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterHealthComposesDownstream() throws IOException { + // The rest row source composes with downstream stats exactly like an index scan. + JSONObject result = executeQuery("| rest '/_cluster/health' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestClusterHealthLocalArg() throws IOException { + // local=true reads health from the local node; on a single-node cluster the row is unchanged. + JSONObject result = + executeQuery("| rest '/_cluster/health' local='true' | stats count() as cnt"); + verifyDataRows(result, rows(1)); + } + + @Test + public void testRestRejectsNonAllowListedEndpoint() { + // /_cat/nodes is not registered in this version; it is refused before any transport call. + assertRestBadRequest("| rest '/_cat/nodes'", "allow-list"); + } + + @Test + public void testRestRejectsEmptyEndpoint() { + assertRestBadRequest("| rest ''", "non-empty path"); + } + + @Test + public void testRestRejectsDisallowedArg() { + assertRestBadRequest("| rest '/_cluster/health' h='name'", "does not accept arg"); + } + + @Test + public void testRestRejectsNegativeCount() { + assertRestBadRequest("| rest '/_cluster/health' count=-1", "non-negative"); + } + + /** + * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) + * with the given substring in the response body. Covers allow-list and bad-argument rejection. + */ + private void assertRestBadRequest(String query, String expectedSubstring) { + ResponseException e = + org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); + org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); + org.junit.Assert.assertTrue( + "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), + e.getMessage().contains(expectedSubstring)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index f6ec903c395..9e3fe563d41 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,9 +5,13 @@ package org.opensearch.sql.plugin; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; +import org.json.JSONArray; +import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -56,4 +60,28 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } + + /** + * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it + * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its + * single-column schema and correct data unchanged when the analytics-engine plugin is present. + */ + @Test + public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity("{\"query\": \"| rest '/_cluster/health' | fields response\"}"); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + + JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); + + JSONArray schema = result.getJSONArray("schema"); + assertEquals(1, schema.length()); + assertEquals("response", schema.getJSONObject(0).getString("name")); + assertEquals("string", schema.getJSONObject(0).getString("type")); + + JSONArray datarows = result.getJSONArray("datarows"); + assertEquals(1, datarows.length()); + assertTrue(datarows.getJSONArray(0).getString(0).contains("number_of_nodes")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index a32dd9fb990..6a6c0bc222c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -33,6 +33,19 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } + @Test + public void testRest() throws IOException { + JSONObject result; + try { + result = executeQuery("| rest '/_cluster/health' | fields response"); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + } + if (isCalciteEnabled()) { + assertFalse(result.getJSONArray("datarows").isEmpty()); + } + } + @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java new file mode 100644 index 00000000000..c4b5e726a54 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; + +/** + * Integration tests that the rest command is subject to the security plugin fine grained access + * control. The command dispatches a standard transport action under the caller identity, so the + * security ActionFilter authorizes it by action name. This version ships only {@code + * /_cluster/health}, which requires the {@code cluster:monitor/health} privilege: a caller holding + * cluster monitor can run it, a caller without it is denied. The command therefore grants no access + * beyond calling the endpoint natively. + */ +public class RestCommandSecurityIT extends SecurityTestBase { + + private static final String MONITOR_USER = "rest_monitor_user"; + private static final String MONITOR_ROLE = "rest_monitor_role"; + + private static final String NO_MONITOR_USER = "rest_no_monitor_user"; + private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; + + @Override + protected void init() throws Exception { + super.init(); + setupRolesAndUsers(); + enableCalcite(); + // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported + // command error. Disable fallback so the denial reason surfaces to the caller. + disallowCalciteFallback(); + } + + private void setupRolesAndUsers() throws IOException { + createRoleWithPermissions( + MONITOR_ROLE, + "*", + new String[] {"cluster:admin/opensearch/ppl", "cluster:monitor/health"}, + new String[] {}); + createUser(MONITOR_USER, MONITOR_ROLE); + + createRoleWithPermissions( + NO_MONITOR_ROLE, "*", new String[] {"cluster:admin/opensearch/ppl"}, new String[] {}); + createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); + } + + @Test + public void monitorUserCanRunClusterHealth() throws IOException { + JSONObject result = + executeQueryAsUser("| rest '/_cluster/health' | fields response", MONITOR_USER); + verifyColumn(result, columnName("response")); + } + + @Test + public void userWithoutClusterMonitorCannotRunClusterHealth() throws IOException { + assertDenied( + "| rest '/_cluster/health' | fields response", NO_MONITOR_USER, "cluster:monitor/health"); + } + + /** + * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on + * the Calcite only rest path surfaces as a client or server error whose body carries the security + * denial reason, so this checks the denial signal rather than a fixed status code. + */ + private void assertDenied(String query, String user, String deniedAction) throws IOException { + try { + executeQueryAsUser(query, user); + fail("Expected a permission denial for user without privilege: " + user); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + String body = TestUtils.getResponseBody(e.getResponse(), false); + assertTrue("Expected an error status, got " + status, status >= 400); + assertTrue( + "Response should indicate a permission denial. Status " + status + ", body: " + body, + body.contains("no permissions") + || body.contains("Forbidden") + || body.contains("security_exception") + || body.contains(deniedAction)); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index f3c3046f41e..c10bc474bae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -28,6 +28,7 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 5ca10c7091c..03d16cad083 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,6 +32,7 @@ plugins { dependencies { api project(':core') + api project(':ppl-rest-spi') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:${resilience4j_version}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java new file mode 100644 index 00000000000..9905ca564ab --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; +import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; + +/** + * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link + * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain + * {@link CalciteEnumerableCatalogScan}. + */ +public class EnumerableCatalogScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalCatalogScan.class, + s -> s.getCatalogTable() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableCatalogScanRule") + .withRuleFactory(EnumerableCatalogScanRule::new); + + protected EnumerableCatalogScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalCatalogScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; + if (scan.getCatalogTable().getSource().isScannable()) { + return new CalciteScannableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } + return new CalciteEnumerableCatalogScan( + scan.getCluster(), + scan.getHints(), + scan.getTable(), + scan.getCatalogTable(), + scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java deleted file mode 100644 index 616d1178873..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; - -/** - * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link - * CalciteEnumerableSystemIndexScan}. - */ -public class EnumerableSystemIndexScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalSystemIndexScan.class, - s -> s.getSysIndex() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableSystemIndexScanRule") - .withRuleFactory(EnumerableSystemIndexScanRule::new); - - /** Creates an EnumerableProjectRule. */ - protected EnumerableSystemIndexScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalSystemIndexScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; - return new CalciteEnumerableSystemIndexScan( - scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 3c8508cc455..c200ffa8909 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = - EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule CATALOG_SCAN_RULE = + EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - SYSTEM_INDEX_SCAN_RULE, + CATALOG_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index ffa0571a9a0..0a7cf512210 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,6 +71,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of("/_cluster/health"), + Function.identity(), + Setting.Property.NodeScope); + public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -387,6 +394,11 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); + registerNonDynamicSettings( + settingBuilder, + clusterSettings, + Key.PPL_REST_ALLOWED_ENDPOINTS, + PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -653,7 +665,9 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - latestSettings.put(key, clusterSettings.get(setting)); + if (clusterSettings.get(setting) != null) { + latestSettings.put(key, clusterSettings.get(setting)); + } } /** @@ -725,6 +739,7 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) + .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 1b7de315fb6..62ab089c1a8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; +import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -15,9 +17,14 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -35,10 +42,29 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isSystemIndex(name)) { - return new OpenSearchSystemIndex(client, settings, name); + if (isRestSource(name)) { + return restTable(name); + } else if (isSystemIndex(name)) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); } else { return new OpenSearchIndex(client, settings, name); } } + + private Table restTable(String name) { + RestSpec spec = decodeRestSpec(name); + RestEndpointRegistry registry = RestEndpointRegistryHolder.get(); + registry.resolve(spec.getEndpoint()); + List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); + if (allowed == null || !allowed.contains(spec.getEndpoint())) { + throw new IllegalArgumentException( + allowed == null || allowed.isEmpty() + ? "the rest command is disabled on this cluster" + : "rest endpoint [" + + spec.getEndpoint() + + "] is not enabled on this cluster. Enabled endpoints: " + + allowed); + } + return new OpenSearchCatalogTable(new RestCatalogSource(registry, spec, client), settings); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java new file mode 100644 index 00000000000..f746beb54a9 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProvider.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.transport.client.node.NodeClient; + +/** + * The built-in {@link RestEndpointProvider}. It ships a single read-only, in-cluster endpoint, + * {@code /_cluster/health}, expressed as a {@link RestEndpointDefinition}. It is a uniform client + * of the same SPI an external plugin uses. Additional endpoints are left to follow-up changes. + * + *

Like any external provider, it fetches at execution time through the transport node client the + * context carries ({@link RestEndpointContext#client()}), so it holds no reference to the sql + * storage client and runs under the caller's security thread-context. + * + *

It returns the full health response in a single JSON {@code response} column; a query extracts + * the fields it needs with the {@code spath} command or the {@code json_extract} function. + */ +public final class CoreEndpointsProvider implements RestEndpointProvider { + + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .argSpec(ArgSpec.builder().arg("local", Set.of("true", "false")).build()) + .handler(CoreEndpointsProvider::clusterHealth) + .build()); + } + + private static List clusterHealth(RestEndpointContext ctx) { + NodeClient client = ctx.client(); + if (client == null) { + throw new IllegalStateException( + "the /_cluster/health rest endpoint requires an in-cluster node client"); + } + ClusterHealthRequest request = new ClusterHealthRequest(); + if (Boolean.parseBoolean(ctx.args().get("local"))) { + request.local(true); + } + ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + return List.of(builder.toString()); + } catch (IOException e) { + throw new IllegalStateException("failed to serialize the /_cluster/health response", e); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java new file mode 100644 index 00000000000..e0c5c77423d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.Map; +import lombok.Getter; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.opensearch.storage.system.CatalogSource; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management + * endpoint resolved against a {@link RestEndpointRegistry} instance, exposing the fixed endpoint + * schema. Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. + */ +@Getter +public class RestCatalogSource implements CatalogSource { + + private final OpenSearchClient client; + private final RestSpec spec; + private final RestEndpointRegistry.Endpoint endpoint; + + public RestCatalogSource(RestEndpointRegistry registry, RestSpec spec, OpenSearchClient client) { + this.client = client; + this.spec = spec; + // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. + this.endpoint = registry.resolve(spec.getEndpoint()); + registry.validate(spec); + } + + @Override + public Map getFieldTypes() { + return endpoint.getSchema(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return new RestRequest(client, endpoint, spec); + } + + @Override + public boolean isScannable() { + return true; + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java new file mode 100644 index 00000000000..ac4a7c72df6 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java @@ -0,0 +1,176 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointHandler; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * The read-only endpoint allow-list, built by merging every {@link RestEndpointProvider} (the + * built-in {@link CoreEndpointsProvider} plus any externally contributed providers) into one map of + * endpoint name to an internal {@link Endpoint}. A built-in and an externally contributed endpoint + * are uniform entries here, except that a built-in name cannot be shadowed by an external provider. + * + *

This is the single place the read-only allow-list is enforced. An endpoint that no provider + * registered, including every mutating endpoint, is rejected by {@link #resolve} with a clear + * exception, and an arg the endpoint's {@link ArgSpec} does not accept is rejected by {@link + * #validate}. Adding an endpoint is a reviewed change to a provider, never arbitrary pass-through. + */ +public final class RestEndpointRegistry { + + private static final Logger LOG = LogManager.getLogger(RestEndpointRegistry.class); + + private final Map registry; + + public RestEndpointRegistry(List providers) { + Map m = new LinkedHashMap<>(); + Set disabled = new HashSet<>(); + for (RestEndpointProvider provider : providers) { + boolean builtIn = provider instanceof CoreEndpointsProvider; + for (RestEndpointDefinition definition : provider.getEndpoints()) { + String name = definition.name(); + if (disabled.contains(name)) { + continue; + } + Endpoint existing = m.get(name); + if (existing == null) { + m.put(name, new Endpoint(definition, builtIn)); + continue; + } + if (existing.isBuiltIn() || builtIn) { + LOG.warn( + "rest endpoint [{}] collides with a built-in endpoint; ignoring the duplicate from" + + " provider [{}]", + name, + provider.getClass().getName()); + continue; + } + LOG.warn( + "rest endpoint [{}] is registered by multiple external providers; disabling it." + + " Conflicting provider [{}]", + name, + provider.getClass().getName()); + m.remove(name); + disabled.add(name); + } + } + this.registry = m; + } + + /** The single column every rest endpoint surfaces: one JSON {@code response} string. */ + static final String RESPONSE_COLUMN = "response"; + + /** A single allow-listed endpoint, adapted from a provider's {@link RestEndpointDefinition}. */ + @Getter + public static final class Endpoint { + private final String path; + private final LinkedHashMap schema; + private final ArgSpec argSpec; + private final RestEndpointHandler handler; + private final boolean builtIn; + + Endpoint(RestEndpointDefinition definition, boolean builtIn) { + this.path = definition.name(); + this.schema = new LinkedHashMap<>(); + this.schema.put(RESPONSE_COLUMN, STRING); + this.argSpec = definition.argSpec(); + this.handler = definition.handler(); + this.builtIn = builtIn; + } + + /** + * Invoke the provider's handler and wrap each returned string into the single {@code response} + * column. Runs at execution time (scan open). A provider that masks sensitive values does so + * before serializing the response it returns, so the values surfaced here are already redacted. + */ + public List toRows(RestEndpointContext ctx) { + List out = new ArrayList<>(); + for (String response : handler.fetch(ctx)) { + LinkedHashMap tuple = new LinkedHashMap<>(); + tuple.put(RESPONSE_COLUMN, response == null ? ExprNullValue.of() : stringValue(response)); + out.add(new ExprTupleValue(tuple)); + } + return out; + } + } + + /** + * Resolve an allow-listed endpoint. Anything no provider registered (unknown path, mutating verb, + * {@code /services/*}, plugin admin endpoints) is refused here. + */ + public Endpoint resolve(String path) { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException( + "rest endpoint must be a non-empty path. Only the following endpoints are supported: " + + registry.keySet()); + } + Endpoint endpoint = registry.get(path); + if (endpoint == null) { + throw new IllegalArgumentException( + "rest endpoint [" + + path + + "] is not allow-listed. Only the following endpoints are supported: " + + registry.keySet()); + } + return endpoint; + } + + /** Validate the count, the reserved timeout token, and every supplied query arg. */ + public void validate(RestSpec spec) { + Endpoint endpoint = resolve(spec.getEndpoint()); + if (spec.getCount() != null && spec.getCount() < 0) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] count must be a non-negative integer, got [" + + spec.getCount() + + "]"); + } + if (spec.getTimeout() != null) { + // The timeout token is reserved in the grammar for forward compatibility, but a single + // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs + // cluster-manager vs client socket timeouts differ per action). Reject it with a clear + // client error rather than silently ignoring it. + throw new IllegalArgumentException( + "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); + } + if (spec.getArgs() != null) { + ArgSpec argSpec = endpoint.getArgSpec(); + for (String arg : spec.getArgs().keySet()) { + if (!argSpec.allows(arg)) { + throw new IllegalArgumentException( + "rest endpoint [" + + spec.getEndpoint() + + "] does not accept arg [" + + arg + + "]. Allowed args: " + + argSpec.allowedArgs()); + } + argSpec.validateValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); + } + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java new file mode 100644 index 00000000000..18669c05187 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryHolder.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +/** + * Bridge for sharing the merged {@link RestEndpointRegistry} between plugin bootstrap (where the + * SQL plugin builds it from the built-in provider plus every provider discovered via {@code + * ExtensiblePlugin.loadExtensions}) and {@code OpenSearchStorageEngine.getTable} (which resolves a + * {@code rest} endpoint at query time). + * + *

Why a static holder: {@code loadExtensions} runs during node bootstrap, before the Node-level + * Guice injector exists, so the merged registry cannot be injected into the storage engine. + * Publishing it here once at bootstrap lets the storage engine read the same instance without going + * through the injector. Mirrors {@code AnalyticsExecutorHolder}. + */ +public final class RestEndpointRegistryHolder { + + private static volatile RestEndpointRegistry registry; + + private RestEndpointRegistryHolder() {} + + public static void set(RestEndpointRegistry instance) { + registry = instance; + } + + public static RestEndpointRegistry get() { + return registry; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java new file mode 100644 index 00000000000..22f73c63005 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Dispatches an allow-listed, read-only management endpoint through the endpoint's handler under + * the caller's security thread-context and returns the response shaped to the endpoint's fixed + * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link + * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to + * the system-index scan family. This is the lazy scan: the handler runs here at {@link #search} + * (execution), never at planning time. + */ +public class RestRequest implements OpenSearchSystemRequest { + + private final OpenSearchClient client; + private final RestEndpointRegistry.Endpoint endpoint; + private final RestSpec spec; + + public RestRequest( + OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { + this.client = client; + this.endpoint = endpoint; + this.spec = spec; + } + + @Override + public List search() { + // The node transport client every provider handler fetches through, sourced from the same + // OpenSearchClient the storage engine uses so it runs under the caller's security + // thread-context. + NodeClient nodeClient = client == null ? null : client.getNodeClient().orElse(null); + RestEndpointContext ctx = RestEndpointContext.of(spec.getArgs(), nodeClient); + List rows = endpoint.toRows(ctx); + if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { + return rows.subList(0, spec.getCount()); + } + return rows; + } + + @Override + public String toString() { + return "RestRequest{endpoint=" + endpoint.getPath() + "}"; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java index fa543ab266b..19d585ea684 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ +/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ @Getter -public abstract class AbstractCalciteSystemIndexScan extends TableScan { - public final OpenSearchSystemIndex sysIndex; +public abstract class AbstractCalciteCatalogScan extends TableScan { + public final OpenSearchCatalogTable catalogTable; protected final RelDataType schema; - protected AbstractCalciteSystemIndexScan( + protected AbstractCalciteCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super(cluster, traitSet, hints, table); - this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); + this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java index b0c92dce8f9..58f86874c73 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java @@ -26,17 +26,22 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan +/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan implements EnumerableRel { - public CalciteEnumerableSystemIndexScan( + public CalciteEnumerableCatalogScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { super( - cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); + cluster, + cluster.traitSetOf(EnumerableConvention.INSTANCE), + hints, + table, + catalogTable, + schema); } @Override @@ -60,7 +65,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -68,10 +73,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchSystemIndexEnumerator( + return new OpenSearchCatalogEnumerator( getFieldPath(), - sysIndex.getSystemIndexBundle().getRight(), - sysIndex.createOpenSearchResourceMonitor()); + catalogTable.getSource().createRequest(), + catalogTable.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java index 012ceec8c13..4278539c2b0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; -/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ -public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { +/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { - public CalciteLogicalSystemIndexScan( - RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { + public CalciteLogicalCatalogScan( + RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - sysIndex, + catalogTable, table.getRowType()); } - protected CalciteLogicalSystemIndexScan( + protected CalciteLogicalCatalogScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchSystemIndex sysIndex, + OpenSearchCatalogTable catalogTable, RelDataType schema) { - super(cluster, traitSet, hints, table, sysIndex, schema); + super(cluster, traitSet, hints, table, catalogTable, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java new file mode 100644 index 00000000000..1d021cda354 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.List; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.sql.calcite.plan.Scannable; + +/** + * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, + * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via + * {@link CatalogSource#isScannable()}. + */ +public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { + public CalciteScannableCatalogScan( + RelOptCluster cluster, + List hints, + RelOptTable table, + OpenSearchCatalogTable catalogTable, + RelDataType schema) { + super(cluster, hints, table, catalogTable, schema); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java new file mode 100644 index 00000000000..85dabe06b9a --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a + * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} + * rather than defining a separate table and Calcite scan hierarchy. + */ +public interface CatalogSource { + + /** Fixed schema mapping each column name to its type. */ + Map getFieldTypes(); + + /** + * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. + */ + OpenSearchSystemRequest createRequest(); + + /** + * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} + * marker for the {@code collect} short-circuit. Defaults to {@code false}. + */ + default boolean isScannable() { + return false; + } + + /** The V2 physical plan path, or throws when the source is Calcite only. */ + PhysicalPlan implementV2(LogicalPlan plan); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java similarity index 76% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java index 1bb8f9d5293..a0e75ce9459 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java @@ -16,8 +16,11 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** Supports a simple iteration over a collection for OpenSearch system index */ -public class OpenSearchSystemIndexEnumerator implements Enumerator { +/** + * Resource-monitored iteration over the rows produced by a read-only catalog {@link + * OpenSearchSystemRequest}. + */ +public class OpenSearchCatalogEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -35,7 +38,7 @@ public class OpenSearchSystemIndexEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchSystemIndexEnumerator( + public OpenSearchCatalogEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; @@ -50,9 +53,13 @@ public OpenSearchSystemIndexEnumerator( @Override public Object current() { - return fields.stream() - .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) - .toArray(); + Object[] row = + fields.stream() + .map(k -> current.tupleValue().getOrDefault(k, ExprNullValue.of()).valueForCalcite()) + .toArray(); + // Calcite represents a single-column row as the bare scalar (the ARRAY row format optimizes to + // SCALAR for a one-field row type), so return the value directly instead of a length-one array. + return row.length == 1 ? row[0] : row; } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java new file mode 100644 index 00000000000..9bcb3a3ff0c --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint + * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a + * pluggable {@link CatalogSource}. + */ +@Getter +public class OpenSearchCatalogTable extends AbstractOpenSearchTable { + + private final CatalogSource source; + private final Settings settings; + + public OpenSearchCatalogTable(CatalogSource source, Settings settings) { + this.source = source; + this.settings = settings; + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch catalog table is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return source.getFieldTypes(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalCatalogScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return source.implementV2(plan); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java deleted file mode 100644 index 6bae4d21aba..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** OpenSearch System Index Table Implementation. */ -@Getter -public class OpenSearchSystemIndex extends AbstractOpenSearchTable { - /** System Index Name. */ - private final Pair systemIndexBundle; - - @Getter private final Settings settings; - - public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { - this.systemIndexBundle = buildIndexBundle(client, indexName); - this.settings = settings; - } - - @Override - public boolean exists() { - return true; // TODO: implement for system index later - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch system index is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return systemIndexBundle.getLeft().getMapping(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); - } - - @VisibleForTesting - @RequiredArgsConstructor - public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { - - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); - } - } - - /** - * Constructor of ElasticsearchSystemIndexName. - * - * @param indexName index name; - */ - private Pair buildIndexBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java new file mode 100644 index 00000000000..3541e745e9d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and + * index field mappings. + */ +public class SystemIndexCatalogSource implements CatalogSource { + + private final Pair bundle; + + public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { + this.bundle = buildBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return bundle.getLeft().getMapping(); + } + + @Override + public OpenSearchSystemRequest createRequest() { + return bundle.getRight(); + } + + @Override + public PhysicalPlan implementV2(LogicalPlan plan) { + return plan.accept( + new DefaultImplementor() { + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(bundle.getRight()); + } + }, + null); + } + + private static Pair buildBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 5024d416086..4d77df2c992 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,12 +9,14 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; +import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -74,6 +76,13 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } + @Test + void restSettingsAreNonDynamic() { + assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); + List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); + assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); + } + @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index fa04395e065..e883ee7b548 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,11 +7,16 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -20,8 +25,12 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; +import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -30,6 +39,13 @@ class OpenSearchStorageEngineTest { @Mock private Settings settings; + @BeforeEach + void publishRestRegistry() { + // restTable() reads the merged registry from the holder (published by SQLPlugin in production); + // publish a built-in-only registry here so the rest endpoints resolve in this unit test. + RestEndpointRegistryHolder.set(new RestEndpointRegistry(List.of(new CoreEndpointsProvider()))); + } + @Test public void getTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); @@ -52,6 +68,71 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); + assertAll( + () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); + } + + @Test + public void wildcardNoLongerEnablesEndpoints() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("*")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableAllowedBySubset() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_cluster/health")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); + assertTrue( + engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) + instanceof OpenSearchCatalogTable); + } + + @Test + public void getRestTableRejectedWhenEndpointNotInSubset() { + // The endpoint resolves in the registry but is absent from the (non-empty) enabled subset. + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) + .thenReturn(List.of("/_some/other")); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("is not enabled on this cluster")); + } + + @Test + public void getRestTableDisabledWhenListEmpty() { + when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); + OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); + String name = + SystemIndexUtils.restTable( + new SystemIndexUtils.RestSpec("/_cluster/health", Map.of(), null, null)); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + engine.getTable( + new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); + assertTrue(e.getMessage().contains("disabled on this cluster")); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java new file mode 100644 index 00000000000..2fad35408b1 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/CoreEndpointsProviderTest.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; +import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Proves the built-in {@code /_cluster/health} provider fetches through the transport node client + * the context carries (the same seam an external provider uses) and returns the response as a + * single JSON {@code response} column, holding no reference to the sql storage client. + */ +class CoreEndpointsProviderTest { + + @Test + void clusterHealthFetchesViaContextNodeClient() throws IOException { + ClusterHealthResponse response = mock(ClusterHealthResponse.class); + when(response.toXContent(any(XContentBuilder.class), any())) + .thenAnswer( + invocation -> { + XContentBuilder builder = invocation.getArgument(0); + return builder + .startObject() + .field("status", "green") + .field("number_of_nodes", 3) + .endObject(); + }); + + NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); + when(nodeClient.admin().cluster().health(any(ClusterHealthRequest.class)).actionGet()) + .thenReturn(response); + + RestEndpointRegistry registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + List rows = + registry.resolve("/_cluster/health").toRows(RestEndpointContext.of(Map.of(), nodeClient)); + + assertEquals(1, rows.size()); + String json = rows.get(0).tupleValue().get("response").stringValue(); + assertTrue(json.contains("\"status\":\"green\"")); + assertTrue(json.contains("\"number_of_nodes\":3")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java new file mode 100644 index 00000000000..e3de1b19a74 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java @@ -0,0 +1,143 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Covers the {@code rest} {@link RestCatalogSource} against the PR1 endpoint set (only {@code + * /_cluster/health}): fixed endpoint schema, allow-list enforcement, response row shaping and + * truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) path. Schema and gating + * resolve against a registry built from the built-in {@link CoreEndpointsProvider}; row shaping and + * truncation use a fake provider returning canned rows, independent of the health transport fetch. + */ +@ExtendWith(MockitoExtension.class) +class RestCatalogSourceTest { + + @Mock private OpenSearchClient client; + + private RestEndpointRegistry registry; + + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + } + + private RestSpec healthSpec() { + return new RestSpec("/_cluster/health", Map.of(), null, null); + } + + private static RestEndpointRegistry fakeHealthRegistry(List responses) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(ctx -> responses) + .build()); + return new RestEndpointRegistry(List.of(provider)); + } + + @Test + void getFieldTypesReturnsFixedEndpointSchema() { + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); + Map fieldTypes = source.getFieldTypes(); + assertThat(fieldTypes, hasEntry("response", STRING)); + } + + @Test + void isScannable() { + assertTrue(new RestCatalogSource(registry, healthSpec(), client).isScannable()); + } + + @Test + void implementV2IsUnsupported() { + RestCatalogSource source = new RestCatalogSource(registry, healthSpec(), client); + assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); + } + + @Test + void constructorRejectsNonAllowListedEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + registry, new RestSpec("/_cluster/reroute", Map.of(), null, null), client)); + } + + @Test + void constructorRejectsDisallowedArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + registry, + new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null), + client)); + } + + @Test + void constructorRejectsNegativeCount() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), -1, null), client)); + } + + @Test + void constructorRejectsTimeoutArg() { + assertThrows( + IllegalArgumentException.class, + () -> + new RestCatalogSource( + registry, new RestSpec("/_cluster/health", Map.of(), null, "5s"), client)); + } + + @Test + void restRequestShapesResponseRows() { + when(client.getNodeClient()).thenReturn(Optional.empty()); + String response = "{\"status\":\"green\",\"number_of_nodes\":1}"; + RestCatalogSource source = + new RestCatalogSource(fakeHealthRegistry(List.of(response)), healthSpec(), client); + List rows = source.createRequest().search(); + assertEquals(1, rows.size()); + assertEquals(response, rows.get(0).tupleValue().get("response").stringValue()); + } + + @Test + void countTruncatesRows() { + // count=0 exercises the truncation path (subList to empty) over a single-row response. + when(client.getNodeClient()).thenReturn(Optional.empty()); + RestCatalogSource source = + new RestCatalogSource( + fakeHealthRegistry(List.of("{\"status\":\"green\"}")), + new RestSpec("/_cluster/health", Map.of(), 0, null), + client); + assertTrue(source.createRequest().search().isEmpty()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java new file mode 100644 index 00000000000..2a950c76f52 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointExtensibilityTest.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.ArgSpec; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Proves the {@code rest} framework treats the built-in {@link CoreEndpointsProvider} and an + * externally contributed {@link RestEndpointProvider} as uniform clients of one registry: endpoints + * from BOTH resolve, validate against the same allow-list, and fetch rows the same way. The + * built-in provider holds no privileged position. + */ +class RestEndpointExtensibilityTest { + + /** A stand-in external plugin provider: contributes one endpoint that echoes a query arg. */ + private static final class FakeEchoProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_plugin/echo") + .argSpec(ArgSpec.builder().arg("text").build()) + .handler(ctx -> List.of(ctx.args().getOrDefault("text", "default"))) + .build()); + } + } + + private RestEndpointRegistry mergedRegistry() { + return new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), new FakeEchoProvider())); + } + + @Test + void bothBuiltInAndExternalEndpointsResolve() { + RestEndpointRegistry registry = mergedRegistry(); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertEquals("/_plugin/echo", registry.resolve("/_plugin/echo").getPath()); + } + + @Test + void externalEndpointFetchesThroughTheSamePath() { + RestEndpointRegistry.Endpoint echo = mergedRegistry().resolve("/_plugin/echo"); + List rows = echo.toRows(RestEndpointContext.of(Map.of("text", "hi"), null)); + assertEquals(1, rows.size()); + assertEquals("hi", rows.get(0).tupleValue().get("response").stringValue()); + } + + @Test + void externalEndpointArgsValidatedByTheSameAllowList() { + RestEndpointRegistry registry = mergedRegistry(); + // Declared arg is accepted. + registry.validate(new RestSpec("/_plugin/echo", Map.of("text", "x"), null, null)); + // Undeclared arg is rejected by the same validation path that guards built-in endpoints. + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> + registry.validate( + new RestSpec("/_plugin/echo", Map.of("not_allowed", "x"), null, null))); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void duplicateOfBuiltInNameIsDroppedSoBuiltInWins() { + // An external provider that re-declares a built-in name (/_cluster/health) is dropped rather + // than failing the build; a built-in name cannot be shadowed, so the built-in wins. + RestEndpointProvider shadowsCore = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(ctx -> List.of()) + .build()); + RestEndpointRegistry registry = + new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadowsCore)); + + assertEquals("/_cluster/health", registry.resolve("/_cluster/health").getPath()); + assertTrue(registry.resolve("/_cluster/health").isBuiltIn()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java new file mode 100644 index 00000000000..c89c8160b16 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java @@ -0,0 +1,188 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.rest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.spi.rest.RestEndpointContext; +import org.opensearch.sql.spi.rest.RestEndpointDefinition; +import org.opensearch.sql.spi.rest.RestEndpointProvider; +import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; + +/** + * Covers the {@code rest} {@link RestEndpointRegistry} against the PR1 endpoint set (only {@code + * /_cluster/health}): allow-list resolution, arg validation, count/timeout gating, and single + * response-column row shaping. Shaping is exercised through a fake provider that returns canned + * response strings, so it stays independent of how any one endpoint fetches. A provider that masks + * sensitive values does so inside its own handler, so there is no framework redaction step here. + */ +class RestEndpointRegistryTest { + + private RestEndpointRegistry registry; + + @BeforeEach + void buildRegistry() { + registry = new RestEndpointRegistry(List.of(new CoreEndpointsProvider())); + } + + private static RestEndpointContext ctx(Map args) { + return RestEndpointContext.of(args, null); + } + + private static RestEndpointRegistry.Endpoint fakeEndpoint(List responses) { + RestEndpointProvider provider = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_test/probe") + .handler(context -> responses) + .build()); + return new RestEndpointRegistry(List.of(provider)).resolve("/_test/probe"); + } + + @Test + void resolveAllowListedEndpoint() { + RestEndpointRegistry.Endpoint endpoint = registry.resolve("/_cluster/health"); + assertEquals("/_cluster/health", endpoint.getPath()); + assertEquals(STRING, endpoint.getSchema().get("response")); + assertEquals(1, endpoint.getSchema().size()); + } + + @Test + void twoExternalProvidersSameName_disableTheName() { + RestEndpointProvider a = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_plugin/dup") + .handler(c -> List.of("{\"a\":\"x\"}")) + .build()); + RestEndpointProvider b = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_plugin/dup") + .handler(c -> List.of("{\"b\":\"y\"}")) + .build()); + RestEndpointRegistry reg = new RestEndpointRegistry(List.of(a, b)); + assertThrows(IllegalArgumentException.class, () -> reg.resolve("/_plugin/dup")); + } + + @Test + void externalProviderCannotShadowBuiltIn() { + RestEndpointProvider shadow = + () -> + List.of( + RestEndpointDefinition.builder() + .name("/_cluster/health") + .handler(c -> List.of("{\"hijacked\":\"yes\"}")) + .build()); + RestEndpointRegistry reg = + new RestEndpointRegistry(List.of(new CoreEndpointsProvider(), shadow)); + RestEndpointRegistry.Endpoint health = reg.resolve("/_cluster/health"); + assertTrue(health.isBuiltIn()); + assertEquals(STRING, health.getSchema().get("response")); + } + + @Test + void resolveRejectsNonAllowListedEndpoint() { + // A mutating endpoint, and any endpoint deferred out of PR1, is simply absent and refused here. + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cluster/reroute")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/_cat/nodes")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve("/services/server/info")); + } + + @Test + void resolveRejectsBlankEndpoint() { + IllegalArgumentException emptyEx = + assertThrows(IllegalArgumentException.class, () -> registry.resolve("")); + assertTrue(emptyEx.getMessage().contains("non-empty path")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(" ")); + assertThrows(IllegalArgumentException.class, () -> registry.resolve(null)); + } + + @Test + void validateRejectsUnknownArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); + } + + @Test + void validateAcceptsAllowedArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); + registry.validate(spec); // no throw + } + + @Test + void validateRejectsDroppedLevelArg() { + // level was dropped (no-op against the fixed cluster-level health schema); now unknown. + RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); + assertTrue(ex.getMessage().contains("does not accept arg")); + } + + @Test + void validateRejectsBadArgValue() { + IllegalArgumentException local = + assertThrows( + IllegalArgumentException.class, + () -> + registry.validate( + new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); + assertTrue(local.getMessage().contains("unsupported value")); + } + + @Test + void validateRejectsNegativeCount() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), -1, null); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); + assertTrue(ex.getMessage().contains("non-negative")); + } + + @Test + void validateAcceptsZeroCount() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), 0, null); + registry.validate(spec); // no throw: 0 is a valid limit + } + + @Test + void validateRejectsTimeoutArg() { + RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); + IllegalArgumentException ex = + assertThrows(IllegalArgumentException.class, () -> registry.validate(spec)); + assertTrue(ex.getMessage().contains("timeout")); + } + + @Test + void rowsAreWrappedIntoResponseColumn() { + String response = "{\"status\":\"green\",\"number_of_nodes\":1}"; + RestEndpointRegistry.Endpoint endpoint = fakeEndpoint(List.of(response)); + List rows = endpoint.toRows(ctx(Map.of())); + assertEquals(1, rows.size()); + assertEquals(STRING, endpoint.getSchema().get("response")); + assertEquals(response, rows.get(0).tupleValue().get("response").stringValue()); + } + + @Test + void nullResponseBecomesNull() { + List withNull = new ArrayList<>(); + withNull.add(null); + List rows = fakeEndpoint(withNull).toRows(ctx(Map.of())); + assertEquals(1, rows.size()); + assertTrue(rows.get(0).tupleValue().get("response").isNull()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java index 0b0aa1ec521..df81225afbe 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java @@ -32,8 +32,12 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; +/** + * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: + * schema delegation, the predefined-table contract, and the V2 physical path. + */ @ExtendWith(MockitoExtension.class) -class OpenSearchSystemIndexTest { +class OpenSearchCatalogTableTest { @Mock private OpenSearchClient client; @@ -41,36 +45,37 @@ class OpenSearchSystemIndexTest { @Mock private Settings settings; + private OpenSearchCatalogTable systemTable(String name) { + return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + } + @Test void testGetFieldTypesOfMetaTable() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - OpenSearchSystemIndex systemIndex = - new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); - final Map fieldTypes = systemIndex.getFieldTypes(); + final Map fieldTypes = + systemTable(mappingTable("test_index")).getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); - assertTrue(systemIndex.exists()); + assertTrue(systemTable(TABLE_INFO).exists()); } @Test void testCreateTable() { - Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + Table systemIndex = systemTable(TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 214c442755f..31f14e3411b 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -105,6 +105,9 @@ import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; +import org.opensearch.sql.opensearch.storage.rest.CoreEndpointsProvider; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; +import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistryHolder; import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine; import org.opensearch.sql.plugin.config.EngineExtensionsHolder; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; @@ -136,6 +139,7 @@ import org.opensearch.sql.spark.transport.model.CancelAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.CreateAsyncQueryActionResponse; import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse; +import org.opensearch.sql.spi.rest.RestEndpointProvider; import org.opensearch.sql.sql.domain.SQLQueryRequest; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.threadpool.ExecutorBuilder; @@ -155,6 +159,7 @@ public class SQLPlugin extends Plugin private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class); private List executionEngineExtensions = List.of(); + private List restEndpointProviders = List.of(); private ClusterService clusterService; /** Settings should be inited when bootstrap the plugin. */ @@ -183,6 +188,16 @@ public void loadExtensions(ExtensionLoader loader) { executionEngineExtensions.size(), executionEngineExtensions.stream().map(e -> e.getClass().getSimpleName()).toList()); } + + List restProviders = loader.loadExtensions(RestEndpointProvider.class); + this.restEndpointProviders = restProviders != null ? List.copyOf(restProviders) : List.of(); + } + + private void publishRestCommandRegistries() { + List providers = new ArrayList<>(); + providers.add(new CoreEndpointsProvider()); + providers.addAll(this.restEndpointProviders); + RestEndpointRegistryHolder.set(new RestEndpointRegistry(providers)); } @Override @@ -379,6 +394,9 @@ public Collection createComponents( this.clusterService = clusterService; this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); this.client = (NodeClient) client; + + publishRestCommandRegistries(); + this.dataSourceService = createDataSourceService(); dataSourceService.createDataSource(defaultOpenSearchDataSourceMetadata()); LocalClusterState.state().setClusterService(clusterService); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 26c71060c2e..609b4a71099 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -110,13 +110,14 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + // Analytics engine serves neither the system catalog nor the rest command's reserved + // in-cluster source; both fall back to the default (Calcite) pipeline. try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean systemCatalog = + boolean defaultPipeline = extractIndexName(query, queryType, context) - .map(RestUnifiedQueryAction::isSystemCatalog) + .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) .orElse(false); - return !systemCatalog; + return !defaultPipeline; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index de1ce111558..516c31940c3 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -179,6 +179,12 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); + } + @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl-rest-spi/README.md b/ppl-rest-spi/README.md new file mode 100644 index 00000000000..2b7caacaaaa --- /dev/null +++ b/ppl-rest-spi/README.md @@ -0,0 +1,171 @@ +# ppl-rest-spi + +A generic way for any OpenSearch plugin to integrate with PPL: contribute your own read-only data +as a table queryable via `rest ''`, without adding a new PPL command or grammar keyword and +without a compile dependency on the sql plugin's internals. Each contributed endpoint composes with +ordinary PPL (`| where`, `| stats`, `| head`), so a plugin extends the query surface without any +change to the language itself. + +This module is intentionally thin: it depends only on OpenSearch core, and the row values it +exchanges are plain `java.lang` types (String / Number / Boolean / nested Map), so there is no +cross-classloader type-identity problem. Whatever type a handler returns, PPL surfaces it as a +string column, and a query casts the fields it needs. + +## Example: `/_cluster/health` + +The built-in `/_cluster/health` endpoint is implemented against this SPI exactly like an external +provider would. It declares a single `response` column and, at execution time, calls the +cluster-health transport action and serializes the whole response into that column: + +```java +RestEndpointDefinition.builder() + .name("/_cluster/health") + .argSpec(ArgSpec.builder().arg("local", Set.of("true", "false")).build()) + .handler(ctx -> { + ClusterHealthResponse health = + ctx.client().admin().cluster().health(new ClusterHealthRequest()).actionGet(); + XContentBuilder json = XContentFactory.jsonBuilder(); + health.toXContent(json, ToXContent.EMPTY_PARAMS); // serialize the full response as-is + return List.of(json.toString()); + }) + .build(); +``` + +Querying it returns one row whose `response` column holds the full health JSON: + +``` +> rest '/_cluster/health' + +response +-------------------------------------------------------------------------------------------- +{"cluster_name":"opensearch-cluster","status":"green","timed_out":false,"number_of_nodes":1, +"number_of_data_nodes":1,"discovered_cluster_manager":true,"active_primary_shards":0, +"active_shards":0,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":0, +"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0, +"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":100.0} +``` + +Pull individual fields downstream with `spath` (or `json_extract`), casting where a numeric type +is needed: + +``` +> rest '/_cluster/health' | spath input=response path=status output=status | fields status + +status +------ +green +``` + +``` +> rest '/_cluster/health' + | spath input=response path=number_of_nodes output=nodes + | where cast(nodes as int) >= 1 + | fields nodes + +nodes +----- +1 +``` + +## Contract + +| Type | Role | +|---|---| +| `RestEndpointProvider` | Your entry point: `List getEndpoints()`. | +| `RestEndpointDefinition` | One endpoint: `name()`, `argSpec()`, `handler()`. Build with `RestEndpointDefinition.builder()`. Every endpoint surfaces a single `response` string column. | +| `ArgSpec` | The query args the endpoint accepts and each arg's allowed value domain; an unknown or out-of-domain arg is rejected. `ArgSpec.NONE` accepts none. | +| `RestEndpointHandler` | `List fetch(RestEndpointContext)`: each string is one row's `response` cell (typically a serialized JSON document). Runs at scan execution (not planning), so the scan is lazy and `EXPLAIN` is side-effect free. | +| `RestEndpointContext` | The validated `args()` plus an optional core `NodeClient` (`client()`) for a handler that issues its own read-only transport action. | + +## Add an endpoint + +1. Depend on the published `ppl-rest-spi` artifact `compileOnly` (the installed sql plugin + provides it at runtime, so it is never bundled), and declare the sql plugin as an extended + plugin so `loadExtensions` discovers your provider: + + ```gradle + dependencies { + compileOnly "org.opensearch.query:ppl-rest-spi:${sqlPluginVersion}" + } + opensearchplugin { extendedPlugins = ['opensearch-sql'] } + ``` + +2. Implement `RestEndpointProvider`: + + ```java + public final class MyRestProvider implements RestEndpointProvider { + @Override + public List getEndpoints() { + return List.of( + RestEndpointDefinition.builder() + .name("/_my/thing") + .argSpec(ArgSpec.builder().arg("verbose", Set.of("true", "false")).build()) + .handler(MyRestProvider::fetch) + .build()); + } + + private static List fetch(RestEndpointContext ctx) { + // ctx.args() is already validated; ctx.client() is a NodeClient for transport calls (may be null). + MyThing thing = readMyThing(ctx); // your own read-only transport call + XContentBuilder json = XContentFactory.jsonBuilder(); + thing.toXContent(json, ToXContent.EMPTY_PARAMS); // serialize the whole response + return List.of(json.toString()); + } + } + ``` + +3. Register the provider as a service in + `META-INF/services/org.opensearch.sql.spi.rest.RestEndpointProvider`, containing your class's + fully-qualified name. + +4. Add the endpoint name to the sql plugin's default allow list. Steps 1 to 3 only *register* the + provider, which does not make the endpoint queryable on its own. A name is queryable only when it + is also present in `plugins.ppl.rest.allowed_endpoints`. Enable your endpoint by submitting a + change to the sql plugin that adds its name to the default list in `OpenSearchSettings.java`: + + ```java + public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = + Setting.listSetting( + Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), + List.of("/_cluster/health", "/_my/thing"), // add your endpoint name here + Function.identity(), + Setting.Property.NodeScope); + ``` + + Once that change is merged and released, the endpoint is enabled by default on every cluster + running that sql version. The sql maintainers' review of this change is the gate that decides + which endpoint names PPL is allowed to expose. + +Then `rest '/_my/thing' verbose=true | spath input=response path=count output=count | where cast(count as int) > 0` composes like any scan; pull fields out of the `response` JSON with `spath` (or `json_extract`) and cast the ones you compute on. + +## Rules and guarantees + +- Endpoints are read-only. The handler produces rows; every value is surfaced as a string column, + and a query extracts and casts the fields it needs (for example with `json_extract` or `spath`). +- `plugins.ppl.rest.allowed_endpoints` is the enable list, whose default is maintained in the sql + plugin's `OpenSearchSettings.java`: a name is queryable only when it is both registered by a + provider and listed there; anything else is rejected before any transport call. Listing a name no + provider registered has no effect. +- Endpoint names are global across all providers, and `allowed_endpoints` does not resolve name + collisions: it only enables names, it does not make two providers that claim the same name + coexist. Collisions are handled separately, at registration time: a built-in name cannot be + shadowed by an external provider (the built-in wins and the external duplicate is dropped with a + logged warning), and if two external providers register the same name that name is disabled + entirely (logged warning, and the node still starts). So even when every name is listed in + `allowed_endpoints`, a name owned by two providers will not silently pick a winner. +- Redaction is the endpoint's own responsibility, applied to the response before it is streamed + back. The framework surfaces exactly what the handler returns and adds no masking step, so there + is no central redaction seam to implement and an endpoint with nothing sensitive does nothing. + Because an endpoint returns a single `response` JSON column, redact the sensitive fields (IPs, + hostnames, tokens) on the response object *before* serializing it into that column, so the masked + value is what is streamed out: + + ```java + .handler(ctx -> { + MyStats stats = fetchStats(ctx); // your own read-only transport call + stats.maskIp(); // redact on the object first + XContentBuilder json = XContentFactory.jsonBuilder(); + stats.toXContent(json, ToXContent.EMPTY_PARAMS); // then serialize the masked response + return List.of(json.toString()); + }) + ``` diff --git a/ppl-rest-spi/build.gradle b/ppl-rest-spi/build.gradle new file mode 100644 index 00000000000..d0c6ccd5f4f --- /dev/null +++ b/ppl-rest-spi/build.gradle @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * The `rest` command extension SPI. + * + * This module is intentionally thin: it declares ONLY the interfaces a plugin implements to + * contribute read-only `rest` endpoints, plus the JDK-typed row/column/arg value objects those + * interfaces exchange. It depends on the OpenSearch core artifact ONLY (for the transport client + * handed to a handler) and on NO sql module, so external plugins can compileOnly it without a + * dependency cycle and without a cross-classloader type-identity problem (row values are plain + * java.lang types). + */ + +plugins { + id 'java-library' + id "io.freefair.lombok" + id 'com.diffplug.spotless' +} + +dependencies { + // Core only. Deliberately no `project(':core')` / `project(':opensearch')` dependency so the + // sql plugin (which depends on this module) never forms a cycle, and so an external plugin can + // depend on this module alone. + compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + + testImplementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" + testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.3') + testImplementation('org.junit.jupiter:junit-jupiter-params:5.9.3') + testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine') + testRuntimeOnly('org.junit.platform:junit-platform-launcher') +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + } +} + +// Published as a standalone Maven artifact so an external plugin can depend on the SPI +// `compileOnly`. Intentionally NOT added to the root `publishedModules` list, which would prefix +// the artifactId with `unified-query-`; this is a public extension SPI, so it keeps the plain +// `ppl-rest-spi` artifactId. +apply plugin: 'maven-publish' + +publishing { + publications { + restSpi(MavenPublication) { + from components.java + groupId = "org.opensearch.query" + artifactId = "ppl-rest-spi" + + pom { + name = "ppl-rest-spi" + description = "OpenSearch PPL rest command extension SPI" + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + developers { + developer { + name = 'OpenSearch' + url = 'https://github.com/opensearch-project/sql' + } + } + } + } + } + + repositories { + maven { + name = "Snapshots" + url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/" + url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO") + credentials(AwsCredentials) { + accessKey = System.getenv("AWS_ACCESS_KEY_ID") + secretKey = System.getenv("AWS_SECRET_ACCESS_KEY") + sessionToken = System.getenv("AWS_SESSION_TOKEN") + } + } + } +} + +spotless { + java { + target fileTree('.') { + include '**/*.java' + exclude '**/build/**', '**/build-*/**' + } + importOrder() + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") + removeUnusedImports() + trimTrailingWhitespace() + endWithNewline() + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java new file mode 100644 index 00000000000..003c56cca6a --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/ArgSpec.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * The query args a {@code rest} endpoint accepts, with the allowed value domain of each. An + * undeclared arg is rejected; a declared arg whose value is outside its domain is rejected; an + * empty domain accepts any value. + */ +public final class ArgSpec { + + public static final ArgSpec NONE = builder().build(); + + // arg name -> allowed values (empty set means "any value allowed"). + private final Map> valueDomains; + + private ArgSpec(Map> valueDomains) { + this.valueDomains = valueDomains; + } + + public Set allowedArgs() { + return valueDomains.keySet(); + } + + public boolean allows(String arg) { + return valueDomains.containsKey(arg); + } + + /** Validate an accepted arg's value against its domain (no-op if the domain is empty). */ + public void validateValue(String endpoint, String arg, String value) { + Set domain = valueDomains.get(arg); + if (domain == null || domain.isEmpty()) { + return; + } + if (value == null || !domain.contains(value.toLowerCase(Locale.ROOT))) { + throw unsupported(endpoint, arg, value, domain); + } + } + + private static IllegalArgumentException unsupported( + String endpoint, String arg, String value, Set domain) { + return new IllegalArgumentException( + "rest endpoint [" + + endpoint + + "] arg [" + + arg + + "] has an unsupported value [" + + value + + "]. Allowed values: " + + domain); + } + + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link ArgSpec}. Declaration order is preserved for stable error messages. */ + public static final class Builder { + private final LinkedHashMap> valueDomains = new LinkedHashMap<>(); + + public Builder arg(String name) { + valueDomains.put(name, Set.of()); + return this; + } + + public Builder arg(String name, Set domain) { + valueDomains.put(name, Set.copyOf(domain)); + return this; + } + + public ArgSpec build() { + return new ArgSpec(new LinkedHashMap<>(valueDomains)); + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java new file mode 100644 index 00000000000..3cf78911deb --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointContext.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.Map; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Per-invocation context handed to {@link RestEndpointHandler#fetch} at execution time (scan open), + * carrying the validated query args and the node transport client a provider may use for its own + * read-only transport action. A provider that already holds a client may ignore {@link #client()}. + */ +public interface RestEndpointContext { + + /** Validated query args for this invocation (never null; empty when none supplied). */ + Map args(); + + /** Node transport client for a provider that issues its own transport action; may be null. */ + NodeClient client(); + + static RestEndpointContext of(Map args, NodeClient client) { + Map safeArgs = args == null ? Map.of() : args; + return new RestEndpointContext() { + @Override + public Map args() { + return safeArgs; + } + + @Override + public NodeClient client() { + return client; + } + }; + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java new file mode 100644 index 00000000000..ca74f768966 --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointDefinition.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.Objects; + +/** + * One read-only {@code rest} endpoint from a {@link RestEndpointProvider}: a unique name (the token + * after {@code rest}, e.g. {@code /_cluster/health}), the {@link ArgSpec} it accepts, and the + * {@link RestEndpointHandler} that produces its rows. Every endpoint surfaces a single {@code + * response} string column; a query extracts the fields it needs with {@code spath} or {@code + * json_extract}. A provider that needs to mask sensitive values does so inside its handler before + * returning the response. Immutable; build with {@link #builder()}. + */ +public interface RestEndpointDefinition { + + String name(); + + ArgSpec argSpec(); + + RestEndpointHandler handler(); + + static Builder builder() { + return new Builder(); + } + + final class Builder { + private String name; + private ArgSpec argSpec = ArgSpec.NONE; + private RestEndpointHandler handler; + + private Builder() {} + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder argSpec(ArgSpec argSpec) { + this.argSpec = argSpec; + return this; + } + + public Builder handler(RestEndpointHandler handler) { + this.handler = handler; + return this; + } + + public RestEndpointDefinition build() { + String endpointName = Objects.requireNonNull(name, "rest endpoint name is required"); + ArgSpec spec = Objects.requireNonNull(argSpec, "argSpec is required"); + RestEndpointHandler endpointHandler = Objects.requireNonNull(handler, "handler is required"); + return new RestEndpointDefinition() { + @Override + public String name() { + return endpointName; + } + + @Override + public ArgSpec argSpec() { + return spec; + } + + @Override + public RestEndpointHandler handler() { + return endpointHandler; + } + }; + } + } +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java new file mode 100644 index 00000000000..ed94d02441b --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointHandler.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; + +/** + * Produces the rows of a {@code rest} endpoint. Invoked at execution time (scan open), never at + * planning, so the scan stays lazy and EXPLAIN is side-effect free; a transport-backed provider may + * block on {@code ctx.client().execute(...).actionGet()} here. Each row is one string, surfaced as + * the single {@code response} column, typically a serialized JSON document; a query extracts and + * casts the fields it needs with {@code spath} or {@code json_extract}. A provider that must mask + * sensitive values does so before serializing the response it returns here. + */ +@FunctionalInterface +public interface RestEndpointHandler { + + /** + * Fetch the rows for one invocation. + * + * @param ctx the validated query args and (optional) transport client for this invocation + * @return one response string per row (never null; empty when there are no rows) + */ + List fetch(RestEndpointContext ctx); +} diff --git a/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java new file mode 100644 index 00000000000..db04c016f8f --- /dev/null +++ b/ppl-rest-spi/src/main/java/org/opensearch/sql/spi/rest/RestEndpointProvider.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spi.rest; + +import java.util.List; + +/** + * The extension point a plugin implements to contribute read-only {@code rest} endpoints, via + * {@code ExtensiblePlugin.loadExtensions(RestEndpointProvider.class)}. The sql plugin merges every + * discovered provider with its own built-in one into a single registry, so built-in and external + * endpoints are uniform clients of the same contract. A provider declares data (name, schema, args) + * and a handler; it never touches the PPL grammar. + */ +public interface RestEndpointProvider { + + /** The endpoints this provider contributes. Called once when the registry is built. */ + List getEndpoints(); +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index fb072ae134f..6f21942c9c2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,6 +13,8 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +REST: 'REST'; +TIMEOUT: 'TIMEOUT'; MAKERESULTS: 'MAKERESULTS'; FORMAT: 'FORMAT'; CSV: 'CSV'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a23a314537d..7ad26e09850 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,6 +45,7 @@ subSearch // commands pplCommands : describeCommand + | restCommand | showDataSourcesCommand | makeresultsCommand | searchCommand @@ -107,6 +108,7 @@ commands commandName : SEARCH | DESCRIBE + | REST | SHOW | WHERE | FIELDS @@ -214,6 +216,16 @@ describeCommand : DESCRIBE tableSourceClause ; + +restCommand + : REST stringLiteral (restArgument)* + ; + +restArgument + : COUNT EQUAL integerLiteral + | TIMEOUT EQUAL stringLiteral + | ident EQUAL literalValue + ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -1871,5 +1883,7 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE + // rest command token, also usable as a free-text search term / identifier + | TIMEOUT | SEP ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index efdbf26a205..7aca48a4666 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -114,6 +114,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -149,6 +150,7 @@ import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.MakeResultsDataParser; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; +import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -260,6 +262,34 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } + /** + * Rest command.
+ * Encodes the validated endpoint spec into a reserved table name (via {@link + * org.opensearch.sql.utils.SystemIndexUtils#restTable}) that resolves through the storage engine + * to a REST source table on the Calcite path, mirroring how DESCRIBE resolves to a system index. + */ + @Override + public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { + String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); + LinkedHashMap args = new LinkedHashMap<>(); + Integer count = null; + String timeout = null; + for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { + if (arg.COUNT() != null) { + count = Integer.parseInt(arg.integerLiteral().getText()); + } else if (arg.TIMEOUT() != null) { + timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); + } else { + args.put( + StringUtils.unquoteIdentifier(arg.ident().getText()), + StringUtils.unquoteText(arg.literalValue().getText())); + } + } + String token = + SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); + return new RestRelation(new QualifiedName(token)); + } + /** makeresults command. */ @Override public UnresolvedPlan visitMakeresultsCommand(OpenSearchPPLParser.MakeresultsCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 398361fcea7..21692bbda87 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -97,6 +97,7 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; +import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -125,6 +126,7 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -168,6 +170,23 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { + if (node instanceof RestRelation) { + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); + StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); + if (spec.getCount() != null) { + sb.append(" count=").append(MASK_LITERAL); + } + if (spec.getTimeout() != null) { + sb.append(" timeout=").append(MASK_LITERAL); + } + if (spec.getArgs() != null) { + for (String key : spec.getArgs().keySet()) { + sb.append(' ').append(key).append('=').append(MASK_LITERAL); + } + } + return sb.toString(); + } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java new file mode 100644 index 00000000000..60a81d90b23 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +import org.junit.Test; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.tree.Project; +import org.opensearch.sql.ast.tree.RestRelation; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.ppl.parser.AstBuilder; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. + * + *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> + * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code + * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the + * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan + * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and + * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the + * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a + * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides + * {@code visitRelation} exactly like {@code DESCRIBE}. + */ +public class CalcitePPLRestTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + private final Settings settings = mock(Settings.class); + + private Node parse(String ppl) { + return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); + } + + @Test + public void restHealthProjectsDeclaredColumns() { + Project project = (Project) parse("| rest \"/_cluster/health\" | fields response"); + RestRelation rest = (RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + // downstream fields compose on top of the rest row source. + assertEquals(1, project.getProjectList().size()); + } + + @Test + public void restReservedNameRoundTrips() { + RestRelation rest = + (RestRelation) + parse("| rest \"/_cluster/health\" count=10 timeout=\"5s\" health=\"green\""); + String reserved = rest.getTableQualifiedName().toString(); + assertTrue(SystemIndexUtils.isRestSource(reserved)); + SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(10), spec.getCount()); + assertEquals("5s", spec.getTimeout()); + assertEquals("green", spec.getArgs().get("health")); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 0650b4d1e4c..b16b44f1c92 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -2019,4 +2019,32 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } + + // rest command tests + + @Test + public void testRestCommand() { + org.opensearch.sql.ast.tree.Project project = + (org.opensearch.sql.ast.tree.Project) + plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertTrue(spec.getArgs().isEmpty()); + } + + @Test + public void testRestCommandWithArgs() { + org.opensearch.sql.ast.tree.RestRelation rest = + (org.opensearch.sql.ast.tree.RestRelation) + plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); + SystemIndexUtils.RestSpec spec = + SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); + assertEquals("/_cluster/health", spec.getEndpoint()); + assertEquals(Integer.valueOf(5), spec.getCount()); + assertEquals("30s", spec.getTimeout()); + assertEquals("indices", spec.getArgs().get("level")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 63f7dea81d7..723d2859891 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -53,6 +53,18 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } + @Test + public void testRestCommand() { + assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); + } + + @Test + public void testRestCommandMasksArgValues() { + assertEquals( + "rest /_cluster/health count=*** timeout=*** level=***", + anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); + } + @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); diff --git a/settings.gradle b/settings.gradle index 5f83c4f5a87..5e04e144468 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ include 'opensearch-sql-plugin' project(':opensearch-sql-plugin').projectDir = file('plugin') include 'api' include 'ppl' +include 'ppl-rest-spi' include 'common' include 'opensearch' include 'core' From 2d1256f0e25701a164e57140eba85e7855f2664f Mon Sep 17 00:00:00 2001 From: Ajimelec <144399074+AjimelecGonzalez@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:06:43 -0700 Subject: [PATCH 19/78] [Feature] `percentfield` and `showperc` with `top` and `rare` commands. (#5642) * Clean docs * Fix CalcitePPLRareTopNTest showperc tests * Calculate percentages before filtering * Added percentfield and changed decimal places from 2 to 6 * Update test errors, SUM to CHECKED_LONG_SUM * Remove percent field check Signed-off-by: Ajimelec Gonzalez --- .../org/opensearch/sql/ast/tree/RareTopN.java | 2 + .../sql/calcite/CalciteRelNodeVisitor.java | 39 +- docs/user/ppl/cmd/rare.md | 51 ++- docs/user/ppl/cmd/top.md | 49 ++- .../calcite/remote/CalciteTopCommandIT.java | 39 ++ .../org/opensearch/sql/ppl/TopCommandIT.java | 16 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 + .../sql/ppl/utils/ArgumentFactory.java | 12 + .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 5 +- .../sql/ppl/antlr/PPLSyntaxParserTest.java | 38 ++ .../ppl/calcite/CalcitePPLRareTopNTest.java | 401 ++++++++++++++++++ .../sql/ppl/parser/AstBuilderTest.java | 90 ++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 26 +- 14 files changed, 762 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java index 6c543ddc8c3..8055c4f92d8 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/RareTopN.java @@ -56,6 +56,8 @@ public enum CommandType { public enum Option { countField, showCount, + percentField, + showPerc, useNull, } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index b7b52e16d82..e1f2e666c86 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -223,6 +223,8 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | -| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `rare-options` | Optional | Additional options for controlling output:
- `showcount`: Whether to create a field in the output containing the frequency count for each combination of values. Default is `true`.
- `countfield`: The name of the field that contains the count. Default is `count`.
- `percentfield`: The name of the field that contains the percentage. Default is `percent`.
- `showperc`: Whether to create a field in the output containing the percentage of each value's count relative to the total. Default is `false`.
- `usenull`: Whether to output null values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | ## Example 1: Finding the least common values without showing counts @@ -125,7 +125,52 @@ fetched rows / total rows = 4/4 +--------------+-----+ ``` -## Example 5: Specifying null value handling +## Example 5: Displaying percentages + + The following query finds the least common severity levels and shows what percentage each represents: + +```ppl +source=otellogs +| rare showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+---------+ +``` + +## Example 6: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| rare showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| DEBUG | 3 | 15.0 | +| WARN | 4 | 20.0 | +| INFO | 6 | 30.0 | +| ERROR | 7 | 35.0 | ++--------------+-------+------+ +``` + +## Example 7: Specifying null value handling The following query uses `usenull=false` to exclude null values: @@ -166,4 +211,4 @@ fetched rows / total rows = 4/4 | @opentelemetry/instrumentation-http | 2 | | null | 16 | +-----------------------------------------------------------------------------+-------+ -``` \ No newline at end of file +``` diff --git a/docs/user/ppl/cmd/top.md b/docs/user/ppl/cmd/top.md index 678b62354a0..162ad56e905 100644 --- a/docs/user/ppl/cmd/top.md +++ b/docs/user/ppl/cmd/top.md @@ -20,7 +20,7 @@ The `top` command supports the following parameters. | Parameter | Required/Optional | Description | | --- | --- | --- | | `` | Optional | The number of results to return. Default is `10`. | -| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | +| `top-options` | Optional | `showcount`: Whether to create a field in the output that represents a count of the tuple of values. Default is `true`.
`countfield`: The name of the field that contains the count. Default is `count`.
`percentfield`: The name of the field that contains the percentage. Default is `percent`.
`showperc`: Whether to create a field in the output that represents the percentage of the count relative to the total. Default is `false`.
`usenull`: Whether to output `null` values. Default is the value of `plugins.ppl.syntax.legacy.preferred`. | | `` | Required | A comma-delimited list of field names. | | `` | Optional | One or more fields to group the results by. | @@ -139,7 +139,52 @@ fetched rows / total rows = 7/7 +----------------------------------+--------------+ ``` -## Example 6: Specifying null value handling +## Example 6: Displaying percentages + +The following query finds the most common severity levels and shows the percentage each represents: + +```ppl +source=otellogs +| top showperc=true severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+---------+ +| severityText | count | percent | +|--------------+-------+---------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+---------+ +``` + +## Example 7: Customizing the percent field name +The following query uses `percentfield` to rename the percentage column from the default `percent` to `pct`: + +```ppl +source=otellogs +| top showperc=true percentfield='pct' severityText +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++--------------+-------+------+ +| severityText | count | pct | +|--------------+-------+------| +| ERROR | 7 | 35.0 | +| INFO | 6 | 30.0 | +| WARN | 4 | 20.0 | +| DEBUG | 3 | 15.0 | ++--------------+-------+------+ +``` + +## Example 8: Specifying null value handling The following query specifies `usenull=false` to exclude null values: diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java index e555576a9cd..78cc4cbee73 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteTopCommandIT.java @@ -6,7 +6,9 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; @@ -41,6 +43,43 @@ public void testTopCommandUseNullFalse() throws IOException { verifyNumOfRows(result, 5); } + @Test + public void testTopCommandShowPerc() throws IOException { + JSONObject result = + executeQuery( + String.format("source=%s | top showperc=true age", TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder( + result, schema("age", "int"), schema("count", "bigint"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 2, 28.571429), + rows(28, 1, 14.285714), + rows(32, 1, 14.285714), + rows(33, 1, 14.285714), + rows(34, 1, 14.285714), + rows(null, 1, 14.285714)); + } + + @Test + public void testTopCommandShowPercWithoutShowCount() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | top showperc=true showcount=false age", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifySchemaInOrder(result, schema("age", "int"), schema("percent", "double")); + verifyNumOfRows(result, 6); + verifyDataRows( + result, + rows(36, 28.571429), + rows(28, 14.285714), + rows(32, 14.285714), + rows(33, 14.285714), + rows(34, 14.285714), + rows(null, 14.285714)); + } + @Test public void testTopCommandLegacyFalse() throws IOException { withSettings( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java index 936e728bd10..8121ce0ea6b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TopCommandIT.java @@ -58,4 +58,20 @@ public void testTopNWithGroup() throws IOException { verifyDataRows(result, rows("F", "TX"), rows("M", "MD")); } } + + @Test + public void testTopWithShowPerc() throws IOException { + JSONObject result = + executeQuery(String.format("source=%s | top showperc=true gender", TEST_INDEX_ACCOUNT)); + if (isCalciteEnabled()) { + verifySchemaInOrder( + result, + schema("gender", "string"), + schema("count", "bigint"), + schema("percent", "double")); + verifyDataRows(result, rows("M", 507, 50.7), rows("F", 493, 49.3)); + } else { + verifyDataRows(result, rows("M"), rows("F")); + } + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 6f21942c9c2..eb3f0bb7eaa 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -190,6 +190,8 @@ UNION: 'UNION'; MAXOUT: 'MAXOUT'; COUNTFIELD: 'COUNTFIELD'; SHOWCOUNT: 'SHOWCOUNT'; +PERCENTFIELD: 'PERCENTFIELD'; +SHOWPERC: 'SHOWPERC'; LIMIT: 'LIMIT'; USEOTHER: 'USEOTHER'; OTHERSTR: 'OTHERSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7ad26e09850..ce949f8534b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -501,6 +501,8 @@ rareTopCommand rareTopOption : COUNTFIELD EQUAL countField = stringLiteral | SHOWCOUNT EQUAL showCount = booleanLiteral + | PERCENTFIELD EQUAL percentField = stringLiteral + | SHOWPERC EQUAL showPerc = booleanLiteral | USENULL EQUAL useNull = booleanLiteral ; @@ -1818,6 +1820,8 @@ searchableKeyWord | ANOMALY_SCORE_THRESHOLD | COUNTFIELD | SHOWCOUNT + | PERCENTFIELD + | SHOWPERC | MAXOUT | PATH | INPUT diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java index 2cdc702b785..3aba4faf99c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/ArgumentFactory.java @@ -307,6 +307,18 @@ public static List getArgumentList( new Argument( RareTopN.Option.showCount.name(), opt.isPresent() ? getArgumentValue(opt.get().showCount) : Literal.TRUE)); + opt = ctx.rareTopOption().stream().filter(op -> op.percentField != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.percentField.name(), + opt.isPresent() + ? getArgumentValue(opt.get().percentField) + : new Literal("percent", DataType.STRING))); + opt = ctx.rareTopOption().stream().filter(op -> op.showPerc != null).findFirst(); + list.add( + new Argument( + RareTopN.Option.showPerc.name(), + opt.isPresent() ? getArgumentValue(opt.get().showPerc) : Literal.FALSE)); opt = ctx.rareTopOption().stream().filter(op -> op.useNull != null).findFirst(); list.add( new Argument( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 21692bbda87..1ccd9561e25 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -479,13 +479,16 @@ public String visitRareTopN(RareTopN node, String context) { Integer noOfResults = node.getNoOfResults(); String countField = (String) arguments.get(RareTopN.Option.countField.name()).getValue(); Boolean showCount = (Boolean) arguments.get(RareTopN.Option.showCount.name()).getValue(); + String percentField = (String) arguments.get(RareTopN.Option.percentField.name()).getValue(); + Boolean showPerc = (Boolean) arguments.get(RareTopN.Option.showPerc.name()).getValue(); Boolean useNull = (Boolean) arguments.get(RareTopN.Option.useNull.name()).getValue(); String fields = visitFieldList(node.getFields()); String group = visitExpressionList(node.getGroupExprList()); String options = UnresolvedPlanHelper.isCalciteEnabled(settings) ? StringUtils.format( - "countield='%s' showcount=%s usenull=%s ", countField, showCount, useNull) + "countfield='%s' showcount=%s percentfield='%s' showperc=%s usenull=%s ", + countField, showCount, percentField, showPerc, useNull) : ""; return StringUtils.format( "%s | %s %d %s%s", diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 0c5ad9dd4fe..42abefc8dc6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -398,6 +398,25 @@ public void testRareCommandWithGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testRareCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | rare showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | rare showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testRareCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | rare showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testTopCommandWithoutNShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("source=t a=1 | top a"); @@ -422,6 +441,25 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { assertNotEquals(null, tree); } + @Test + public void testTopCommandWithShowPercShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithShowPercAndGroupByShouldPass() { + ParseTree tree = new PPLSyntaxParser().parse("source=t | top showperc=true a by b"); + assertNotEquals(null, tree); + } + + @Test + public void testTopCommandWithPercentFieldShouldPass() { + ParseTree tree = + new PPLSyntaxParser().parse("source=t | top showperc=true percentfield='pct' a"); + assertNotEquals(null, tree); + } + @Test public void testCanParseMultiMatchRelevanceFunction() { assertNotEquals( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java index 21aa15c2e64..356e8a533af 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRareTopNTest.java @@ -222,6 +222,206 @@ public void failWithDuplicatedName() { } } + @Test + public void testRareShowPerc() { + String ppl = "source=EMP | rare showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; percent=7.142857\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithGroupBy() { + String ppl = "source=EMP | rare showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` NULLS LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithoutShowCount() { + String ppl = "source=EMP | rare showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; percent=7.142857\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercCustomField() { + String ppl = "source=EMP | rare percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=PRESIDENT; count=1; pct=7.142857\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testRareShowPercWithLimit() { + String ppl = "source=EMP | rare 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // PRESIDENT has 1 out of 14 total employees = 7.142857% + String expectedResult = "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` NULLS" + + " LAST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTop() { String ppl = "source=EMP | top JOB"; @@ -408,4 +608,205 @@ public void testTopUseNullFalse() { + "WHERE `_row_number_rare_top_` <= 10"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testTopShowPerc() { + String ppl = "source=EMP | top showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; percent=28.571429\n" + + "JOB=SALESMAN; count=4; percent=28.571429\n" + + "JOB=MANAGER; count=3; percent=21.428571\n" + + "JOB=ANALYST; count=2; percent=14.285714\n" + + "JOB=PRESIDENT; count=1; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithGroupBy() { + String ppl = "source=EMP | top showperc=true JOB by DEPTNO"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3])\n" + + " LogicalFilter(condition=[<=($4, 10)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2], percent=[$3]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (PARTITION BY $0 ORDER BY $2 DESC, $1)])\n" + + " LogicalProject(DEPTNO=[$0], JOB=[$1], count=[$2]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($2):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($2) OVER (PARTITION BY $0)):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0, 1}], count=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "DEPTNO=20; JOB=ANALYST; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=CLERK; count=2; percent=40.0\n" + + "DEPTNO=20; JOB=MANAGER; count=1; percent=20.0\n" + + "DEPTNO=10; JOB=CLERK; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=MANAGER; count=1; percent=33.333333\n" + + "DEPTNO=10; JOB=PRESIDENT; count=1; percent=33.333333\n" + + "DEPTNO=30; JOB=SALESMAN; count=4; percent=66.666667\n" + + "DEPTNO=30; JOB=CLERK; count=1; percent=16.666667\n" + + "DEPTNO=30; JOB=MANAGER; count=1; percent=16.666667\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `DEPTNO`, `JOB`, `count`, `percent`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, `count`, `percent`, ROW_NUMBER() OVER (PARTITION BY" + + " `DEPTNO` ORDER BY `count` DESC NULLS FIRST, `JOB` NULLS LAST)" + + " `_row_number_rare_top_`\n" + + "FROM (SELECT `DEPTNO`, `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS" + + " DOUBLE) / CAST(SUM(COUNT(*)) OVER (PARTITION BY `DEPTNO` RANGE BETWEEN UNBOUNDED" + + " PRECEDING AND UNBOUNDED FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`, `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithoutShowCount() { + String ppl = "source=EMP | top showcount=false showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; percent=28.571429\n" + + "JOB=SALESMAN; percent=28.571429\n" + + "JOB=MANAGER; percent=21.428571\n" + + "JOB=ANALYST; percent=14.285714\n" + + "JOB=PRESIDENT; percent=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercCustomField() { + String ppl = "source=EMP | top percentfield='pct' showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], pct=[$2])\n" + + " LogicalFilter(condition=[<=($3, 10)])\n" + + " LogicalProject(JOB=[$0], count=[$1], pct=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " pct=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = + "" + + "JOB=CLERK; count=4; pct=28.571429\n" + + "JOB=SALESMAN; count=4; pct=28.571429\n" + + "JOB=MANAGER; count=3; pct=21.428571\n" + + "JOB=ANALYST; count=2; pct=14.285714\n" + + "JOB=PRESIDENT; count=1; pct=7.142857\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `pct`\n" + + "FROM (SELECT `JOB`, `count`, `pct`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `pct`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 10"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testTopShowPercWithLimit() { + String ppl = "source=EMP | top 1 showperc=true JOB"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(JOB=[$0], count=[$1], percent=[$2])\n" + + " LogicalFilter(condition=[<=($3, 1)])\n" + + " LogicalProject(JOB=[$0], count=[$1], percent=[$2]," + + " _row_number_rare_top_=[ROW_NUMBER() OVER (ORDER BY $1 DESC, $0)])\n" + + " LogicalProject(JOB=[$0], count=[$1]," + + " percent=[ROUND(/(*(100.0:DECIMAL(4, 1), CAST($1):DOUBLE NOT NULL)," + + " CAST(CHECKED_LONG_SUM($1) OVER ()):DOUBLE NOT NULL), 6)])\n" + + " LogicalAggregate(group=[{0}], count=[COUNT()])\n" + + " LogicalProject(JOB=[$2])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + // Should show percentage relative to full dataset, not 100% + // CLERK has 4 out of 14 total employees = 28.571429% + String expectedResult = "JOB=CLERK; count=4; percent=28.571429\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `JOB`, `count`, `percent`\n" + + "FROM (SELECT `JOB`, `count`, `percent`, ROW_NUMBER() OVER (ORDER BY `count` DESC" + + " NULLS FIRST, `JOB` NULLS LAST) `_row_number_rare_top_`\n" + + "FROM (SELECT `JOB`, COUNT(*) `count`, ROUND(100.0 * CAST(COUNT(*) AS DOUBLE) /" + + " CAST(SUM(COUNT(*)) OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED" + + " FOLLOWING) AS DOUBLE), 6) `percent`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `JOB`) `t1`) `t2`\n" + + "WHERE `_row_number_rare_top_` <= 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index b16b44f1c92..9f874423712 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -741,6 +741,8 @@ public void testRareCommand() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -757,6 +759,8 @@ public void testRareCommandWithGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -773,12 +777,50 @@ public void testRareCommandWithMultipleFields() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), field("b"))); } + @Test + public void testRareCommandWithShowPerc() { + assertEqual( + "source=t | rare showperc=true a", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testRareCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | rare showperc=true a by b", + rareTopN( + relation("t"), + CommandType.RARE, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithN() { assertEqual( @@ -790,6 +832,8 @@ public void testTopCommandWithN() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -806,6 +850,8 @@ public void testTopCommandWithoutNAndGroupBy() { argument("noOfResults", intLiteral(10)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), emptyList(), field("a"))); @@ -822,6 +868,8 @@ public void testTopCommandWithNAndGroupBy() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("b")), field("a"))); @@ -838,6 +886,8 @@ public void testTopCommandWithMultipleFields() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(true))), exprList(field("c")), field("a"), @@ -855,11 +905,49 @@ public void testTopCommandWithUseNullFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); } + @Test + public void testTopCommandWithShowPerc() { + assertEqual( + "source=t | top showperc=true a", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + emptyList(), + field("a"))); + } + + @Test + public void testTopCommandWithShowPercAndGroupBy() { + assertEqual( + "source=t | top showperc=true a by b", + rareTopN( + relation("t"), + CommandType.TOP, + exprList( + argument("noOfResults", intLiteral(10)), + argument("countField", stringLiteral("count")), + argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(true)), + argument("useNull", booleanLiteral(true))), + exprList(field("b")), + field("a"))); + } + @Test public void testTopCommandWithLegacyFalse() { when(settings.getSettingValue(Key.PPL_SYNTAX_LEGACY_PREFERRED)).thenReturn(false); @@ -872,6 +960,8 @@ public void testTopCommandWithLegacyFalse() { argument("noOfResults", intLiteral(1)), argument("countField", stringLiteral("count")), argument("showCount", booleanLiteral(true)), + argument("percentField", stringLiteral("percent")), + argument("showPerc", booleanLiteral(false)), argument("useNull", booleanLiteral(false))), exprList(field("b")), field("a"))); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 723d2859891..00a3693594e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -441,20 +441,38 @@ public void testTopCommandWithNAndGroupBy() { public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | rare 10 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | rare a by b")); } + @Test + public void testRareCommandWithShowPercWithCalCite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | rare 10 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier", + anonymize("source=t | rare showperc=true a ")); + } + @Test public void testTopCommandWithNAndGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=table | top 1 countield='count' showcount=true usenull=true identifier by" - + " identifier", + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=false usenull=true identifier by identifier", anonymize("source=t | top 1 a by b")); } + @Test + public void testTopCommandWithShowPercWithCalcite() { + when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); + assertEquals( + "source=table | top 1 countfield='count' showcount=true percentfield='percent'" + + " showperc=true usenull=true identifier by identifier", + anonymize("source=t | top 1 showperc=true a by b")); + } + @Test public void testAndExpression() { assertEquals( From f4ede11fccd4e246e8f5e8efc173ed61d1d39c80 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:16:10 -0700 Subject: [PATCH 20/78] Accept plain Calcite types against UDT operand signatures (#5675) PPLOperandTypes.SCALAR_TYPES declares DATE/TIME/TIMESTAMP/IP/BINARY operands as UDTs, but typesMatch rejected a pair outright whenever only one side extended AbstractExprRelDataType. The analytics engine builds its row types from plain Calcite types (date -> TIMESTAMP(3), ip and binary -> VARBINARY) plus markers deriving from Calcite's AbstractSqlType, so every such operand failed the check. The result was a self-contradictory error, because getAllowedSignatures renders via the UDT tag while getActualSignature renders via convertRelDataTypeToExprType -- both print TIMESTAMP: Aggregation function LIST expects field type {...|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]}, but got [TIMESTAMP] Map the UDT tag to the SqlTypeNames a backend would emit for the same logical type. Comparing backing types would not work, since the UDTs are all VARCHAR-backed. The mapping is expressed over SqlTypeName rather than by calling convertAnalyticsEngineRelDataTypeToExprType, because analytics-api is a compileOnly dependency of core and loading those marker classes throws NoClassDefFoundError wherever it is off the runtime classpath. Signed-off-by: Kai Huang --- .../expression/function/PPLTypeChecker.java | 38 +++++++- .../function/PPLUdtSignatureMatchTest.java | 87 +++++++++++++++++++ 2 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java index c3de664443d..4925b35b649 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java @@ -557,19 +557,51 @@ public List> getParameterTypes() { * ExprUDT} tag — comparing {@code getClass()} is unsafe because addCharsetAndCollation collapses * ExprDateType/ExprTimeType/ExprTimeStampType/ExprBinaryType down to ExprSqlType, so different * UDTs would appear equal. Plain types match by SqlTypeName. + * + *

A UDT signature also accepts the equivalent plain Calcite type. Signatures such as {@code + * PPLOperandTypes.ANY_SCALAR} declare temporal/IP/BINARY operands as UDTs, but the analytics + * engine builds its row types from plain Calcite types plus its own markers, which extend + * Calcite's {@code AbstractSqlType} rather than {@link AbstractExprRelDataType}. Matching on + * class identity alone made {@code list()} fail with an error that listed the very + * type it had just rejected. */ private static boolean typesMatch(RelDataType expected, RelDataType actual) { if (expected instanceof AbstractExprRelDataType expUdt && actual instanceof AbstractExprRelDataType actUdt) { return expUdt.getUdt() == actUdt.getUdt(); } - if (expected instanceof AbstractExprRelDataType - || actual instanceof AbstractExprRelDataType) { - return false; + if (expected instanceof AbstractExprRelDataType expUdt) { + return matchesPlainType(expUdt.getUdt(), actual); + } + if (actual instanceof AbstractExprRelDataType actUdt) { + return matchesPlainType(actUdt.getUdt(), expected); } return expected.getSqlTypeName() == actual.getSqlTypeName(); } + /** + * Whether a plain Calcite type is the non-UDT spelling of {@code udt}. The UDTs themselves are + * all VARCHAR-backed, so this maps the tag to the {@link SqlTypeName}s a backend would produce + * for the same logical type instead of comparing backing types. + */ + private static boolean matchesPlainType(ExprUDT udt, RelDataType plain) { + return switch (udt) { + case EXPR_DATE -> plain.getSqlTypeName() == SqlTypeName.DATE; + case EXPR_TIME -> + switch (plain.getSqlTypeName()) { + case TIME, TIME_TZ, TIME_WITH_LOCAL_TIME_ZONE -> true; + default -> false; + }; + case EXPR_TIMESTAMP -> + switch (plain.getSqlTypeName()) { + case TIMESTAMP, TIMESTAMP_TZ, TIMESTAMP_WITH_LOCAL_TIME_ZONE -> true; + default -> false; + }; + // ip and binary both land as VARBINARY. + case EXPR_IP, EXPR_BINARY -> SqlTypeName.BINARY_TYPES.contains(plain.getSqlTypeName()); + }; + } + // Util Functions /** diff --git a/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java b/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java new file mode 100644 index 00000000000..2388721435f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/PPLUdtSignatureMatchTest.java @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; + +/** + * Exercises {@link PPLTypeChecker#wrapUDT} against plain Calcite types. Signatures declare + * temporal/IP/BINARY operands as UDTs, but the analytics engine builds row types from plain Calcite + * types, so a UDT signature must still accept the equivalent plain type. + */ +class PPLUdtSignatureMatchTest { + + private static final OpenSearchTypeFactory TF = OpenSearchTypeFactory.TYPE_FACTORY; + + /** The checker behind {@code list()}. */ + private static final PPLTypeChecker ANY_SCALAR = + PPLTypeChecker.wrapUDT( + ((UDFOperandMetadata.UDTOperandMetadata) PPLOperandTypes.ANY_SCALAR).allowedParamTypes()); + + private static RelDataType nullable(RelDataType type) { + return TF.createTypeWithNullability(type, true); + } + + private static boolean accepts(RelDataType type) { + return ANY_SCALAR.checkOperandTypes(List.of(type)); + } + + @Test + void plainTimestampMatchesTimestampUdt() { + // date -> TIMESTAMP(3), date_nanos -> TIMESTAMP(9) on the analytics route. + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIMESTAMP, 3)))); + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIMESTAMP, 9)))); + } + + @Test + void plainDateAndTimeMatchTheirUdts() { + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.DATE)))); + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.TIME)))); + } + + @Test + void plainVarbinaryMatchesBinaryUdt() { + // ip and binary both map to VARBINARY on the analytics route. + assertTrue(accepts(nullable(TF.createSqlType(SqlTypeName.VARBINARY)))); + } + + @Test + void udtOperandsStillMatch() { + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_TIMESTAMP))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_DATE))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_TIME))); + assertTrue(accepts(TF.createUDT(ExprUDT.EXPR_IP))); + } + + @Test + void plainScalarsStillMatch() { + assertTrue(accepts(TF.createSqlType(SqlTypeName.INTEGER))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.BIGINT))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.VARCHAR))); + assertTrue(accepts(TF.createSqlType(SqlTypeName.BOOLEAN))); + } + + @Test + void nonScalarsAreStillRejected() { + assertFalse( + accepts(TF.createArrayType(TF.createSqlType(SqlTypeName.INTEGER), -1)), + "ANY_SCALAR must not accept arrays"); + assertFalse( + accepts( + TF.createMapType( + TF.createSqlType(SqlTypeName.VARCHAR), TF.createSqlType(SqlTypeName.INTEGER))), + "ANY_SCALAR must not accept maps"); + } +} From 33b703dff84d94cc4d5d36d991fac320f5ca9833 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 16:54:27 -0700 Subject: [PATCH 21/78] feat(ci): add PPL lint rule validation check (eventstats PoC) Add a cross-repository CI check that keeps the OpenSearch-Dashboards PPL lint rule 'unsupported-window-function-in-eventstats' and the SQL backend in agreement. Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an OSD checkout and asserts the rule's diagnostic counts. Backend half: a Gradle integration test sends the same queries to the live /_plugins/_ppl endpoint of the SQL plugin built from the checkout. Both halves consume one shared contract file. - integ-test/.../ppl-lint/unsupported-window-function-in-eventstats.spec.json - scripts/ppl-lint/run-frontend-contract.mjs - integ-test/.../calcite/remote/PplLintRuleValidationIT.java - .github/workflows/ppl-lint-rule-validation.yml - scripts/ppl-lint-rule-validation.sh Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 148 ++++++++++++++++ .../remote/PplLintRuleValidationIT.java | 130 ++++++++++++++ ...ed-window-function-in-eventstats.spec.json | 29 ++++ scripts/ppl-lint-rule-validation.sh | 105 +++++++++++ scripts/ppl-lint/run-frontend-contract.mjs | 164 ++++++++++++++++++ 5 files changed, 576 insertions(+) create mode 100644 .github/workflows/ppl-lint-rule-validation.yml create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java create mode 100644 integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json create mode 100755 scripts/ppl-lint-rule-validation.sh create mode 100644 scripts/ppl-lint/run-frontend-contract.mjs diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml new file mode 100644 index 00000000000..61ee0be6fb8 --- /dev/null +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -0,0 +1,148 @@ +name: PPL lint rule validation + +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule +# `unsupported-window-function-in-eventstats` and the SQL backend must agree. +# +# Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an +# OSD checkout and asserts the rule's diagnostic counts. +# Backend half: a Gradle integration test sends the same queries to the live +# `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. +# +# The OSD detector is loaded from `main` by default (PR and nightly runs), so a +# removed or changed detector is detected. `workflow_dispatch` can target a +# specific OSD ref to reproduce a run or test an unmerged OSD branch. + +on: + pull_request: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + inputs: + osd_ref: + description: OSD commit or branch to test instead of main + required: false + type: string + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + ppl-lint-rule-validation: + name: PPL lint rule validation + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve OSD ref + id: osd-ref + env: + REQUESTED_REF: ${{ inputs.osd_ref }} + run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: opensearch-project/OpenSearch-Dashboards + ref: ${{ steps.osd-ref.outputs.ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-sha + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding + # it, so an OSD toolchain bump does not silently drift this job. + - name: Read OSD Node version + id: osd-node + run: echo "version=$(cat .ci/OpenSearch-Dashboards/.nvmrc)" >> "$GITHUB_OUTPUT" + + - name: Set up Node ${{ steps.osd-node.outputs.version }} + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version: ${{ steps.osd-node.outputs.version }} + + - name: Read OSD Yarn version + id: osd-yarn + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + echo "version=$yarn_version" >> "$GITHUB_OUTPUT" + + - name: Pin Yarn ${{ steps.osd-yarn.outputs.version }} + run: npm install -g "yarn@${{ steps.osd-yarn.outputs.version }}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: yarn osd bootstrap + + # The Gradle test cluster runs the version from build.gradle's + # `opensearch.version` default (e.g. 3.8.0-SNAPSHOT). Export the release + # portion (3.8.0) as PPL_SQL_VERSION so the frontend applies the same + # version filtering the backend does, without maintaining a second string. + - name: Resolve OpenSearch version + id: os-version + run: | + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + version="${raw%%-*}" + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" + + # The frontend contract only runs Node (no OpenSearch cluster), so it runs + # as the default container user. Only the Gradle integTest below must run + # as a non-root user, because OpenSearch refuses to start as root. + - name: Run frontend contract (OSD analyzer) + working-directory: .ci/OpenSearch-Dashboards + env: + PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json + PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }} + run: | + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + | tee "$GITHUB_WORKSPACE/frontend-contract.log" + + - name: Run backend integration test (live /_plugins/_ppl) + run: | + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + + - name: Upload failure artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-rule-validation-artifacts + path: | + frontend-contract.log + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java new file mode 100644 index 00000000000..ac92f979595 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Backend half of the PPL lint rule validation contract. + * + *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from + * the current checkout and asserts, per contract case, that: + * + *

    + *
  • a rejected query returns the contracted HTTP status and structured error body ({@code + * status}, {@code error.type}, {@code error.reason}); and + *
  • a valid control query returns HTTP 200 with data. + *
+ * + *

The contract file is shared verbatim with the SQL-owned OSD frontend adapter ({@code + * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer + * diagnostic and the SQL backend behavior. The rejection-body parsing mirrors the existing {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT} pattern; the live fixture and + * Calcite setup follow {@link org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + */ +public class PplLintRuleValidationIT extends PPLIntegTestCase { + + private static final String CONTRACT_RESOURCE = + "src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json"; + + @Override + public void init() throws Exception { + super.init(); + // eventstats via the Calcite path. Disallow fallback so an unsupported window function is + // rejected rather than silently degrading to the V2 engine. + enableCalcite(); + disallowCalciteFallback(); + loadIndex(Index.ACCOUNT); + } + + @Test + public void testValidatesUnsupportedWindowFunctionContract() throws IOException { + JSONObject contract = loadContract(); + String index = contract.getString("index"); + JSONArray cases = contract.getJSONArray("cases"); + + for (int i = 0; i < cases.length(); i++) { + JSONObject testCase = cases.getJSONObject(i); + String caseId = testCase.getString("id"); + String query = testCase.getString("query").replace("{{index}}", index); + JSONObject backendExpected = testCase.getJSONObject("backendExpected"); + int expectedStatus = backendExpected.getInt("httpStatus"); + + if (expectedStatus == 200) { + verifyAcceptedCase(caseId, query); + } else { + verifyRejectedCase(caseId, query, expectedStatus, backendExpected.getJSONObject("body")); + } + } + } + + /** A valid control query must return HTTP 200. executeQuery already asserts the 200 status. */ + private void verifyAcceptedCase(String caseId, String query) throws IOException { + JSONObject response = executeQuery(query); + assertTrue( + "case \"" + + caseId + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + } + + /** + * A rejected query must throw a {@link ResponseException} whose response carries the contracted + * HTTP status and structured error fields. executeQuery internally asserts 200, so a non-200 + * response surfaces as a ResponseException before it can return. + */ + private void verifyRejectedCase( + String caseId, String query, int expectedStatus, JSONObject expectedBody) { + ResponseException exception = assertThrows(ResponseException.class, () -> executeQuery(query)); + + int actualStatus = exception.getResponse().getStatusLine().getStatusCode(); + assertEquals( + "case \"" + caseId + "\": unexpected HTTP status for query: " + query, + expectedStatus, + actualStatus); + + JSONObject body; + try { + body = new JSONObject(getResponseBody(exception.getResponse(), true)); + } catch (IOException e) { + throw new RuntimeException( + "case \"" + caseId + "\": failed to read rejection response body for query: " + query, e); + } + + assertEquals( + "case \"" + caseId + "\": unexpected top-level status field for query: " + query, + expectedBody.getInt("status"), + body.getInt("status")); + + JSONObject expectedError = expectedBody.getJSONObject("error"); + JSONObject actualError = body.getJSONObject("error"); + + assertEquals( + "case \"" + caseId + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + assertEquals( + "case \"" + caseId + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } + + private JSONObject loadContract() throws IOException { + String path = TestUtils.getResourceFilePath(CONTRACT_RESOURCE); + return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); + } +} diff --git a/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..02fbab79a3c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,29 @@ +{ + "ruleId": "unsupported-window-function-in-eventstats", + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "eventstats-rank", + "query": "source={{index}} | eventstats rank() as rank_value", + "frontendDiagnosticCount": 1, + "backendExpected": { + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + }, + { + "id": "eventstats-avg-control", + "query": "source={{index}} | eventstats avg(age) as avg_age", + "frontendDiagnosticCount": 0, + "backendExpected": { + "httpStatus": 200 + } + } + ] +} diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh new file mode 100755 index 00000000000..1552353d052 --- /dev/null +++ b/scripts/ppl-lint-rule-validation.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Local developer entry point for the PPL lint rule validation contract. +# +# Runs both halves of the cross-repository check from a SQL checkout: +# 1. Frontend: loads the compiled OpenSearch-Dashboards (OSD) PPL analyzer and +# asserts the rule's diagnostic counts against the shared contract. +# 2. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout. +# +# Usage: +# # OSD main frontend check plus SQL backend IT (fetches OSD into .ci/) +# ./scripts/ppl-lint-rule-validation.sh +# +# # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) +# OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh +# +# # Reproduce a CI run against a specific OSD revision +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# +# # Skip one half +# SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_FRONTEND=1 ./scripts/ppl-lint-rule-validation.sh + +set -euo pipefail + +SQL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SQL_ROOT" + +OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" +OSD_REF="${OSD_REF:-main}" +DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" +CONTRACT_FILE="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json" +FRONTEND_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + +log() { echo "[ppl-lint-rule-validation] $*"; } + +resolve_opensearch_version() { + local raw + raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | + sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') + echo "${raw%%-*}" +} + +run_frontend() { + local osd_checkout="$1" + + if [[ ! -d "$osd_checkout/node_modules" ]]; then + log "Bootstrapping OSD at $osd_checkout (this can take a while)..." + (cd "$osd_checkout" && yarn osd bootstrap) + else + log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." + fi + + local os_version + os_version="$(resolve_opensearch_version)" + log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version)..." + ( + cd "$osd_checkout" + PPL_LINT_CONTRACT_FILE="$CONTRACT_FILE" \ + PPL_SQL_VERSION="$os_version" \ + node -r ./src/setup_node_env "$FRONTEND_SCRIPT" + ) +} + +if [[ "${SKIP_FRONTEND:-0}" != "1" ]]; then + if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then + OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" + log "Using existing OSD checkout: $OSD_CHECKOUT" + else + OSD_CHECKOUT="$DEFAULT_OSD_CHECKOUT" + if [[ ! -d "$OSD_CHECKOUT/.git" ]]; then + log "Cloning OSD ($OSD_REF) into $OSD_CHECKOUT ..." + mkdir -p "$(dirname "$OSD_CHECKOUT")" + git clone --depth 1 --branch "$OSD_REF" "$OSD_REPO_URL" "$OSD_CHECKOUT" 2>/dev/null || + git clone "$OSD_REPO_URL" "$OSD_CHECKOUT" + fi + log "Checking out OSD ref: $OSD_REF" + git -C "$OSD_CHECKOUT" fetch --depth 1 origin "$OSD_REF" 2>/dev/null || true + git -C "$OSD_CHECKOUT" checkout "$OSD_REF" 2>/dev/null || + git -C "$OSD_CHECKOUT" checkout FETCH_HEAD + fi + + OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" + log "OSD revision under test: $OSD_SHA" + + run_frontend "$OSD_CHECKOUT" + log "Frontend contract passed." +else + log "SKIP_FRONTEND=1 — skipping the OSD frontend contract." +fi + +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + log "Running backend integration test: $IT_CLASS" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" + log "Backend integration test passed." +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test." +fi + +log "Done." diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs new file mode 100644 index 00000000000..603e3feab6c --- /dev/null +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -0,0 +1,164 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * SQL-owned frontend contract adapter for the PPL lint rule validation CI. + * + * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, + * for example: + * + * cd .ci/OpenSearch-Dashboards + * PPL_LINT_CONTRACT_FILE= \ + * PPL_SQL_VERSION= \ + * node -r ./src/setup_node_env \ + * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + * + * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation + * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles + * `packages/osd-monaco/src/**` TypeScript on `require()` regardless of where the + * entry script lives. That is what lets this SQL-owned `.mjs` load the compiled + * OSD analyzer without OSD's own Jest. + * + * The analyzer, catalog and detector registry are NOT re-exported from the + * `@osd/monaco` package barrel, so they are loaded via their deep module paths. + * Because this is an ES module, `require` is obtained with `createRequire`, and + * the modules are resolved against the OSD checkout root (process.cwd()) rather + * than the location of this script (which lives in the SQL repo, not OSD). + */ + +import assert from 'assert'; +import fs from 'fs'; +import path from 'path'; +import { createRequire } from 'module'; + +const RULE_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; +const CATALOG_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/src/ppl/lint/detector_registry'; + +function fail(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-frontend-contract] FAIL: ${message}`); + process.exit(1); +} + +function loadContract() { + const contractFile = process.env.PPL_LINT_CONTRACT_FILE; + if (!contractFile) { + fail('PPL_LINT_CONTRACT_FILE is not set.'); + } + if (!fs.existsSync(contractFile)) { + fail(`Contract file not found: ${contractFile}`); + } + try { + return JSON.parse(fs.readFileSync(contractFile, 'utf8')); + } catch (error) { + fail(`Could not parse contract file ${contractFile}: ${error.message}`); + return undefined; // unreachable + } +} + +function loadOsdAnalyzer() { + // Resolve the OSD compiled analyzer from the checkout root. In CI this script + // runs after `cd .ci/OpenSearch-Dashboards`, so process.cwd() is that root. + const osdRoot = process.cwd(); + const require = createRequire(path.join(osdRoot, 'noop.js')); + + const resolveOsd = (relativeModule) => { + const absolute = path.join(osdRoot, relativeModule); + if (!fs.existsSync(`${absolute}.ts`) && !fs.existsSync(`${absolute}.js`)) { + fail( + `Expected OSD module not found under the checkout root: ${relativeModule}\n` + + `Resolved OSD root: ${osdRoot}\n` + + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` + ); + } + return require(absolute); + }; + + const { PPLLanguageAnalyzer } = resolveOsd(RULE_MODULE); + const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + const { getDetector } = resolveOsd(DETECTOR_REGISTRY_MODULE); + + if (typeof PPLLanguageAnalyzer !== 'function') { + fail(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); + } + return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot }; +} + +function assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector) { + const entry = getBundledCatalog().find((candidate) => candidate.id === ruleId); + if (!entry) { + fail(`Rule "${ruleId}" is not present in the OSD bundled catalog.`); + } + if (!entry.enabled) { + fail(`Rule "${ruleId}" is present but disabled in the OSD bundled catalog.`); + } + if (entry.severity !== 'error') { + fail(`Rule "${ruleId}" severity is "${entry.severity}", expected "error".`); + } + if (typeof getDetector(entry.detector) !== 'function') { + fail(`Rule "${ruleId}" has no registered detector "${entry.detector}".`); + } + return entry; +} + +function main() { + const contract = loadContract(); + const ruleId = contract.ruleId; + const index = contract.index; + const sqlVersion = process.env.PPL_SQL_VERSION; + + const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = loadOsdAnalyzer(); + + const entry = assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] OSD root: ${osdRoot}\n` + + `[ppl-lint-frontend-contract] rule "${ruleId}" enabled=${entry.enabled} severity=${entry.severity} detector="${entry.detector}"\n` + + `[ppl-lint-frontend-contract] PPL_SQL_VERSION=${sqlVersion || '(unset)'}\n` + + `[ppl-lint-frontend-contract] running ${contract.cases.length} case(s) against index "${index}"` + ); + + const analyzer = new PPLLanguageAnalyzer(); + const failures = []; + + for (const testCase of contract.cases) { + const query = testCase.query.split('{{index}}').join(index); + const result = analyzer.lint(query, { + dataSourceVersion: sqlVersion, + isCalcite: true, + }); + const matches = result.diagnostics.filter((diagnostic) => diagnostic.ruleId === ruleId); + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] ${testCase.id}: expected ${testCase.frontendDiagnosticCount}, ` + + `got ${matches.length} — ${query}` + ); + + try { + assert.strictEqual( + matches.length, + testCase.frontendDiagnosticCount, + `case "${testCase.id}": expected ${testCase.frontendDiagnosticCount} "${ruleId}" ` + + `diagnostic(s) but received ${matches.length} for query: ${query}` + ); + } catch (error) { + failures.push(error.message); + } + } + + if (failures.length > 0) { + fail(`${failures.length} case(s) failed:\n- ${failures.join('\n- ')}`); + } + + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-frontend-contract] PASS: all ${contract.cases.length} case(s) matched for "${ruleId}".` + ); +} + +main(); From 8b5b3b17db2586f02871eede835ba52732c91ef3 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 17:41:10 -0700 Subject: [PATCH 22/78] style(ci): use $(...) instead of legacy backticks in ppl-lint workflow Addresses shellcheck SC2006/SC2046 on the chown/su lines so actionlint runs clean. Behavior is unchanged. Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-rule-validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 61ee0be6fb8..4d545ea677b 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -133,8 +133,8 @@ jobs: - name: Run backend integration test (live /_plugins/_ppl) run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" - name: Upload failure artifacts if: ${{ failure() }} From 4bc258e7442dd2b70ccf2436b06c57ff262bcbea Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 20:59:33 -0700 Subject: [PATCH 23/78] ci: re-trigger PPL lint rule validation after Actions recovery Signed-off-by: Hanyu Wei From 6dd18d08a7205692db861830a686f3f16c564102 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 22:01:59 -0700 Subject: [PATCH 24/78] fix(ci): split ppl-lint validation into frontend/backend jobs The OpenSearch CI container is Amazon Linux 2 (glibc 2.26), but OSD requires Node 22 whose prebuilt binary needs glibc >= 2.27. Running the Node frontend contract inside that container failed with 'GLIBC_2.27 not found'. Split into two required jobs: 'frontend' runs the OSD analyzer contract on a bare ubuntu-latest runner (modern glibc, actions/setup-node works), and 'backend' keeps the Gradle integration test in the CI container where the OpenSearch test cluster needs it. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 4d545ea677b..9e9d5e11dc8 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -3,10 +3,17 @@ name: PPL lint rule validation # Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule # `unsupported-window-function-in-eventstats` and the SQL backend must agree. # -# Frontend half: a SQL-owned Node script loads the compiled OSD analyzer from an -# OSD checkout and asserts the rule's diagnostic counts. -# Backend half: a Gradle integration test sends the same queries to the live -# `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. +# Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD +# analyzer from an OSD checkout and asserts the rule's diagnostic counts. This +# runs on a bare ubuntu-latest runner because OSD requires a modern Node whose +# prebuilt binary needs a newer glibc than the OpenSearch CI container (Amazon +# Linux 2) provides. +# Backend half (backend job): a Gradle integration test sends the same queries +# to the live `/_plugins/_ppl` endpoint of the SQL plugin built from this +# checkout. This runs inside the OpenSearch CI container because the Gradle +# test cluster needs it. +# +# Both jobs are required; a failure on either side fails the SQL PR check. # # The OSD detector is loaded from `main` by default (PR and nightly runs), so a # removed or changed detector is detected. `workflow_dispatch` can target a @@ -24,23 +31,10 @@ on: type: string jobs: - Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main - with: - product: opensearch - - ppl-lint-rule-validation: - name: PPL lint rule validation - needs: Get-CI-Image-Tag + frontend: + name: Frontend contract (OSD analyzer) runs-on: ubuntu-latest - container: - image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} - steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -58,49 +52,33 @@ jobs: path: .ci/OpenSearch-Dashboards - name: Record OSD revision - id: osd-sha run: | sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) - echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" - - name: Set up JDK 21 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 - with: - distribution: 'temurin' - java-version: 21 - # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding # it, so an OSD toolchain bump does not silently drift this job. - - name: Read OSD Node version - id: osd-node - run: echo "version=$(cat .ci/OpenSearch-Dashboards/.nvmrc)" >> "$GITHUB_OUTPUT" - - - name: Set up Node ${{ steps.osd-node.outputs.version }} + - name: Set up Node from OSD .nvmrc uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: - node-version: ${{ steps.osd-node.outputs.version }} + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - - name: Read OSD Yarn version - id: osd-yarn + - name: Pin Yarn from OSD engines working-directory: .ci/OpenSearch-Dashboards run: | yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') - echo "version=$yarn_version" >> "$GITHUB_OUTPUT" - - - name: Pin Yarn ${{ steps.osd-yarn.outputs.version }} - run: npm install -g "yarn@${{ steps.osd-yarn.outputs.version }}" + npm install -g "yarn@${yarn_version}" - name: Cache OSD Yarn dependencies uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 with: path: | ~/.cache/yarn - key: ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} restore-keys: | - ${{ runner.os }}-osd-yarn-node${{ steps.osd-node.outputs.version }}- + ${{ runner.os }}-osd-yarn- - name: Bootstrap OpenSearch-Dashboards working-directory: .ci/OpenSearch-Dashboards @@ -118,10 +96,7 @@ jobs: echo "version=$version" >> "$GITHUB_OUTPUT" echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" - # The frontend contract only runs Node (no OpenSearch cluster), so it runs - # as the default container user. Only the Gradle integTest below must run - # as a non-root user, because OpenSearch refuses to start as root. - - name: Run frontend contract (OSD analyzer) + - name: Run frontend contract working-directory: .ci/OpenSearch-Dashboards env: PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json @@ -131,7 +106,42 @@ jobs: "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ | tee "$GITHUB_WORKSPACE/frontend-contract.log" - - name: Run backend integration test (live /_plugins/_ppl) + - name: Upload frontend log + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-frontend-contract-log + path: frontend-contract.log + + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend: + name: Backend integration test (live /_plugins/_ppl) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. + - name: Run backend integration test run: | chown -R 1000:1000 "$(pwd)" su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" @@ -141,8 +151,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true with: - name: ppl-lint-rule-validation-artifacts + name: ppl-lint-backend-artifacts path: | - frontend-contract.log integ-test/build/reports/** integ-test/build/testclusters/*/logs/* From 71eb0e712be9f2d5c4dc8dae02f65a2fe371d316 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 19 Jul 2026 23:38:44 -0700 Subject: [PATCH 25/78] feat(ci): extend PPL lint contract to all reachable rules (schema v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalize the single-rule PPL lint validation contract (eventstats PoC) into a schema-v2 corpus that pins every reachable OSD analyzer rule to live /_plugins/_ppl behavior. Both halves read the same reviewed contract files so neither the analyzer diagnostic nor the engine behavior can drift without a red build. Verified live end-to-end: frontend 13/13 against the OSD main analyzer; backend 13/13 (pr) and 21/21 (nightly) against a live test cluster. Contract schema v2 (integ-test/src/test/resources/ppl-lint/contracts/*.spec.json + manifest.json): - backend.kind discriminator: rejection | result-shape | advisory (explain reserved for the nightly-only explain rule class once it lands on OSD main). - per-contract backendFixture.clusterSettings so contracts that disagree on fallback/join settings each set what they need (eventstats needs calciteFallback=false; dedup-consecutive needs true) — validated in one run. - per-case minVersionRequired/engineRequired so both halves skip identically. - wiring block asserted deep-equal against the OSD catalog (drift tripwire). - frontendContext.deriveFromMapping single-sources fields/typeMap for the field-validation existence pass. - error.reason values snapshotted from the observed engine response, not hand-typed (join/multisearch AST-build-time throws yield generic "Invalid Query"; union/replace carry the specific message). Frontend adapter (run-frontend-contract.mjs): contract discovery via manifest, schedule + version/engine gating, catalog wiring assertion, compiled-simplified and runtime-bundle grammar surfaces (runtime-only rules whose parser rules are absent on the checkout's grammar assert wiring then skip cleanly), collect-all failures, and a frontend-report.json for disagreement diffing. Nightly adds a coverage assertion that every enabled catalog rule has a contract. Backend IT: parameterized over the contract corpus with per-kind verifiers (verifyRejectedCase / verifyResultShape / verifyAdvisory200), per-contract cluster-setting apply+reset, GET / cluster-version gating, and a backend-report.json recording observed status/type/reason per rejection. Sends queries with a JSON-escaped body so contract queries containing quotes (grok field=body "...") reach the engine faithfully instead of tripping a core request-payload parse error. Honors -Dppl.lint.schedule=pr|nightly, forwarded to the forked test JVM via integ-test/build.gradle. Workflow + repro script: derive schedule (PR -> pr, cron -> nightly), read the contract dir, upload frontend/backend reports + corpus artifacts. Rules covered: eventstats window fn, division-by-zero, head-without-sort, disabled-join-type, field-validation (shape + existence) on PR; plus dedup-consecutive and the runtime-only union/multisearch/replace on nightly. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 95 +++- .gitignore | 6 + integ-test/build.gradle | 11 + .../remote/PplLintRuleValidationIT.java | 495 ++++++++++++++++-- .../dedup-consecutive-unsupported.spec.json | 42 ++ .../contracts/disabled-join-type.spec.json | 52 ++ .../contracts/division-by-zero.spec.json | 44 ++ .../contracts/field-validation.spec.json | 66 +++ .../contracts/head-without-sort.spec.json | 38 ++ .../ppl-lint/contracts/manifest.json | 15 + .../multisearch-min-subsearch.spec.json | 47 ++ .../replace-wildcard-asymmetry.spec.json | 53 ++ .../contracts/union-min-datasets.spec.json | 47 ++ ...ed-window-function-in-eventstats.spec.json | 49 ++ ...ed-window-function-in-eventstats.spec.json | 29 - scripts/ppl-lint-rule-validation.sh | 19 +- scripts/ppl-lint/run-frontend-contract.mjs | 470 ++++++++++++++--- 17 files changed, 1399 insertions(+), 179 deletions(-) create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/manifest.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json delete mode 100644 integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 9e9d5e11dc8..53f4d14ca74 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,23 +1,30 @@ name: PPL lint rule validation -# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rule -# `unsupported-window-function-in-eventstats` and the SQL backend must agree. +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rules and the +# SQL backend must agree. A shared, reviewed corpus of contract files pins each +# rule's OSD analyzer diagnostic count to the live SQL engine's behavior, so +# neither side can drift unilaterally without a red build. # # Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD -# analyzer from an OSD checkout and asserts the rule's diagnostic counts. This -# runs on a bare ubuntu-latest runner because OSD requires a modern Node whose -# prebuilt binary needs a newer glibc than the OpenSearch CI container (Amazon -# Linux 2) provides. -# Backend half (backend job): a Gradle integration test sends the same queries -# to the live `/_plugins/_ppl` endpoint of the SQL plugin built from this -# checkout. This runs inside the OpenSearch CI container because the Gradle -# test cluster needs it. +# analyzer from an OSD checkout and asserts each rule's diagnostic counts and +# catalog wiring. This runs on a bare ubuntu-latest runner because OSD requires a +# modern Node whose prebuilt binary needs a newer glibc than the OpenSearch CI +# container (Amazon Linux 2) provides. +# Backend half (backend job): a Gradle integration test sends the same queries to +# the live `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. +# This runs inside the OpenSearch CI container because the Gradle test cluster +# needs it. # # Both jobs are required; a failure on either side fails the SQL PR check. # -# The OSD detector is loaded from `main` by default (PR and nightly runs), so a -# removed or changed detector is detected. `workflow_dispatch` can target a -# specific OSD ref to reproduce a run or test an unmerged OSD branch. +# Schedule: pull_request and workflow_dispatch run the fast, deterministic +# `schedule:pr` subset. The nightly cron runs the full corpus — runtime-only +# rules, advisory/soft-oracle rules, and a coverage assertion that every enabled +# OSD catalog rule has a contract file. +# +# The OSD detector is loaded from `main` by default, so a removed or changed +# detector is caught. `workflow_dispatch` can target a specific OSD ref to +# reproduce a run or pre-validate an unmerged OSD branch. on: pull_request: @@ -29,6 +36,11 @@ on: description: OSD commit or branch to test instead of main required: false type: string + schedule: + description: Contract schedule to run (pr or nightly) + required: false + default: pr + type: string jobs: frontend: @@ -44,6 +56,22 @@ jobs: REQUESTED_REF: ${{ inputs.osd_ref }} run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + echo "Contract schedule: \`$value\`" >> "$GITHUB_STEP_SUMMARY" + - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -99,20 +127,25 @@ jobs: - name: Run frontend contract working-directory: .ci/OpenSearch-Dashboards env: - PPL_LINT_CONTRACT_FILE: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json + PPL_LINT_CONTRACT_DIR: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/contracts + PPL_LINT_SCHEDULE: ${{ steps.schedule.outputs.value }} PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }} + PPL_LINT_REPORT: ${{ github.workspace }}/frontend-report.json run: | node -r ./src/setup_node_env \ "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ | tee "$GITHUB_WORKSPACE/frontend-contract.log" - - name: Upload frontend log - if: ${{ failure() }} + - name: Upload frontend report and corpus + if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true with: - name: ppl-lint-frontend-contract-log - path: frontend-contract.log + name: ppl-lint-frontend-report + path: | + frontend-contract.log + frontend-report.json + integ-test/src/test/resources/ppl-lint/contracts Get-CI-Image-Tag: uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main @@ -134,6 +167,21 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: @@ -144,7 +192,16 @@ jobs: - name: Run backend integration test run: | chown -R 1000:1000 "$(pwd)" - su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} -Dppl.lint.report=$(pwd)/backend-report.json" + + - name: Upload backend report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend-report + path: | + backend-report.json - name: Upload failure artifacts if: ${{ failure() }} diff --git a/.gitignore b/.gitignore index dcaf14d2ad7..997143fc731 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,9 @@ http-client.env.json .claude/settings.local.json .clinerules memory-bank + +# PPL lint rule validation contract run artifacts (uploaded in CI, not committed) +frontend-report.json +backend-report.json +backend-report-nightly.json +frontend-contract.log diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 29627ff7b60..d2f89bb17e3 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -168,6 +168,17 @@ tasks.withType(licenseHeaders.class) { additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' } +// Forward the PPL lint rule validation contract knobs to every integ test JVM +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly) and +// an optional path to write the observed-vs-expected report. Applied globally so +// every RestIntegTestTask that runs the class picks it up without per-task edits. +tasks.withType(Test).configureEach { + systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") + if (System.getProperty("ppl.lint.report") != null) { + systemProperty "ppl.lint.report", System.getProperty("ppl.lint.report") + } +} + validateNebulaPom.enabled = false loggerUsageCheck.enabled = false diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index ac92f979595..211ad341293 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -6,96 +6,188 @@ package org.opensearch.sql.calcite.remote; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.legacy.TestUtils; import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the PPL lint rule validation contract. + * Backend half of the schema-v2 PPL lint rule validation contract. * *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from - * the current checkout and asserts, per contract case, that: + * the current checkout. For every contract case (see {@code + * src/test/resources/ppl-lint/contracts/*.spec.json}) it applies the case's cluster settings and + * asserts, per {@code backend.kind}: * *

    - *
  • a rejected query returns the contracted HTTP status and structured error body ({@code - * status}, {@code error.type}, {@code error.reason}); and - *
  • a valid control query returns HTTP 200 with data. + *
  • {@code rejection} — the query returns the contracted HTTP status and structured error body + * ({@code status}, {@code error.type}, {@code error.reason}); + *
  • {@code result-shape} — a 200 response whose {@code datarows} match {@code datarowsNonEmpty} + * / {@code datarowsCount} / {@code columnAllNull}; + *
  • {@code advisory} — a 200 response (soft oracle for rules whose backend behavior can't be + * confirmed by a single run, e.g. head nondeterminism, fallback warnings). *
* - *

The contract file is shared verbatim with the SQL-owned OSD frontend adapter ({@code + *

The contract files are shared verbatim with the SQL-owned OSD frontend adapter ({@code * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer - * diagnostic and the SQL backend behavior. The rejection-body parsing mirrors the existing {@link - * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT} pattern; the live fixture and - * Calcite setup follow {@link org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + * diagnostic count and the SQL backend behavior; neither side can drift without a red build. The + * rejection-body parsing mirrors {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT}; the Calcite setup follows {@link + * org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + * + *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): PR runs only the + * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus including the + * runtime-only and softer-oracle rules. */ public class PplLintRuleValidationIT extends PPLIntegTestCase { - private static final String CONTRACT_RESOURCE = - "src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json"; + private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; + private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; + + /** Which contracts to run this session; PR is the fast blocking subset. */ + private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + + private int[] clusterVersion; @Override public void init() throws Exception { super.init(); - // eventstats via the Calcite path. Disallow fallback so an unsupported window function is - // rejected rather than silently degrading to the V2 engine. enableCalcite(); - disallowCalciteFallback(); - loadIndex(Index.ACCOUNT); + // Seed the union of every index every scheduled contract needs, once. + for (String indexEnum : requiredIndexEnums()) { + loadIndex(Index.valueOf(indexEnum)); + } + clusterVersion = fetchClusterVersion(); } @Test - public void testValidatesUnsupportedWindowFunctionContract() throws IOException { - JSONObject contract = loadContract(); + public void testValidatesLintRuleContracts() throws IOException { + List contracts = loadScheduledContracts(); + List failures = new ArrayList<>(); + JSONArray report = new JSONArray(); + + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + runContract(contract, ruleId, failures, report); + } + + writeReport(report); + + if (!failures.isEmpty()) { + fail( + "PPL lint backend contract failures (" + + failures.size() + + "):\n- " + + String.join("\n- ", failures)); + } + } + + private void runContract( + JSONObject contract, String ruleId, List failures, JSONArray report) + throws IOException { String index = contract.getString("index"); JSONArray cases = contract.getJSONArray("cases"); + JSONObject fixture = contract.optJSONObject("backendFixture"); - for (int i = 0; i < cases.length(); i++) { - JSONObject testCase = cases.getJSONObject(i); - String caseId = testCase.getString("id"); - String query = testCase.getString("query").replace("{{index}}", index); - JSONObject backendExpected = testCase.getJSONObject("backendExpected"); - int expectedStatus = backendExpected.getInt("httpStatus"); + List applied = applyClusterSettings(fixture); + try { + for (int i = 0; i < cases.length(); i++) { + JSONObject testCase = cases.getJSONObject(i); + String caseId = testCase.getString("id"); + String query = testCase.getString("query").replace("{{index}}", index); - if (expectedStatus == 200) { - verifyAcceptedCase(caseId, query); - } else { - verifyRejectedCase(caseId, query, expectedStatus, backendExpected.getJSONObject("body")); + String minVersion = testCase.optString("minVersionRequired", null); + if (minVersion != null && !versionAtLeast(minVersion)) { + log(ruleId, caseId, "SKIP (needs >= " + minVersion + ")"); + continue; + } + + JSONObject backend = resolveBackend(testCase); + String kind = backend.getString("kind"); + JSONObject entry = reportEntry(ruleId, caseId, query, kind); + try { + verifyCase(kind, caseId, query, backend, entry); + entry.put("outcome", "pass"); + log(ruleId, caseId, "PASS (" + kind + ")"); + } catch (AssertionError | RuntimeException e) { + entry.put("outcome", "fail").put("error", String.valueOf(e.getMessage())); + failures.add("[" + ruleId + "/" + caseId + "] " + e.getMessage()); + log(ruleId, caseId, "FAIL (" + kind + "): " + e.getMessage()); + } + report.put(entry); } + } finally { + resetClusterSettings(applied); } } - /** A valid control query must return HTTP 200. executeQuery already asserts the 200 status. */ - private void verifyAcceptedCase(String caseId, String query) throws IOException { - JSONObject response = executeQuery(query); - assertTrue( - "case \"" - + caseId - + "\": expected a datarows array in the 200 response for query: " - + query, - response.has("datarows")); + private void verifyCase( + String kind, String caseId, String query, JSONObject backend, JSONObject reportEntry) + throws IOException { + switch (kind) { + case "rejection": + verifyRejectedCase( + caseId, + query, + backend.getInt("httpStatus"), + backend.getJSONObject("body"), + reportEntry); + break; + case "result-shape": + verifyResultShape(caseId, query, backend.optJSONObject("expect")); + break; + case "advisory": + verifyAdvisory200(caseId, query); + break; + default: + throw new IllegalArgumentException( + "case \"" + caseId + "\": unknown backend.kind \"" + kind + "\""); + } } - /** - * A rejected query must throw a {@link ResponseException} whose response carries the contracted - * HTTP status and structured error fields. executeQuery internally asserts 200, so a non-200 - * response surfaces as a ResponseException before it can return. - */ + /** Back-compat: accept both v2 {@code backend} and the legacy {@code backendExpected} shape. */ + private JSONObject resolveBackend(JSONObject testCase) { + if (testCase.has("backend")) { + return testCase.getJSONObject("backend"); + } + JSONObject legacy = testCase.getJSONObject("backendExpected"); + int status = legacy.getInt("httpStatus"); + JSONObject backend = new JSONObject(); + if (status == 200) { + return backend.put("kind", "result-shape").put("httpStatus", 200); + } + return backend + .put("kind", "rejection") + .put("httpStatus", status) + .put("body", legacy.getJSONObject("body")); + } + + /** A rejected query must throw with the contracted status and structured error fields. */ private void verifyRejectedCase( - String caseId, String query, int expectedStatus, JSONObject expectedBody) { - ResponseException exception = assertThrows(ResponseException.class, () -> executeQuery(query)); + String caseId, + String query, + int expectedStatus, + JSONObject expectedBody, + JSONObject reportEntry) { + ResponseException exception = assertThrows(ResponseException.class, () -> runPplQuery(query)); int actualStatus = exception.getResponse().getStatusLine().getStatusCode(); - assertEquals( - "case \"" + caseId + "\": unexpected HTTP status for query: " + query, - expectedStatus, - actualStatus); JSONObject body; try { @@ -105,26 +197,321 @@ private void verifyRejectedCase( "case \"" + caseId + "\": failed to read rejection response body for query: " + query, e); } + // Record the observed status/type/reason before asserting so backend-report.json + // carries the byte-exact engine wording even for a failing case — this is what + // the snapshot should be updated to when the contract is deliberately changed. + JSONObject observed = new JSONObject().put("httpStatus", actualStatus); + JSONObject actualError = body.optJSONObject("error"); + if (actualError != null) { + observed.put("type", actualError.opt("type")).put("reason", actualError.opt("reason")); + } + reportEntry.put("observed", observed); + + assertEquals( + "case \"" + caseId + "\": unexpected HTTP status for query: " + query, + expectedStatus, + actualStatus); + assertEquals( "case \"" + caseId + "\": unexpected top-level status field for query: " + query, expectedBody.getInt("status"), body.getInt("status")); JSONObject expectedError = expectedBody.getJSONObject("error"); - JSONObject actualError = body.getJSONObject("error"); assertEquals( "case \"" + caseId + "\": unexpected error.type for query: " + query, expectedError.getString("type"), actualError.getString("type")); - assertEquals( - "case \"" + caseId + "\": unexpected error.reason for query: " + query, - expectedError.getString("reason"), - actualError.getString("reason")); + if (expectedError.has("reason")) { + assertEquals( + "case \"" + caseId + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } } - private JSONObject loadContract() throws IOException { - String path = TestUtils.getResourceFilePath(CONTRACT_RESOURCE); + /** A result-shape case returns 200 whose datarows match the declared expectations. */ + private void verifyResultShape(String caseId, String query, JSONObject expect) + throws IOException { + JSONObject response = runPplQuery(query); + assertTrue( + "case \"" + + caseId + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + if (expect == null) { + return; + } + JSONArray datarows = response.getJSONArray("datarows"); + + if (expect.optBoolean("datarowsNonEmpty", false)) { + assertTrue( + "case \"" + caseId + "\": expected non-empty datarows for query: " + query, + datarows.length() > 0); + } + if (expect.has("datarowsCount")) { + assertEquals( + "case \"" + caseId + "\": unexpected datarows count for query: " + query, + expect.getInt("datarowsCount"), + datarows.length()); + } + if (expect.has("columnAllNull")) { + String column = expect.getString("columnAllNull"); + int columnIndex = schemaColumnIndex(response, column); + assertTrue( + "case \"" + + caseId + + "\": column \"" + + column + + "\" not found in schema for query: " + + query, + columnIndex >= 0); + assertTrue( + "case \"" + + caseId + + "\": expected non-empty datarows to check null column for query: " + + query, + datarows.length() > 0); + for (int r = 0; r < datarows.length(); r++) { + JSONArray row = datarows.getJSONArray(r); + assertTrue( + "case \"" + + caseId + + "\": expected column \"" + + column + + "\" to be null in every row but row " + + r + + " was " + + row.get(columnIndex) + + " for query: " + + query, + row.isNull(columnIndex)); + } + } + } + + /** An advisory case only requires the query to be accepted (HTTP 200 with data). */ + private void verifyAdvisory200(String caseId, String query) throws IOException { + JSONObject response = runPplQuery(query); + assertTrue( + "case \"" + + caseId + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + } + + /** + * POST a PPL query to {@code /_plugins/_ppl} with a JSON-escaped body. The inherited {@code + * executeQuery} raw-interpolates the query into {@code {"query":"%s"}}, so a contract query that + * contains a double quote (e.g. {@code grok field=body "%{WORD:w}"}) would break the request + * payload and surface a spurious core-REST parse error instead of the real engine behavior. Build + * the body with a JSON serializer so any query is sent faithfully. Asserts HTTP 200 (a non-200 + * surfaces as a ResponseException, which the rejection path expects). + */ + private JSONObject runPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private int schemaColumnIndex(JSONObject response, String column) { + if (!response.has("schema")) { + return -1; + } + JSONArray schema = response.getJSONArray("schema"); + for (int i = 0; i < schema.length(); i++) { + JSONObject col = schema.getJSONObject(i); + String name = col.optString("alias", col.optString("name", "")); + if (column.equals(name) || column.equals(col.optString("name", ""))) { + return i; + } + } + return -1; + } + + // --- cluster settings ------------------------------------------------------ + + /** + * Apply the contract's cluster settings and return the list of settings changed so the caller can + * reset them afterwards. Grouped per-contract (not global) because contracts disagree: eventstats + * needs {@code calciteFallback=false} to force rejection, while dedup-consecutive needs it {@code + * true} to succeed via V2 fallback. + */ + private List applyClusterSettings(JSONObject fixture) throws IOException { + List applied = new ArrayList<>(); + if (fixture == null) { + return applied; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null) { + return applied; + } + if (settings.has("calcite")) { + if (settings.getBoolean("calcite")) { + enableCalcite(); + } else { + disableCalcite(); + } + } + if (settings.has("calciteFallback")) { + if (settings.getBoolean("calciteFallback")) { + allowCalciteFallback(); + } else { + disallowCalciteFallback(); + } + } + if (settings.has("allJoinTypesAllowed")) { + String key = Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES.getKeyValue(); + String value = Boolean.toString(settings.getBoolean("allJoinTypesAllowed")); + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, value)); + applied.add(key); + } + return applied; + } + + /** + * Reset each explicitly-applied dynamic setting to its cluster default by writing a null value. + * (calcite/calciteFallback are toggled via the inherited helpers and re-set explicitly by each + * contract, so only the persistent settings applied here are reset.) + */ + private void resetClusterSettings(List appliedKeys) { + for (String key : appliedKeys) { + try { + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, null)); + } catch (IOException e) { + // Best-effort reset; the next contract sets what it needs explicitly, so + // keep the failure visible without failing the suite. + System.err.println("[ppl-lint] failed to reset a cluster setting: " + e.getMessage()); + } + } + } + + // --- version gating -------------------------------------------------------- + + private int[] fetchClusterVersion() { + try { + Response response = client().performRequest(new Request("GET", "/")); + JSONObject body = new JSONObject(getResponseBody(response, false)); + String number = body.getJSONObject("version").getString("number"); + return parseVersion(number); + } catch (Exception e) { + // Unknown version → do not skip anything. + return null; + } + } + + private boolean versionAtLeast(String required) { + if (clusterVersion == null) { + return true; + } + int[] want = parseVersion(required); + for (int i = 0; i < 3; i++) { + if (clusterVersion[i] > want[i]) return true; + if (clusterVersion[i] < want[i]) return false; + } + return true; + } + + private int[] parseVersion(String raw) { + String cleaned = raw.split("-")[0]; + String[] parts = cleaned.split("\\."); + int[] v = new int[] {0, 0, 0}; + for (int i = 0; i < 3 && i < parts.length; i++) { + try { + v[i] = Integer.parseInt(parts[i]); + } catch (NumberFormatException ignored) { + v[i] = 0; + } + } + return v; + } + + // --- contract loading ------------------------------------------------------ + + private List loadScheduledContracts() throws IOException { + List result = new ArrayList<>(); + for (String fileName : manifestContractNames()) { + JSONObject contract = loadContractFile(CONTRACT_DIR + "/" + fileName); + String contractSchedule = contract.optString("schedule", "pr"); + if ("pr".equals(schedule) && !"pr".equals(contractSchedule)) { + continue; // PR runs only PR-scheduled contracts; nightly runs all. + } + result.add(contract); + } + return result; + } + + private List manifestContractNames() throws IOException { + JSONObject manifest = loadContractFile(MANIFEST); + JSONArray contracts = manifest.getJSONArray("contracts"); + List names = new ArrayList<>(); + for (int i = 0; i < contracts.length(); i++) { + names.add(contracts.getString(i)); + } + return names; + } + + /** Union of index enums required by the contracts scheduled to run this session. */ + private Set requiredIndexEnums() throws IOException { + Set indices = new LinkedHashSet<>(); + for (JSONObject contract : loadScheduledContracts()) { + JSONObject fixture = contract.optJSONObject("backendFixture"); + if (fixture == null) { + continue; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + continue; + } + for (int i = 0; i < declared.length(); i++) { + indices.add(declared.getString(i)); + } + } + if (indices.isEmpty()) { + indices.add("ACCOUNT"); + } + return indices; + } + + private JSONObject loadContractFile(String resourcePath) throws IOException { + String path = TestUtils.getResourceFilePath(resourcePath); return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); } + + // --- reporting ------------------------------------------------------------- + + private JSONObject reportEntry(String ruleId, String caseId, String query, String kind) { + return new JSONObject() + .put("ruleId", ruleId) + .put("caseId", caseId) + .put("query", query) + .put("kind", kind); + } + + private void writeReport(JSONArray report) { + String target = System.getProperty("ppl.lint.report"); + if (target == null || target.isEmpty()) { + return; + } + try { + Files.write(Paths.get(target), report.toString(2).getBytes()); + } catch (IOException e) { + System.err.println("[ppl-lint] could not write backend report to " + target + ": " + e); + } + } + + private void log(String ruleId, String caseId, String message) { + System.out.println( + String.format( + Locale.ROOT, "[ppl-lint-backend-contract] %s/%s: %s", ruleId, caseId, message)); + } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json new file mode 100644 index 00000000000..4c292860f6c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 2, + "ruleId": "dedup-consecutive-unsupported", + "oracleClass": "advisory", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "dedup-consecutive-unsupported", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.3.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": true } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "dedup-consecutive-true", + "query": "source={{index}} | dedup firstname consecutive=true", + "minVersionRequired": "3.3.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 1, "severity": "warning" }, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + { + "id": "dedup-plain-control", + "query": "source={{index}} | dedup firstname", + "minVersionRequired": "3.3.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json new file mode 100644 index 00000000000..3e7c7302f77 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 2, + "ruleId": "disabled-join-type", + "oracleClass": "rejection", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "disabled-join-type", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false, "allJoinTypesAllowed": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "right-join-disabled", + "query": "source={{index}} | right join left=l right=r on l.account_number=r.account_number {{index}}", + "frontend": { "diagnosticCount": 1, "severity": "warning" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + { + "id": "cross-join-disabled", + "query": "source={{index}} | cross join left=l right=r on l.account_number=r.account_number {{index}}", + "frontend": { "diagnosticCount": 1, "severity": "warning" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + { + "id": "inner-join-control", + "query": "source={{index}} | join left=l right=r on l.account_number=r.account_number {{index}} | head 1", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json new file mode 100644 index 00000000000..514476d746a --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": 2, + "ruleId": "division-by-zero", + "oracleClass": "result-shape", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "division-by-zero", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "divide-by-zero-literal", + "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1", + "frontend": { "diagnosticCount": 1, "severity": "warning" }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + }, + { + "id": "divide-by-nonzero-control", + "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + }, + { + "id": "modulo-by-zero-flagged", + "query": "source={{index}} | eval m = balance % 0 | fields m | head 1", + "frontend": { "diagnosticCount": 1, "severity": "warning" }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json new file mode 100644 index 00000000000..e383f773bb9 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 2, + "ruleId": "field-validation", + "oracleClass": "rejection", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "field-validation", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "visibleIndices": ["{{index}}"], + "deriveFromMapping": { + "account_number": "long", + "balance": "long", + "age": "long", + "firstname": "text", + "lastname": "text", + "gender": "text", + "address": "text", + "employer": "text", + "email": "text", + "city": "text", + "state": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "unknown-field-existence", + "query": "source={{index}} | where nonexistent_field > 3", + "frontend": { "diagnosticCount": 1, "severity": "error", "matchMessage": "nonexistent_field" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } + } + }, + { + "id": "grok-field-slot-shape-typo", + "query": "source={{index}} | grok field=firstname \"%{WORD:w}\"", + "frontend": { "diagnosticCount": 1, "severity": "error" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } + } + }, + { + "id": "known-field-control", + "query": "source={{index}} | where age > 30 | head 1", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json new file mode 100644 index 00000000000..1d800dfcd6d --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 2, + "ruleId": "head-without-sort", + "oracleClass": "advisory", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "head-without-sort", + "enabled": true, + "severity": "info", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "head-without-sort", + "query": "source={{index}} | head 5", + "frontend": { "diagnosticCount": 1, "severity": "info" }, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + { + "id": "head-with-sort-control", + "query": "source={{index}} | sort age | head 5", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json new file mode 100644 index 00000000000..31b54ecf18b --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -0,0 +1,15 @@ +{ + "schemaVersion": 2, + "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The frontend adapter (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files.", + "contracts": [ + "unsupported-window-function-in-eventstats.spec.json", + "division-by-zero.spec.json", + "head-without-sort.spec.json", + "disabled-join-type.spec.json", + "field-validation.spec.json", + "dedup-consecutive-unsupported.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json" + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json new file mode 100644 index 00000000000..5d64a7ba674 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -0,0 +1,47 @@ +{ + "schemaVersion": 2, + "ruleId": "multisearch-min-subsearch", + "oracleClass": "rejection", + "grammarSurface": "runtime-bundle", + "schedule": "nightly", + "requiredParserRules": ["multisearchCommand", "subSearch"], + "wiring": { + "detector": "multisearch-min-subsearch", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "multisearch-single-subsearch", + "query": "| multisearch [ search source={{index}} ]", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 1, "severity": "error" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } + } + }, + { + "id": "multisearch-two-subsearches-control", + "query": "| multisearch [ search source={{index}} ] [ search source={{index}} ]", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json new file mode 100644 index 00000000000..36a0231c423 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 2, + "ruleId": "replace-wildcard-asymmetry", + "oracleClass": "rejection", + "grammarSurface": "runtime-bundle", + "schedule": "nightly", + "requiredParserRules": ["replacePair", "stringLiteral"], + "wiring": { + "detector": "replace-wildcard-asymmetry", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "replace-wildcard-count-mismatch", + "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 1, "severity": "error" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } + } + }, + { + "id": "replace-symmetric-control", + "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json new file mode 100644 index 00000000000..eb13103a37d --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -0,0 +1,47 @@ +{ + "schemaVersion": 2, + "ruleId": "union-min-datasets", + "oracleClass": "rejection", + "grammarSurface": "runtime-bundle", + "schedule": "nightly", + "requiredParserRules": ["unionCommand", "unionDataset", "pplCommands"], + "wiring": { + "detector": "union-min-datasets", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.7.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "union-single-dataset", + "query": "| union [ source={{index}} ]", + "minVersionRequired": "3.7.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 1, "severity": "error" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } + } + }, + { + "id": "union-two-datasets-control", + "query": "| union [ source={{index}} ] [ source={{index}} ]", + "minVersionRequired": "3.7.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..8ad00479144 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 2, + "ruleId": "unsupported-window-function-in-eventstats", + "oracleClass": "rejection", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "unsupported-window-function-in-eventstats", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "cases": [ + { + "id": "eventstats-rank", + "query": "source={{index}} | eventstats rank() as rank_value", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 1, "severity": "error" }, + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + } + } + }, + { + "id": "eventstats-avg-control", + "query": "source={{index}} | eventstats avg(age) as avg_age", + "minVersionRequired": "3.4.0", + "engineRequired": "calcite", + "frontend": { "diagnosticCount": 0 }, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json deleted file mode 100644 index 02fbab79a3c..00000000000 --- a/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "ruleId": "unsupported-window-function-in-eventstats", - "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "eventstats-rank", - "query": "source={{index}} | eventstats rank() as rank_value", - "frontendDiagnosticCount": 1, - "backendExpected": { - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "CalciteUnsupportedException", - "reason": "Unexpected window function: rank" - } - } - } - }, - { - "id": "eventstats-avg-control", - "query": "source={{index}} | eventstats avg(age) as avg_age", - "frontendDiagnosticCount": 0, - "backendExpected": { - "httpStatus": 200 - } - } - ] -} diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh index 1552353d052..515dda9c3bf 100755 --- a/scripts/ppl-lint-rule-validation.sh +++ b/scripts/ppl-lint-rule-validation.sh @@ -24,6 +24,9 @@ # # Skip one half # SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh # SKIP_FRONTEND=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Run the full nightly corpus (runtime-only + advisory rules + coverage) +# PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh set -euo pipefail @@ -33,9 +36,11 @@ cd "$SQL_ROOT" OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" OSD_REF="${OSD_REF:-main}" DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" -CONTRACT_FILE="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/unsupported-window-function-in-eventstats.spec.json" +CONTRACT_DIR="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/contracts" FRONTEND_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" +# pr (fast, blocking subset) or nightly (full corpus + coverage assertion). +PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" log() { echo "[ppl-lint-rule-validation] $*"; } @@ -58,11 +63,13 @@ run_frontend() { local os_version os_version="$(resolve_opensearch_version)" - log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version)..." + log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version, schedule=$PPL_LINT_SCHEDULE)..." ( cd "$osd_checkout" - PPL_LINT_CONTRACT_FILE="$CONTRACT_FILE" \ + PPL_LINT_CONTRACT_DIR="$CONTRACT_DIR" \ + PPL_LINT_SCHEDULE="$PPL_LINT_SCHEDULE" \ PPL_SQL_VERSION="$os_version" \ + PPL_LINT_REPORT="$SQL_ROOT/frontend-report.json" \ node -r ./src/setup_node_env "$FRONTEND_SCRIPT" ) } @@ -95,8 +102,10 @@ else fi if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then - log "Running backend integration test: $IT_CLASS" - ./gradlew :integ-test:integTest --tests "$IT_CLASS" + log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ + -Dppl.lint.report="$SQL_ROOT/backend-report.json" log "Backend integration test passed." else log "SKIP_BACKEND=1 — skipping the SQL backend integration test." diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 603e3feab6c..e3163e2ad44 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -10,8 +10,10 @@ * for example: * * cd .ci/OpenSearch-Dashboards - * PPL_LINT_CONTRACT_FILE= \ + * PPL_LINT_CONTRACT_DIR= \ + * PPL_LINT_SCHEDULE=pr \ * PPL_SQL_VERSION= \ + * PPL_LINT_REPORT= \ * node -r ./src/setup_node_env \ * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" * @@ -26,9 +28,18 @@ * Because this is an ES module, `require` is obtained with `createRequire`, and * the modules are resolved against the OSD checkout root (process.cwd()) rather * than the location of this script (which lives in the SQL repo, not OSD). + * + * This adapter is the frontend half of a schema-v2 cross-repository differential + * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). + * It asserts three things per rule: + * 1. Wiring: the OSD catalog entry deep-equals the contract's `wiring` block, + * so a silently removed/retyped/regated detector reds the build. + * 2. Diagnostics: for each case the analyzer emits exactly the contracted + * number of `ruleId` diagnostics (the differential the backend half pins to + * live-engine behavior). + * 3. Coverage (nightly only): every enabled catalog rule has a contract file. */ -import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { createRequire } from 'module'; @@ -36,129 +47,444 @@ import { createRequire } from 'module'; const RULE_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; const CATALOG_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/src/ppl/lint/detector_registry'; +const LINT_RUNNER_MODULE = 'packages/osd-monaco/src/ppl/lint/lint_runner'; +const RULE_INDEX_MODULE = 'packages/osd-monaco/src/ppl/lint/rule_index'; +const GRAMMAR_MODULE = 'packages/osd-antlr-grammar/target/index.js'; +// Explain lint lives only on OSD branches that ship the explain rule class; the +// adapter feature-detects it and skips explain cases when it is absent. +const RUN_EXPLAIN_MODULE = 'packages/osd-monaco/src/ppl/lint/explain/run_explain_lint'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-frontend-contract] ${message}`); +} -function fail(message) { +function fatal(message) { // eslint-disable-next-line no-console - console.error(`[ppl-lint-frontend-contract] FAIL: ${message}`); - process.exit(1); + console.error(`[ppl-lint-frontend-contract] FATAL: ${message}`); + process.exit(2); } -function loadContract() { - const contractFile = process.env.PPL_LINT_CONTRACT_FILE; - if (!contractFile) { - fail('PPL_LINT_CONTRACT_FILE is not set.'); +/** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ +function loadContracts() { + const dir = process.env.PPL_LINT_CONTRACT_DIR; + const single = process.env.PPL_LINT_CONTRACT_FILE; + + if (single) { + if (!fs.existsSync(single)) { + fatal(`Contract file not found: ${single}`); + } + return [{ file: single, spec: JSON.parse(fs.readFileSync(single, 'utf8')) }]; } - if (!fs.existsSync(contractFile)) { - fail(`Contract file not found: ${contractFile}`); + + if (!dir) { + fatal('Set PPL_LINT_CONTRACT_DIR (a directory of *.spec.json) or PPL_LINT_CONTRACT_FILE.'); } - try { - return JSON.parse(fs.readFileSync(contractFile, 'utf8')); - } catch (error) { - fail(`Could not parse contract file ${contractFile}: ${error.message}`); - return undefined; // unreachable + if (!fs.existsSync(dir)) { + fatal(`Contract directory not found: ${dir}`); + } + + const manifestPath = path.join(dir, 'manifest.json'); + let files; + if (fs.existsSync(manifestPath)) { + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + if (!Array.isArray(manifest.contracts)) { + fatal(`manifest.json must have a "contracts" array of file names.`); + } + files = manifest.contracts.map((name) => path.join(dir, name)); + } else { + files = fs + .readdirSync(dir) + .filter((f) => f.endsWith('.spec.json')) + .sort() + .map((f) => path.join(dir, f)); } + + return files.map((file) => { + if (!fs.existsSync(file)) { + fatal(`Contract referenced by manifest not found: ${file}`); + } + return { file, spec: JSON.parse(fs.readFileSync(file, 'utf8')) }; + }); } -function loadOsdAnalyzer() { - // Resolve the OSD compiled analyzer from the checkout root. In CI this script - // runs after `cd .ci/OpenSearch-Dashboards`, so process.cwd() is that root. +function loadOsd() { const osdRoot = process.cwd(); const require = createRequire(path.join(osdRoot, 'noop.js')); - const resolveOsd = (relativeModule) => { + const resolveOsd = (relativeModule, { optional = false } = {}) => { const absolute = path.join(osdRoot, relativeModule); - if (!fs.existsSync(`${absolute}.ts`) && !fs.existsSync(`${absolute}.js`)) { - fail( + const exists = + fs.existsSync(absolute) || + fs.existsSync(`${absolute}.ts`) || + fs.existsSync(`${absolute}.js`); + if (!exists) { + if (optional) { + return undefined; + } + fatal( `Expected OSD module not found under the checkout root: ${relativeModule}\n` + `Resolved OSD root: ${osdRoot}\n` + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` ); } - return require(absolute); + try { + return require(absolute); + } catch (error) { + if (optional) { + return undefined; + } + throw error; + } }; const { PPLLanguageAnalyzer } = resolveOsd(RULE_MODULE); const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); const { getDetector } = resolveOsd(DETECTOR_REGISTRY_MODULE); + const { runLint } = resolveOsd(LINT_RUNNER_MODULE); + const ruleIndex = resolveOsd(RULE_INDEX_MODULE); + const grammar = resolveOsd(GRAMMAR_MODULE, { optional: true }); + const explain = resolveOsd(RUN_EXPLAIN_MODULE, { optional: true }); if (typeof PPLLanguageAnalyzer !== 'function') { - fail(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); + fatal(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); + } + + return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, runLint, ruleIndex, grammar, explain, osdRoot }; +} + +/** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ +function parseVersion(v) { + if (!v) return undefined; + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v)); + if (!m) return undefined; + return [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)]; +} + +function versionGte(a, b) { + const pa = parseVersion(a); + const pb = parseVersion(b); + if (!pa || !pb) return true; // unknown → do not skip + for (let i = 0; i < 3; i++) { + if (pa[i] > pb[i]) return true; + if (pa[i] < pb[i]) return false; } - return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot }; + return true; } -function assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector) { - const entry = getBundledCatalog().find((candidate) => candidate.id === ruleId); +/** + * Assert the OSD catalog entry deep-equals the contract's `wiring` block. This is + * the primary OSD-drift tripwire: if a detector is removed, retyped, re-gated or + * its severity changed, this fails before any query runs. + */ +function checkWiring(spec, catalog, getDetector, failures) { + const { ruleId, wiring } = spec; + const entry = catalog.find((c) => c.id === ruleId); if (!entry) { - fail(`Rule "${ruleId}" is not present in the OSD bundled catalog.`); + failures.push(`[${ruleId}] not present in the OSD bundled catalog.`); + return undefined; } - if (!entry.enabled) { - fail(`Rule "${ruleId}" is present but disabled in the OSD bundled catalog.`); + if (!wiring) { + return entry; // no wiring block to assert } - if (entry.severity !== 'error') { - fail(`Rule "${ruleId}" severity is "${entry.severity}", expected "error".`); + + const checks = [ + ['detector', wiring.detector, entry.detector], + ['enabled', wiring.enabled, entry.enabled], + ['severity', wiring.severity, entry.severity], + ['runtimeOnly', !!wiring.runtimeOnly, !!entry.runtimeOnly], + ['needsContext', !!wiring.needsContext, !!entry.needsContext], + ['needsExplain', !!wiring.needsExplain, !!entry.needsExplain], + ]; + for (const [name, expected, actual] of checks) { + if (expected !== undefined && expected !== actual) { + failures.push(`[${ruleId}] wiring.${name} expected ${JSON.stringify(expected)} but catalog has ${JSON.stringify(actual)}.`); + } + } + + if (wiring.appliesTo) { + const a = entry.appliesTo || {}; + for (const key of ['minVersion', 'maxVersion', 'engine']) { + if (wiring.appliesTo[key] !== undefined && wiring.appliesTo[key] !== a[key]) { + failures.push(`[${ruleId}] wiring.appliesTo.${key} expected ${JSON.stringify(wiring.appliesTo[key])} but catalog has ${JSON.stringify(a[key])}.`); + } + } } - if (typeof getDetector(entry.detector) !== 'function') { - fail(`Rule "${ruleId}" has no registered detector "${entry.detector}".`); + + if (wiring.detector && typeof getDetector(wiring.detector) !== 'function') { + failures.push(`[${ruleId}] has no registered detector "${wiring.detector}".`); } + return entry; } -function main() { - const contract = loadContract(); - const ruleId = contract.ruleId; - const index = contract.index; - const sqlVersion = process.env.PPL_SQL_VERSION; +/** + * Build the per-case lint context. Derives `fields`/`typeMap` from the + * `deriveFromMapping` block (a single source shared with the backend seeding), + * and sets an enable override for default-off rules that declare `forceEnable`. + */ +function buildContext(spec, sqlVersion) { + const fc = spec.frontendContext || {}; + const context = { + isCalcite: fc.isCalcite !== false, + dataSourceVersion: sqlVersion, + grammarSurface: spec.grammarSurface === 'runtime-bundle' ? 'runtime-bundle' : 'compiled-simplified', + }; - const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = loadOsdAnalyzer(); + const mapping = fc.deriveFromMapping; + if (mapping && typeof mapping === 'object') { + const fields = new Set(); + const typeMap = new Map(); + for (const [name, type] of Object.entries(mapping)) { + fields.add(name); + typeMap.set(name, type); + } + context.fields = fields; + context.typeMap = typeMap; + } + if (Array.isArray(fc.disabledObjectFields) && fc.disabledObjectFields.length > 0) { + context.disabledObjectFields = new Set(fc.disabledObjectFields); + } + if (Array.isArray(fc.visibleIndices) && fc.visibleIndices.length > 0) { + context.visibleIndices = fc.visibleIndices.map((i) => i.split('{{index}}').join(spec.index)); + } + if (fc.settings && typeof fc.settings === 'object') { + context.settings = fc.settings; + } + if (fc.forceEnable) { + context.overrides = { [spec.ruleId]: { enabled: true } }; + } + return context; +} - const entry = assertRuleIsWiredUp(ruleId, getBundledCatalog, getDetector); +/** Count diagnostics for this rule via the compiled-simplified analyzer. */ +function lintCompiled(analyzer, query, context, ruleId) { + const result = analyzer.lint(query, context); + return result.diagnostics.filter((d) => d.ruleId === ruleId); +} - // eslint-disable-next-line no-console - console.log( - `[ppl-lint-frontend-contract] OSD root: ${osdRoot}\n` + - `[ppl-lint-frontend-contract] rule "${ruleId}" enabled=${entry.enabled} severity=${entry.severity} detector="${entry.detector}"\n` + - `[ppl-lint-frontend-contract] PPL_SQL_VERSION=${sqlVersion || '(unset)'}\n` + - `[ppl-lint-frontend-contract] running ${contract.cases.length} case(s) against index "${index}"` - ); +/** + * Count diagnostics for a runtime-only rule by parsing with the exported runtime + * grammar and running the detector registry directly. This exercises OSD-main's + * runtime grammar, NOT the cluster-versioned bundle production fetches, so it is + * a wiring/coverage check rather than a true cluster-grammar fidelity check. + * Returns undefined when the runtime grammar can't reach the rule on this OSD + * checkout (the rule's parser rules are absent) so the caller can skip cleanly. + */ +function lintRuntime(osd, spec, query, context, ruleId) { + const { grammar, runLint, ruleIndex } = osd; + if (!grammar || !grammar.OpenSearchPPLParser || !grammar.OpenSearchPPLLexer) { + return undefined; + } + const antlr = requireAntlr(osd.osdRoot); + if (!antlr) { + return undefined; + } + const { OpenSearchPPLLexer, OpenSearchPPLParser } = grammar; + const runtimeMap = new Map(); + const names = OpenSearchPPLParser.ruleNames || []; + for (let i = 0; i < names.length; i++) { + runtimeMap.set(names[i], i); + } + + // The exported runtime grammar on this OSD checkout may predate the command a + // runtime-only rule keys off (union/multisearch/replace are absent on the + // legacy `opensearch_ppl` grammar). Detecting the absence here lets the caller + // record a clean skip — the wiring assertion already ran — instead of a false + // "0 diagnostics" failure. + const required = spec.requiredParserRules || []; + for (const name of required) { + if (!runtimeMap.has(name)) { + return undefined; + } + } + + const input = antlr.CharStream.fromString(query); + const lexer = new OpenSearchPPLLexer(input); + const tokenStream = new antlr.CommonTokenStream(lexer); + const parser = new OpenSearchPPLParser(tokenStream); + parser.removeErrorListeners(); + const tree = parser.root ? parser.root() : parser.pplStatement && parser.pplStatement(); + if (!tree) { + return undefined; + } + + const ruleNameToIndex = ruleIndex.createRuntimeRuleNameToIndex(runtimeMap); + + const diagnostics = runLint(tree, { + ruleNameToIndex, + dataSourceVersion: context.dataSourceVersion, + context: { ...context, grammarSurface: 'runtime-bundle' }, + }); + return diagnostics.filter((d) => d.ruleId === ruleId); +} + +let cachedAntlr; +function requireAntlr(osdRoot) { + if (cachedAntlr !== undefined) { + return cachedAntlr || undefined; + } + try { + const require = createRequire(path.join(osdRoot, 'noop.js')); + cachedAntlr = require('antlr4ng'); + } catch { + cachedAntlr = null; + } + return cachedAntlr || undefined; +} + +function main() { + const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; + const sqlVersion = process.env.PPL_SQL_VERSION; + const reportPath = process.env.PPL_LINT_REPORT; + + const osd = loadOsd(); + const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = osd; + const catalog = getBundledCatalog(); const analyzer = new PPLLanguageAnalyzer(); + + const contracts = loadContracts(); const failures = []; + const report = { osdRoot, schedule, sqlVersion, results: [] }; - for (const testCase of contract.cases) { - const query = testCase.query.split('{{index}}').join(index); - const result = analyzer.lint(query, { - dataSourceVersion: sqlVersion, - isCalcite: true, - }); - const matches = result.diagnostics.filter((diagnostic) => diagnostic.ruleId === ruleId); + log(`OSD root: ${osdRoot}`); + log(`schedule=${schedule} PPL_SQL_VERSION=${sqlVersion || '(unset)'} contracts=${contracts.length}`); - // eslint-disable-next-line no-console - console.log( - `[ppl-lint-frontend-contract] ${testCase.id}: expected ${testCase.frontendDiagnosticCount}, ` + - `got ${matches.length} — ${query}` - ); + for (const { file, spec } of contracts) { + const ruleId = spec.ruleId; + const index = spec.index; + + // A contract runs on PR only when scheduled for PR; nightly runs everything. + const contractSchedule = spec.schedule || 'pr'; + if (schedule === 'pr' && contractSchedule !== 'pr') { + log(`SKIP ${ruleId} (schedule=${contractSchedule}, running ${schedule}) — ${path.basename(file)}`); + continue; + } + + const entry = checkWiring(spec, catalog, getDetector, failures); + if (!entry) { + continue; + } + const context = buildContext(spec, sqlVersion); + const isRuntime = context.grammarSurface === 'runtime-bundle'; + + for (const testCase of spec.cases || []) { + const query = testCase.query.split('{{index}}').join(index); + const fe = testCase.frontend || {}; + const expected = fe.diagnosticCount; + + // Per-case version/engine gate mirrors the backend so both halves skip + // identically instead of disagreeing on a self-suppressed rule. + if (testCase.minVersionRequired && !versionGte(sqlVersion, testCase.minVersionRequired)) { + log(`SKIP ${ruleId}/${testCase.id} (needs >= ${testCase.minVersionRequired}, have ${sqlVersion || 'unknown'})`); + continue; + } + if (testCase.engineRequired === 'calcite' && context.isCalcite !== true) { + log(`SKIP ${ruleId}/${testCase.id} (needs calcite engine)`); + continue; + } + + let matches; + if (testCase.explainFixture) { + matches = lintExplain(osd, spec, testCase, context, ruleId); + if (matches === undefined) { + log(`SKIP ${ruleId}/${testCase.id} (explain lint unavailable on this OSD checkout)`); + continue; + } + } else if (isRuntime) { + matches = lintRuntime(osd, spec, query, context, ruleId); + if (matches === undefined) { + // Runtime grammar can't reach this rule on this OSD checkout: the + // wiring assertion above still ran, so record a skip (not a failure). + log(`SKIP ${ruleId}/${testCase.id} (runtime grammar rule absent on this OSD checkout; wiring asserted)`); + report.results.push({ ruleId, caseId: testCase.id, query, expected, actual: null, skipped: 'runtime-grammar-absent' }); + continue; + } + } else { + matches = lintCompiled(analyzer, query, context, ruleId); + } + + const actual = matches.length; + const ok = actual === expected; + + log(` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${testCase.id}: expected ${expected}, got ${actual} — ${query}`); + + const severityOk = + !fe.severity || actual === 0 || matches.every((m) => m.severity === fe.severity); + const messageOk = + !fe.matchMessage || matches.some((m) => (m.message || '').includes(fe.matchMessage)); + + report.results.push({ ruleId, caseId: testCase.id, query, expected, actual, severities: matches.map((m) => m.severity) }); + + if (!ok) { + failures.push(`[${ruleId}/${testCase.id}] expected ${expected} "${ruleId}" diagnostic(s), got ${actual} for: ${query}`); + } + if (!severityOk) { + failures.push(`[${ruleId}/${testCase.id}] expected severity "${fe.severity}" for: ${query}`); + } + if (!messageOk) { + failures.push(`[${ruleId}/${testCase.id}] expected message to contain "${fe.matchMessage}" for: ${query}`); + } + } + } + + // Nightly-only coverage: every enabled catalog rule must have a contract file. + if (schedule === 'nightly') { + const covered = new Set(contracts.map(({ spec }) => spec.ruleId)); + for (const rule of catalog) { + if (rule.enabled && !covered.has(rule.id)) { + failures.push(`[coverage] enabled catalog rule "${rule.id}" has no contract file.`); + } + } + } + + if (reportPath) { + report.failures = failures; try { - assert.strictEqual( - matches.length, - testCase.frontendDiagnosticCount, - `case "${testCase.id}": expected ${testCase.frontendDiagnosticCount} "${ruleId}" ` + - `diagnostic(s) but received ${matches.length} for query: ${query}` - ); + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); + log(`wrote report to ${reportPath}`); } catch (error) { - failures.push(error.message); + log(`WARN: could not write report to ${reportPath}: ${error.message}`); } } if (failures.length > 0) { - fail(`${failures.length} case(s) failed:\n- ${failures.join('\n- ')}`); + // eslint-disable-next-line no-console + console.error(`[ppl-lint-frontend-contract] FAIL: ${failures.length} problem(s):\n- ${failures.join('\n- ')}`); + process.exit(1); } - // eslint-disable-next-line no-console - console.log( - `[ppl-lint-frontend-contract] PASS: all ${contract.cases.length} case(s) matched for "${ruleId}".` - ); + log(`PASS: all contracts agreed with the OSD analyzer (schedule=${schedule}).`); +} + +/** + * Explain-case handling. Loads the captured plan fixture and runs the OSD explain + * lint over it. Returns undefined when the explain rule class is not present on + * this OSD checkout (feature-detected via the optional module). + */ +function lintExplain(osd, spec, testCase, context, ruleId) { + if (!osd.explain || typeof osd.explain.runExplainLint !== 'function') { + return undefined; + } + const dir = process.env.PPL_LINT_CONTRACT_DIR; + if (!dir) { + return undefined; + } + const fixturePath = path.join(dir, testCase.explainFixture); + if (!fs.existsSync(fixturePath)) { + return undefined; + } + const plan = JSON.parse(fs.readFileSync(fixturePath, 'utf8')); + const query = testCase.query.split('{{index}}').join(spec.index); + const diagnostics = osd.explain.runExplainLint(plan, { + query, + overrides: context.overrides, + dataSourceVersion: context.dataSourceVersion, + isCalcite: context.isCalcite, + }); + return (diagnostics || []).filter((d) => d.ruleId === ruleId); } main(); From 930bfe5eee6619339e4ae88274bdcdcba94e4ee3 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 20 Jul 2026 15:46:42 -0700 Subject: [PATCH 26/78] feat(ci): reconnect PPL lint validation to candidate runtime grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Take the cross-repository PPL lint check from the PoC to the design in ppl-lint-ci-validation-design.md. The detector half now lints against the *candidate* runtime grammar bundle the SQL PR builds — through OSD's production headless lint API — instead of the compiled analyzer or a hand-rolled reparse of OSD main's checked-in grammar. Both halves validate the same grammar, so a parser/semantic change that invalidates a lint rule reds the build. SQL-side changes (the OSD headless API ships separately): - PplLintRuleValidationIT: export the candidate grammar bundle (GET /_plugins/_ppl/_grammar) + a target manifest {engineVersion, grammarHash, grammarBundle} while the cluster is alive; read schema-v3 specs; select the one expectations[] entry matching the backend version (zero/>1 fails); record the observed backend behavior per query for the differential. - integ-test/build.gradle: forward -Dppl.lint.grammar.bundle / -Dppl.lint.target to the test JVM alongside the existing ppl.lint.* knobs. - run-frontend-contract.mjs: deserialize the candidate bundle via the OSD headless API and lint each query with lintQueryWithBundle (runtime-bundle surface, so the runtime-only arity rules fire); pin dataSourceVersion + knownVersion to the candidate version; assert the detector-vs-backend differential from the backend report; fail loud on a missing bundle. - workflow: linear backend-validation -> detector-validation -> validation-result pipeline; artifacts are the only bridge between jobs. validation-result is the single always() required check (red unless both jobs succeed) and writes the per-rule PR summary; assemble-run-manifest.mjs emits run-manifest.json with the immutable SQL + OSD SHAs, mode, backend version, grammar hash, and enforced set. A workflow_dispatch osd_ref run is pre-merge evidence, not a protection result. - contracts: migrate all 9 specs to schema v3 (named queries{role,query} + version-scoped expectations[]); partition manifest.json into enforced (eventstats, multisearch, union, replace), pendingReview (field-validation), and nonEnforcing. Union/multisearch triggers are query-initial, not pipe-first: OSD prepends a synthetic source prefix to pipe-first queries, which would desync the two halves. - Add scripts/ppl-lint/README.md documenting inputs, local reproduction, the contract format, and the failure table. Verified end to end against a live cluster: the backend IT exports a real candidate bundle and the detector runner agrees on all four enforced rules (triggers rejected + 1 diagnostic, controls accepted + 0); an intentional expectation mismatch reds the runner. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 282 ++++++---- .gitignore | 7 +- integ-test/build.gradle | 14 +- .../remote/PplLintRuleValidationIT.java | 414 ++++++++++---- .../dedup-consecutive-unsupported.spec.json | 40 +- .../contracts/disabled-join-type.spec.json | 66 ++- .../contracts/division-by-zero.spec.json | 50 +- .../contracts/field-validation.spec.json | 67 ++- .../contracts/head-without-sort.spec.json | 37 +- .../ppl-lint/contracts/manifest.json | 26 +- .../multisearch-min-subsearch.spec.json | 50 +- .../replace-wildcard-asymmetry.spec.json | 58 +- .../contracts/union-min-datasets.spec.json | 51 +- ...ed-window-function-in-eventstats.spec.json | 51 +- scripts/ppl-lint-rule-validation.sh | 93 ++-- scripts/ppl-lint/README.md | 195 +++++++ scripts/ppl-lint/assemble-run-manifest.mjs | 157 ++++++ scripts/ppl-lint/run-frontend-contract.mjs | 506 ++++++++++-------- 18 files changed, 1520 insertions(+), 644 deletions(-) create mode 100644 scripts/ppl-lint/README.md create mode 100644 scripts/ppl-lint/assemble-run-manifest.mjs diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 53f4d14ca74..83fc01d0734 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,30 +1,40 @@ name: PPL lint rule validation -# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint rules and the -# SQL backend must agree. A shared, reviewed corpus of contract files pins each -# rule's OSD analyzer diagnostic count to the live SQL engine's behavior, so -# neither side can drift unilaterally without a red build. +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and +# the SQL backend must agree on the SAME candidate runtime grammar. A shared, +# reviewed corpus of contract files pins each rule's OSD detector diagnostic +# count to the live SQL engine's behavior, so neither side can drift unilaterally +# without a red build. # -# Frontend half (frontend job): a SQL-owned Node script loads the compiled OSD -# analyzer from an OSD checkout and asserts each rule's diagnostic counts and -# catalog wiring. This runs on a bare ubuntu-latest runner because OSD requires a -# modern Node whose prebuilt binary needs a newer glibc than the OpenSearch CI -# container (Amazon Linux 2) provides. -# Backend half (backend job): a Gradle integration test sends the same queries to -# the live `/_plugins/_ppl` endpoint of the SQL plugin built from this checkout. -# This runs inside the OpenSearch CI container because the Gradle test cluster -# needs it. +# The workflow is a linear three-job pipeline (design §3.1): # -# Both jobs are required; a failure on either side fails the SQL PR check. +# backend-validation ──(artifacts)──▶ detector-validation ──▶ validation-result # -# Schedule: pull_request and workflow_dispatch run the fast, deterministic -# `schedule:pr` subset. The nightly cron runs the full corpus — runtime-only -# rules, advisory/soft-oracle rules, and a coverage assertion that every enabled -# OSD catalog rule has a contract file. +# 1. backend-validation (Amazon Linux CI container): builds the SQL PR, starts the +# Gradle test cluster, runs the contract trigger/control queries against the +# live /_plugins/_ppl endpoint, and — while the cluster is alive — exports the +# candidate runtime grammar bundle (GET /_plugins/_ppl/_grammar) plus a target +# manifest and the observed backend report. Those three files are the ONLY +# bridge to the next job; the test cluster is never passed between jobs. +# 2. detector-validation (ubuntu-latest): checks out and bootstraps OSD as a Node +# code dependency (no OSD server, no Monaco, no browser), deserializes the +# candidate bundle through OSD's production headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. OSD needs a newer Node/glibc than the CI container provides, +# hence a separate Ubuntu job. +# 3. validation-result: the single stable required check. Fails unless BOTH +# validation jobs succeeded (an always() result job so a skipped detector +# cannot mask a backend failure), writes the compact per-rule PR summary, and +# uploads the run manifest recording the exact SQL SHA, OSD SHA, mode, backend +# version, and grammar hash. # -# The OSD detector is loaded from `main` by default, so a removed or changed -# detector is caught. `workflow_dispatch` can target a specific OSD ref to -# reproduce a run or pre-validate an unmerged OSD branch. +# Modes (design §3.4, §4.1.1): +# - pull_request: SQL PR validation against OSD `main`. The ONLY enforcing mode; +# this is what branch protection pins to. Runs the fast schedule:pr subset. +# - workflow_dispatch (osd_ref): pre-merge evidence for an unmerged OSD branch. +# Records the resolved immutable OSD commit SHA but CANNOT satisfy branch +# protection — only the pull_request run does. +# - schedule (nightly): the full corpus + a coverage assertion. on: pull_request: @@ -33,7 +43,7 @@ on: workflow_dispatch: inputs: osd_ref: - description: OSD commit or branch to test instead of main + description: OSD commit or branch to validate instead of main (pre-merge evidence only) required: false type: string schedule: @@ -43,13 +53,96 @@ on: type: string jobs: - frontend: - name: Frontend contract (OSD analyzer) + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend-validation: + name: Backend validation (live /_plugins/_ppl + grammar export) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. The + # IT exports the candidate grammar bundle + target manifest while the cluster + # is alive; those become the artifacts the detector job lints against. + - name: Run backend integration test and export candidate grammar + run: | + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.report=$(pwd)/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/target.json" + + - name: Upload backend artifacts (bundle + target + report) + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: | + backend-report.json + ppl-grammar-bundle.json + target.json + + - name: Upload backend failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend-logs + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + + detector-validation: + name: Detector validation (OSD headless lint on candidate bundle) + needs: backend-validation runs-on: ubuntu-latest + outputs: + osd_ref: ${{ steps.osd-ref.outputs.ref }} + osd_sha: ${{ steps.osd-rev.outputs.sha }} + schedule: ${{ steps.schedule.outputs.value }} steps: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + # The required pull_request run always validates against OSD `main`; only a + # manual workflow_dispatch may target an unmerged OSD ref, and that run is + # pre-merge evidence, not a branch-protection result (design §4.1.1). - name: Resolve OSD ref id: osd-ref env: @@ -70,7 +163,12 @@ jobs: value="pr" fi echo "value=$value" >> "$GITHUB_OUTPUT" - echo "Contract schedule: \`$value\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-backend + path: artifacts - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -79,9 +177,13 @@ jobs: ref: ${{ steps.osd-ref.outputs.ref }} path: .ci/OpenSearch-Dashboards + # Resolve the (possibly mutable) ref to the immutable commit SHA actually + # tested, so the run manifest pins exactly what ran (design §4.1.1, T11). - name: Record OSD revision + id: osd-rev run: | sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding @@ -112,103 +214,93 @@ jobs: working-directory: .ci/OpenSearch-Dashboards run: yarn osd bootstrap - # The Gradle test cluster runs the version from build.gradle's - # `opensearch.version` default (e.g. 3.8.0-SNAPSHOT). Export the release - # portion (3.8.0) as PPL_SQL_VERSION so the frontend applies the same - # version filtering the backend does, without maintaining a second string. - - name: Resolve OpenSearch version - id: os-version - run: | - raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') - version="${raw%%-*}" - echo "version=$version" >> "$GITHUB_OUTPUT" - echo "OpenSearch version: \`$version\` (from \`$raw\`)" >> "$GITHUB_STEP_SUMMARY" - - - name: Run frontend contract + - name: Run detector validation against the candidate bundle working-directory: .ci/OpenSearch-Dashboards env: PPL_LINT_CONTRACT_DIR: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/contracts PPL_LINT_SCHEDULE: ${{ steps.schedule.outputs.value }} - PPL_SQL_VERSION: ${{ steps.os-version.outputs.version }} - PPL_LINT_REPORT: ${{ github.workspace }}/frontend-report.json + PPL_LINT_GRAMMAR_BUNDLE: ${{ github.workspace }}/artifacts/ppl-grammar-bundle.json + PPL_LINT_TARGET_MANIFEST: ${{ github.workspace }}/artifacts/target.json + PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json + PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json run: | node -r ./src/setup_node_env \ "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ - | tee "$GITHUB_WORKSPACE/frontend-contract.log" + | tee "$GITHUB_WORKSPACE/detector-contract.log" - - name: Upload frontend report and corpus + - name: Upload detector report and corpus if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true with: - name: ppl-lint-frontend-report + name: ppl-lint-detector path: | - frontend-contract.log - frontend-report.json + detector-contract.log + detector-report.json integ-test/src/test/resources/ppl-lint/contracts - Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main - with: - product: opensearch - - backend: - name: Backend integration test (live /_plugins/_ppl) - needs: Get-CI-Image-Tag + validation-result: + name: validation-result + if: ${{ always() }} + needs: + - backend-validation + - detector-validation runs-on: ubuntu-latest - container: - image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} - steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Resolve contract schedule - id: schedule - env: - REQUESTED_SCHEDULE: ${{ inputs.schedule }} - EVENT_NAME: ${{ github.event_name }} - run: | - if [ -n "$REQUESTED_SCHEDULE" ]; then - value="$REQUESTED_SCHEDULE" - elif [ "$EVENT_NAME" = "schedule" ]; then - value="nightly" - else - value="pr" - fi - echo "value=$value" >> "$GITHUB_OUTPUT" + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: artifacts - - name: Set up JDK 21 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + - name: Download detector artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true with: - distribution: 'temurin' - java-version: 21 + name: ppl-lint-detector + path: artifacts - # OpenSearch refuses to start as root, so run Gradle as a non-root user. - - name: Run backend integration test - run: | - chown -R 1000:1000 "$(pwd)" - su "$(id -un 1000)" -c "./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} -Dppl.lint.report=$(pwd)/backend-report.json" + # Assemble the run manifest and the compact per-rule PR summary from the + # reports both jobs uploaded. The manifest records the immutable SQL + OSD + # SHAs so any run is exactly reproducible (design §3.3, §4.4). + - name: Assemble run manifest and summary + env: + SQL_SHA: ${{ github.sha }} + OSD_REF: ${{ needs.detector-validation.outputs.osd_ref }} + OSD_SHA: ${{ needs.detector-validation.outputs.osd_sha }} + EVENT_NAME: ${{ github.event_name }} + SCHEDULE: ${{ needs.detector-validation.outputs.schedule }} + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: node "$GITHUB_WORKSPACE/scripts/ppl-lint/assemble-run-manifest.mjs" - - name: Upload backend report + - name: Upload run manifest if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true with: - name: ppl-lint-backend-report + name: ppl-lint-run-manifest path: | - backend-report.json + run-manifest.json - - name: Upload failure artifacts - if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true - with: - name: ppl-lint-backend-artifacts - path: | - integ-test/build/reports/** - integ-test/build/testclusters/*/logs/* + # The sole branch-protection check: red unless BOTH validation jobs + # succeeded. Because this job runs with always(), a skipped detector job + # (e.g. backend failed first) still reds the result instead of appearing + # green (design §4.4). A workflow_dispatch run is pre-merge evidence and is + # intentionally not what repo admins pin to branch protection. + - name: Require both validation jobs to have succeeded + env: + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: | + echo "backend-validation: $BACKEND_RESULT" + echo "detector-validation: $DETECTOR_RESULT" + if [ "$BACKEND_RESULT" != "success" ] || [ "$DETECTOR_RESULT" != "success" ]; then + echo "::error::PPL lint rule validation failed (backend=$BACKEND_RESULT detector=$DETECTOR_RESULT)." + exit 1 + fi + echo "PPL lint rule validation passed: backend and detector agree on the candidate grammar." diff --git a/.gitignore b/.gitignore index 997143fc731..05b8f803ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,10 @@ http-client.env.json memory-bank # PPL lint rule validation contract run artifacts (uploaded in CI, not committed) -frontend-report.json backend-report.json backend-report-nightly.json -frontend-contract.log +detector-report.json +detector-contract.log +ppl-grammar-bundle.json +target.json +run-manifest.json diff --git a/integ-test/build.gradle b/integ-test/build.gradle index d2f89bb17e3..bdb126a0090 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -169,13 +169,17 @@ tasks.withType(licenseHeaders.class) { } // Forward the PPL lint rule validation contract knobs to every integ test JVM -// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly) and -// an optional path to write the observed-vs-expected report. Applied globally so -// every RestIntegTestTask that runs the class picks it up without per-task edits. +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), an +// optional path to write the observed backend report, and — while the cluster is +// alive — optional paths to export the candidate runtime grammar bundle and its +// target manifest for the detector-validation job. Applied globally so every +// RestIntegTestTask that runs the class picks it up without per-task edits. tasks.withType(Test).configureEach { systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") - if (System.getProperty("ppl.lint.report") != null) { - systemProperty "ppl.lint.report", System.getProperty("ppl.lint.report") + ["ppl.lint.report", "ppl.lint.grammar.bundle", "ppl.lint.target"].each { prop -> + if (System.getProperty(prop) != null) { + systemProperty prop, System.getProperty(prop) + } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 211ad341293..4185511ab04 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -28,12 +29,14 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the schema-v2 PPL lint rule validation contract. + * Backend half of the schema-v3 PPL lint rule validation contract. * *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from - * the current checkout. For every contract case (see {@code - * src/test/resources/ppl-lint/contracts/*.spec.json}) it applies the case's cluster settings and - * asserts, per {@code backend.kind}: + * the current checkout. For every contract (see {@code + * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} + * entry that matches the candidate backend version (exactly one must match, or the contract fails + * before any query runs), applies the contract's cluster settings, and asserts, per query's {@code + * backend.kind}: * *

    *
  • {@code rejection} — the query returns the contracted HTTP status and structured error body @@ -44,26 +47,34 @@ * confirmed by a single run, e.g. head nondeterminism, fallback warnings). *
* - *

The contract files are shared verbatim with the SQL-owned OSD frontend adapter ({@code + *

The contract files are shared verbatim with the SQL-owned OSD detector runner ({@code * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer * diagnostic count and the SQL backend behavior; neither side can drift without a red build. The * rejection-body parsing mirrors {@link * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT}; the Calcite setup follows {@link * org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. * + *

While the ephemeral cluster is alive, the test also exports the candidate runtime grammar + * bundle it built ({@code GET /_plugins/_ppl/_grammar}) and a small target manifest pairing the + * bundle with the backend version and grammar hash. These become workflow artifacts that the + * detector-validation job injects into OSD's headless lint API, so both halves validate against the + * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code + * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. + * *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): PR runs only the - * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus including the - * runtime-only and softer-oracle rules. + * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus. */ public class PplLintRuleValidationIT extends PPLIntegTestCase { private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; + private static final String GRAMMAR_API_ENDPOINT = "/_plugins/_ppl/_grammar"; /** Which contracts to run this session; PR is the fast blocking subset. */ private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); private int[] clusterVersion; + private String engineVersionRaw; @Override public void init() throws Exception { @@ -82,6 +93,11 @@ public void testValidatesLintRuleContracts() throws IOException { List failures = new ArrayList<>(); JSONArray report = new JSONArray(); + // Export the candidate grammar bundle + target manifest while the cluster is + // alive. Runs before the contract loop so the artifacts are emitted even if a + // contract later fails. + exportGrammarArtifacts(failures); + for (JSONObject contract : contracts) { String ruleId = contract.getString("ruleId"); runContract(contract, ruleId, failures, report); @@ -102,33 +118,44 @@ private void runContract( JSONObject contract, String ruleId, List failures, JSONArray report) throws IOException { String index = contract.getString("index"); - JSONArray cases = contract.getJSONArray("cases"); + JSONObject queries = contract.getJSONObject("queries"); + JSONArray expectations = contract.getJSONArray("expectations"); JSONObject fixture = contract.optJSONObject("backendFixture"); + boolean calciteOn = fixtureCalciteEnabled(fixture); List applied = applyClusterSettings(fixture); try { - for (int i = 0; i < cases.length(); i++) { - JSONObject testCase = cases.getJSONObject(i); - String caseId = testCase.getString("id"); - String query = testCase.getString("query").replace("{{index}}", index); - - String minVersion = testCase.optString("minVersionRequired", null); - if (minVersion != null && !versionAtLeast(minVersion)) { - log(ruleId, caseId, "SKIP (needs >= " + minVersion + ")"); + JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); + if (selected == null) { + return; // no/ambiguous version expectation — failure already recorded. + } + JSONObject expectedQueries = selected.getJSONObject("queries"); + for (String queryName : expectedQueries.keySet()) { + if (!queries.has(queryName)) { + failures.add( + "[" + + ruleId + + "] expectation references unknown query \"" + + queryName + + "\" (not in the top-level queries map)"); continue; } - - JSONObject backend = resolveBackend(testCase); + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = expectedQueries.getJSONObject(queryName); + JSONObject backend = expected.getJSONObject("backend"); String kind = backend.getString("kind"); - JSONObject entry = reportEntry(ruleId, caseId, query, kind); + + JSONObject entry = reportEntry(ruleId, queryName, role, query, kind); try { - verifyCase(kind, caseId, query, backend, entry); + verifyCase(kind, queryName, query, backend, entry); entry.put("outcome", "pass"); - log(ruleId, caseId, "PASS (" + kind + ")"); + log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); } catch (AssertionError | RuntimeException e) { entry.put("outcome", "fail").put("error", String.valueOf(e.getMessage())); - failures.add("[" + ruleId + "/" + caseId + "] " + e.getMessage()); - log(ruleId, caseId, "FAIL (" + kind + "): " + e.getMessage()); + failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); + log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); } report.put(entry); } @@ -137,107 +164,140 @@ private void runContract( } } + /** + * Select the single {@code expectations[]} entry that applies to the candidate backend version + * and engine. Exactly one must match: zero means the rule test does not cover this version + * (design §9), and more than one means overlapping ranges — both fail before execution (§5.3). + */ + private JSONObject selectExpectation( + String ruleId, JSONArray expectations, boolean calciteOn, List failures) { + List matches = new ArrayList<>(); + for (int i = 0; i < expectations.length(); i++) { + JSONObject exp = expectations.getJSONObject(i); + if (!versionMatchesRange(exp.optString("version", null))) { + continue; + } + String engine = exp.optString("engine", ""); + if ("calcite".equals(engine) && !calciteOn) { + continue; + } + matches.add(exp); + } + String versionLabel = engineVersionRaw == null ? "unknown" : engineVersionRaw; + if (matches.size() == 1) { + return matches.get(0); + } + if (matches.isEmpty()) { + failures.add( + "[" + ruleId + "] no version expectation matches backend version " + versionLabel); + } else { + failures.add( + "[" + + ruleId + + "] " + + matches.size() + + " expectations match backend version " + + versionLabel + + " (exactly one required)"); + } + return null; + } + private void verifyCase( - String kind, String caseId, String query, JSONObject backend, JSONObject reportEntry) + String kind, String queryName, String query, JSONObject backend, JSONObject entry) throws IOException { + BackendObservation obs = observeBackend(query); + entry.put("rejected", obs.rejected); + entry.put("observed", obs.toJson()); switch (kind) { case "rejection": - verifyRejectedCase( - caseId, - query, - backend.getInt("httpStatus"), - backend.getJSONObject("body"), - reportEntry); + assertRejection( + queryName, query, obs, backend.getInt("httpStatus"), backend.getJSONObject("body")); break; case "result-shape": - verifyResultShape(caseId, query, backend.optJSONObject("expect")); + assertResultShape(queryName, query, obs, backend.optJSONObject("expect")); break; case "advisory": - verifyAdvisory200(caseId, query); + assertAdvisory(queryName, query, obs); break; default: throw new IllegalArgumentException( - "case \"" + caseId + "\": unknown backend.kind \"" + kind + "\""); + "case \"" + queryName + "\": unknown backend.kind \"" + kind + "\""); } } - /** Back-compat: accept both v2 {@code backend} and the legacy {@code backendExpected} shape. */ - private JSONObject resolveBackend(JSONObject testCase) { - if (testCase.has("backend")) { - return testCase.getJSONObject("backend"); - } - JSONObject legacy = testCase.getJSONObject("backendExpected"); - int status = legacy.getInt("httpStatus"); - JSONObject backend = new JSONObject(); - if (status == 200) { - return backend.put("kind", "result-shape").put("httpStatus", 200); - } - return backend - .put("kind", "rejection") - .put("httpStatus", status) - .put("body", legacy.getJSONObject("body")); + /** + * Run the query once and categorize the observed backend behavior independently of the + * expectation, so the report carries the true behavior even when a case fails (e.g. a trigger the + * backend unexpectedly accepted). A non-2xx surfaces as a {@link ResponseException} from the REST + * client, which is the rejection signal. + */ + private BackendObservation observeBackend(String query) throws IOException { + try { + JSONObject response = runPplQuery(query); + return BackendObservation.accepted(response); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + JSONObject body; + try { + body = new JSONObject(getResponseBody(e.getResponse(), true)); + } catch (IOException ioe) { + throw new RuntimeException( + "failed to read rejection response body for query: " + query, ioe); + } + return BackendObservation.rejected(status, body); + } } - /** A rejected query must throw with the contracted status and structured error fields. */ - private void verifyRejectedCase( - String caseId, + /** A rejected query must have thrown with the contracted status and structured error fields. */ + private void assertRejection( + String queryName, String query, + BackendObservation obs, int expectedStatus, - JSONObject expectedBody, - JSONObject reportEntry) { - ResponseException exception = assertThrows(ResponseException.class, () -> runPplQuery(query)); - - int actualStatus = exception.getResponse().getStatusLine().getStatusCode(); - - JSONObject body; - try { - body = new JSONObject(getResponseBody(exception.getResponse(), true)); - } catch (IOException e) { - throw new RuntimeException( - "case \"" + caseId + "\": failed to read rejection response body for query: " + query, e); - } - - // Record the observed status/type/reason before asserting so backend-report.json - // carries the byte-exact engine wording even for a failing case — this is what - // the snapshot should be updated to when the contract is deliberately changed. - JSONObject observed = new JSONObject().put("httpStatus", actualStatus); - JSONObject actualError = body.optJSONObject("error"); - if (actualError != null) { - observed.put("type", actualError.opt("type")).put("reason", actualError.opt("reason")); - } - reportEntry.put("observed", observed); - + JSONObject expectedBody) { + assertTrue( + "case \"" + + queryName + + "\": expected the backend to REJECT the query but it was accepted: " + + query, + obs.rejected); assertEquals( - "case \"" + caseId + "\": unexpected HTTP status for query: " + query, + "case \"" + queryName + "\": unexpected HTTP status for query: " + query, expectedStatus, - actualStatus); - + obs.status); assertEquals( - "case \"" + caseId + "\": unexpected top-level status field for query: " + query, + "case \"" + queryName + "\": unexpected top-level status field for query: " + query, expectedBody.getInt("status"), - body.getInt("status")); + obs.body.getInt("status")); JSONObject expectedError = expectedBody.getJSONObject("error"); - + JSONObject actualError = obs.body.getJSONObject("error"); assertEquals( - "case \"" + caseId + "\": unexpected error.type for query: " + query, + "case \"" + queryName + "\": unexpected error.type for query: " + query, expectedError.getString("type"), actualError.getString("type")); if (expectedError.has("reason")) { assertEquals( - "case \"" + caseId + "\": unexpected error.reason for query: " + query, + "case \"" + queryName + "\": unexpected error.reason for query: " + query, expectedError.getString("reason"), actualError.getString("reason")); } } /** A result-shape case returns 200 whose datarows match the declared expectations. */ - private void verifyResultShape(String caseId, String query, JSONObject expect) - throws IOException { - JSONObject response = runPplQuery(query); + private void assertResultShape( + String queryName, String query, BackendObservation obs, JSONObject expect) { assertTrue( "case \"" - + caseId + + queryName + + "\": expected a 200 result but the backend rejected the query: " + + query, + !obs.rejected); + JSONObject response = obs.response; + assertTrue( + "case \"" + + queryName + "\": expected a datarows array in the 200 response for query: " + query, response.has("datarows")); @@ -248,12 +308,12 @@ private void verifyResultShape(String caseId, String query, JSONObject expect) if (expect.optBoolean("datarowsNonEmpty", false)) { assertTrue( - "case \"" + caseId + "\": expected non-empty datarows for query: " + query, + "case \"" + queryName + "\": expected non-empty datarows for query: " + query, datarows.length() > 0); } if (expect.has("datarowsCount")) { assertEquals( - "case \"" + caseId + "\": unexpected datarows count for query: " + query, + "case \"" + queryName + "\": unexpected datarows count for query: " + query, expect.getInt("datarowsCount"), datarows.length()); } @@ -262,7 +322,7 @@ private void verifyResultShape(String caseId, String query, JSONObject expect) int columnIndex = schemaColumnIndex(response, column); assertTrue( "case \"" - + caseId + + queryName + "\": column \"" + column + "\" not found in schema for query: " @@ -270,7 +330,7 @@ private void verifyResultShape(String caseId, String query, JSONObject expect) columnIndex >= 0); assertTrue( "case \"" - + caseId + + queryName + "\": expected non-empty datarows to check null column for query: " + query, datarows.length() > 0); @@ -278,7 +338,7 @@ private void verifyResultShape(String caseId, String query, JSONObject expect) JSONArray row = datarows.getJSONArray(r); assertTrue( "case \"" - + caseId + + queryName + "\": expected column \"" + column + "\" to be null in every row but row " @@ -293,14 +353,20 @@ private void verifyResultShape(String caseId, String query, JSONObject expect) } /** An advisory case only requires the query to be accepted (HTTP 200 with data). */ - private void verifyAdvisory200(String caseId, String query) throws IOException { - JSONObject response = runPplQuery(query); + private void assertAdvisory(String queryName, String query, BackendObservation obs) { + assertTrue( + "case \"" + + queryName + + "\": expected the query to be accepted (advisory) but it was " + + "rejected: " + + query, + !obs.rejected); assertTrue( "case \"" - + caseId + + queryName + "\": expected a datarows array in the 200 response for query: " + query, - response.has("datarows")); + obs.response.has("datarows")); } /** @@ -338,8 +404,91 @@ private int schemaColumnIndex(JSONObject response, String column) { return -1; } + /** Observed backend behavior for one query, captured before asserting the expectation. */ + private static final class BackendObservation { + final boolean rejected; + final int status; + final JSONObject body; // rejection body, or null when accepted + final JSONObject response; // accepted 200 response, or null when rejected + + private BackendObservation(boolean rejected, int status, JSONObject body, JSONObject response) { + this.rejected = rejected; + this.status = status; + this.body = body; + this.response = response; + } + + static BackendObservation accepted(JSONObject response) { + return new BackendObservation(false, 200, null, response); + } + + static BackendObservation rejected(int status, JSONObject body) { + return new BackendObservation(true, status, body, null); + } + + JSONObject toJson() { + JSONObject o = new JSONObject().put("httpStatus", status).put("rejected", rejected); + if (body != null) { + JSONObject err = body.optJSONObject("error"); + if (err != null) { + o.put("type", err.opt("type")).put("reason", err.opt("reason")); + } + } + return o; + } + } + + // --- grammar bundle export ------------------------------------------------- + + /** + * Fetch the candidate runtime grammar bundle and write it plus a target manifest, so the + * detector-validation job can lint against the SAME grammar this backend built. Best-effort by + * design: a run without {@code -Dppl.lint.grammar.bundle} (local dev) exports nothing; in CI a + * fetch/write failure is a real failure — a missing bundle means the detector half cannot run. + */ + private void exportGrammarArtifacts(List failures) { + String bundlePath = System.getProperty("ppl.lint.grammar.bundle"); + if (bundlePath == null || bundlePath.isEmpty()) { + return; + } + try { + Response response = client().performRequest(new Request("GET", GRAMMAR_API_ENDPOINT)); + String bundleBody = getResponseBody(response, true); + Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); + + JSONObject bundle = new JSONObject(bundleBody); + String grammarHash = bundle.optString("grammarHash", ""); + + String targetPath = System.getProperty("ppl.lint.target"); + if (targetPath != null && !targetPath.isEmpty()) { + JSONObject target = + new JSONObject() + .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) + .put("grammarHash", grammarHash) + .put("grammarBundle", Paths.get(bundlePath).getFileName().toString()); + Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); + } + log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); + } catch (Exception e) { + failures.add( + "[grammar-export] failed to fetch/write " + GRAMMAR_API_ENDPOINT + ": " + e.getMessage()); + } + } + // --- cluster settings ------------------------------------------------------ + /** True when the contract's fixture leaves Calcite enabled (the default). */ + private boolean fixtureCalciteEnabled(JSONObject fixture) { + if (fixture == null) { + return true; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null || !settings.has("calcite")) { + return true; + } + return settings.getBoolean("calcite"); + } + /** * Apply the contract's cluster settings and return the list of settings changed so the caller can * reset them afterwards. Grouped per-contract (not global) because contracts disagree: eventstats @@ -402,6 +551,7 @@ private int[] fetchClusterVersion() { Response response = client().performRequest(new Request("GET", "/")); JSONObject body = new JSONObject(getResponseBody(response, false)); String number = body.getJSONObject("version").getString("number"); + engineVersionRaw = number; return parseVersion(number); } catch (Exception e) { // Unknown version → do not skip anything. @@ -409,18 +559,72 @@ private int[] fetchClusterVersion() { } } - private boolean versionAtLeast(String required) { + /** + * Test a space-separated semver range (e.g. {@code ">=3.6.0 <3.8.0"}) against the candidate + * backend version. An empty/absent range or an unknown cluster version matches (do not + * over-filter). Supports the {@code >= > <= < =} comparators the design uses. + */ + private boolean versionMatchesRange(String range) { + if (range == null || range.trim().isEmpty()) { + return true; + } if (clusterVersion == null) { return true; } - int[] want = parseVersion(required); - for (int i = 0; i < 3; i++) { - if (clusterVersion[i] > want[i]) return true; - if (clusterVersion[i] < want[i]) return false; + for (String token : range.trim().split("\\s+")) { + if (!satisfiesComparator(token)) { + return false; + } } return true; } + private boolean satisfiesComparator(String token) { + String op; + String ver; + if (token.startsWith(">=")) { + op = ">="; + ver = token.substring(2); + } else if (token.startsWith("<=")) { + op = "<="; + ver = token.substring(2); + } else if (token.startsWith(">")) { + op = ">"; + ver = token.substring(1); + } else if (token.startsWith("<")) { + op = "<"; + ver = token.substring(1); + } else if (token.startsWith("=")) { + op = "="; + ver = token.substring(1); + } else { + op = "="; + ver = token; + } + int cmp = compareVersion(clusterVersion, parseVersion(ver)); + switch (op) { + case ">=": + return cmp >= 0; + case "<=": + return cmp <= 0; + case ">": + return cmp > 0; + case "<": + return cmp < 0; + default: + return cmp == 0; + } + } + + private int compareVersion(int[] a, int[] b) { + for (int i = 0; i < 3; i++) { + if (a[i] != b[i]) { + return Integer.compare(a[i], b[i]); + } + } + return 0; + } + private int[] parseVersion(String raw) { String cleaned = raw.split("-")[0]; String[] parts = cleaned.split("\\."); @@ -489,10 +693,12 @@ private JSONObject loadContractFile(String resourcePath) throws IOException { // --- reporting ------------------------------------------------------------- - private JSONObject reportEntry(String ruleId, String caseId, String query, String kind) { + private JSONObject reportEntry( + String ruleId, String queryName, String role, String query, String kind) { return new JSONObject() .put("ruleId", ruleId) - .put("caseId", caseId) + .put("queryName", queryName) + .put("role", role) .put("query", query) .put("kind", kind); } @@ -503,7 +709,7 @@ private void writeReport(JSONArray report) { return; } try { - Files.write(Paths.get(target), report.toString(2).getBytes()); + Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { System.err.println("[ppl-lint] could not write backend report to " + target + ": " + e); } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index 4c292860f6c..ca414f6f103 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -1,7 +1,6 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "dedup-consecutive-unsupported", - "oracleClass": "advisory", "grammarSurface": "compiled-simplified", "schedule": "nightly", "wiring": { @@ -21,22 +20,31 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "dedup-consecutive-true", - "query": "source={{index}} | dedup firstname consecutive=true", - "minVersionRequired": "3.3.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 1, "severity": "warning" }, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "queries": { + "dedup-consecutive-true": { + "role": "trigger", + "query": "source={{index}} | dedup firstname consecutive=true" }, + "dedup-plain-control": { + "role": "control", + "query": "source={{index}} | dedup firstname" + } + }, + "expectations": [ { - "id": "dedup-plain-control", - "query": "source={{index}} | dedup firstname", - "minVersionRequired": "3.3.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "version": ">=3.3.0", + "engine": "calcite", + "queries": { + "dedup-consecutive-true": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + "dedup-plain-control": { + "detectorCount": 0, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index 3e7c7302f77..bd0769934ea 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -1,9 +1,8 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "disabled-join-type", - "oracleClass": "rejection", "grammarSurface": "compiled-simplified", - "schedule": "pr", + "schedule": "nightly", "wiring": { "detector": "disabled-join-type", "enabled": true, @@ -21,32 +20,47 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "right-join-disabled", - "query": "source={{index}} | right join left=l right=r on l.account_number=r.account_number {{index}}", - "frontend": { "diagnosticCount": 1, "severity": "warning" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } - } + "queries": { + "right-join-disabled": { + "role": "trigger", + "query": "source={{index}} | right join left=l right=r on l.account_number=r.account_number {{index}}" }, - { - "id": "cross-join-disabled", - "query": "source={{index}} | cross join left=l right=r on l.account_number=r.account_number {{index}}", - "frontend": { "diagnosticCount": 1, "severity": "warning" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } - } + "cross-join-disabled": { + "role": "trigger", + "query": "source={{index}} | cross join left=l right=r on l.account_number=r.account_number {{index}}" }, + "inner-join-control": { + "role": "control", + "query": "source={{index}} | join left=l right=r on l.account_number=r.account_number {{index}} | head 1" + } + }, + "expectations": [ { - "id": "inner-join-control", - "query": "source={{index}} | join left=l right=r on l.account_number=r.account_number {{index}} | head 1", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "version": ">=0.0.0", + "queries": { + "right-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + "cross-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + "inner-join-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 514476d746a..d9071cc4978 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,9 +1,8 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "division-by-zero", - "oracleClass": "result-shape", "grammarSurface": "compiled-simplified", - "schedule": "pr", + "schedule": "nightly", "wiring": { "detector": "division-by-zero", "enabled": true, @@ -21,24 +20,39 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "divide-by-zero-literal", - "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1", - "frontend": { "diagnosticCount": 1, "severity": "warning" }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + "queries": { + "divide-by-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1" }, - { - "id": "divide-by-nonzero-control", - "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "divide-by-nonzero-control": { + "role": "control", + "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" }, + "modulo-by-zero-flagged": { + "role": "trigger", + "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" + } + }, + "expectations": [ { - "id": "modulo-by-zero-flagged", - "query": "source={{index}} | eval m = balance % 0 | fields m | head 1", - "frontend": { "diagnosticCount": 1, "severity": "warning" }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + "version": ">=0.0.0", + "queries": { + "divide-by-zero-literal": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + }, + "divide-by-nonzero-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + }, + "modulo-by-zero-flagged": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index e383f773bb9..c67343ef23a 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -1,9 +1,8 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "field-validation", - "oracleClass": "rejection", "grammarSurface": "compiled-simplified", - "schedule": "pr", + "schedule": "nightly", "wiring": { "detector": "field-validation", "enabled": true, @@ -35,32 +34,48 @@ } }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "unknown-field-existence", - "query": "source={{index}} | where nonexistent_field > 3", - "frontend": { "diagnosticCount": 1, "severity": "error", "matchMessage": "nonexistent_field" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } - } + "queries": { + "unknown-field-existence": { + "role": "trigger", + "query": "source={{index}} | where nonexistent_field > 3" }, - { - "id": "grok-field-slot-shape-typo", - "query": "source={{index}} | grok field=firstname \"%{WORD:w}\"", - "frontend": { "diagnosticCount": 1, "severity": "error" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } - } + "grok-field-slot-shape-typo": { + "role": "trigger", + "query": "source={{index}} | grok field=firstname \"%{WORD:w}\"" }, + "known-field-control": { + "role": "control", + "query": "source={{index}} | where age > 30 | head 1" + } + }, + "expectations": [ { - "id": "known-field-control", - "query": "source={{index}} | where age > 30 | head 1", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "version": ">=3.4.0", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "matchMessage": "nonexistent_field", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 1d800dfcd6d..e86a19c1002 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -1,9 +1,8 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "head-without-sort", - "oracleClass": "advisory", "grammarSurface": "compiled-simplified", - "schedule": "pr", + "schedule": "nightly", "wiring": { "detector": "head-without-sort", "enabled": true, @@ -21,18 +20,30 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "head-without-sort", - "query": "source={{index}} | head 5", - "frontend": { "diagnosticCount": 1, "severity": "info" }, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "queries": { + "head-without-sort": { + "role": "trigger", + "query": "source={{index}} | head 5" }, + "head-with-sort-control": { + "role": "control", + "query": "source={{index}} | sort age | head 5" + } + }, + "expectations": [ { - "id": "head-with-sort-control", - "query": "source={{index}} | sort age | head 5", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "version": ">=0.0.0", + "queries": { + "head-without-sort": { + "detectorCount": 1, + "severity": "info", + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + "head-with-sort-control": { + "detectorCount": 0, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index 31b54ecf18b..805e1be9fd8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,6 +1,6 @@ { - "schemaVersion": 2, - "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The frontend adapter (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files.", + "schemaVersion": 3, + "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", "contracts": [ "unsupported-window-function-in-eventstats.spec.json", "division-by-zero.spec.json", @@ -11,5 +11,25 @@ "multisearch-min-subsearch.spec.json", "union-min-datasets.spec.json", "replace-wildcard-asymmetry.spec.json" - ] + ], + "enforced": [ + "unsupported-window-function-in-eventstats.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json" + ], + "pendingReview": [ + "field-validation.spec.json" + ], + "nonEnforcing": [ + "division-by-zero.spec.json", + "head-without-sort.spec.json", + "disabled-join-type.spec.json", + "dedup-consecutive-unsupported.spec.json" + ], + "notes": { + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required validation-result check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. field-validation self-suppresses without field context and is a semantic rule rather than a clean HTTP-400 grammar rejection.", + "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." + } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 5d64a7ba674..018d086fec3 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -1,10 +1,10 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "multisearch-min-subsearch", - "oracleClass": "rejection", "grammarSurface": "runtime-bundle", - "schedule": "nightly", + "schedule": "pr", "requiredParserRules": ["multisearchCommand", "subSearch"], + "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", "wiring": { "detector": "multisearch-min-subsearch", "enabled": true, @@ -22,26 +22,34 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "multisearch-single-subsearch", - "query": "| multisearch [ search source={{index}} ]", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 1, "severity": "error" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } - } + "queries": { + "multisearch-single-subsearch": { + "role": "trigger", + "query": "multisearch [ search source={{index}} ]" }, + "multisearch-two-subsearches-control": { + "role": "control", + "query": "multisearch [ search source={{index}} ] [ search source={{index}} ]" + } + }, + "expectations": [ { - "id": "multisearch-two-subsearches-control", - "query": "| multisearch [ search source={{index}} ] [ search source={{index}} ]", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "version": ">=3.4.0", + "queries": { + "multisearch-single-subsearch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } + } + }, + "multisearch-two-subsearches-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 36a0231c423..8946dad60a9 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -1,9 +1,8 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "replace-wildcard-asymmetry", - "oracleClass": "rejection", "grammarSurface": "runtime-bundle", - "schedule": "nightly", + "schedule": "pr", "requiredParserRules": ["replacePair", "stringLiteral"], "wiring": { "detector": "replace-wildcard-asymmetry", @@ -22,32 +21,41 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ + "queries": { + "replace-wildcard-count-mismatch": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname" + }, + "replace-symmetric-control": { + "role": "control", + "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1" + } + }, + "expectations": [ { - "id": "replace-wildcard-count-mismatch", - "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 1, "severity": "error" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } } + }, + "replace-symmetric-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } } } - }, - { - "id": "replace-symmetric-control", - "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index eb13103a37d..7f110cc9423 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -1,10 +1,10 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "union-min-datasets", - "oracleClass": "rejection", "grammarSurface": "runtime-bundle", - "schedule": "nightly", + "schedule": "pr", "requiredParserRules": ["unionCommand", "unionDataset", "pplCommands"], + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", "wiring": { "detector": "union-min-datasets", "enabled": true, @@ -22,26 +22,35 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ - { - "id": "union-single-dataset", - "query": "| union [ source={{index}} ]", - "minVersionRequired": "3.7.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 1, "severity": "error" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } - } + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "expectations": [ { - "id": "union-two-datasets-control", - "query": "| union [ source={{index}} ] [ source={{index}} ]", - "minVersionRequired": "3.7.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } } ] } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 8ad00479144..e1254b14583 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -1,7 +1,6 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "ruleId": "unsupported-window-function-in-eventstats", - "oracleClass": "rejection", "grammarSurface": "compiled-simplified", "schedule": "pr", "wiring": { @@ -21,29 +20,37 @@ "isCalcite": true }, "index": "opensearch-sql_test_index_account", - "cases": [ + "queries": { + "eventstats-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats rank() as rank_value" + }, + "eventstats-avg-control": { + "role": "control", + "query": "source={{index}} | eventstats avg(age) as avg_age" + } + }, + "expectations": [ { - "id": "eventstats-rank", - "query": "source={{index}} | eventstats rank() as rank_value", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 1, "severity": "error" }, - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + "version": ">=3.4.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } } } - }, - { - "id": "eventstats-avg-control", - "query": "source={{index}} | eventstats avg(age) as avg_age", - "minVersionRequired": "3.4.0", - "engineRequired": "calcite", - "frontend": { "diagnosticCount": 0 }, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } } ] } diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh index 515dda9c3bf..295168f8aad 100755 --- a/scripts/ppl-lint-rule-validation.sh +++ b/scripts/ppl-lint-rule-validation.sh @@ -5,27 +5,36 @@ # # Local developer entry point for the PPL lint rule validation contract. # -# Runs both halves of the cross-repository check from a SQL checkout: -# 1. Frontend: loads the compiled OpenSearch-Dashboards (OSD) PPL analyzer and -# asserts the rule's diagnostic counts against the shared contract. -# 2. Backend: runs the Gradle integration test against a live /_plugins/_ppl -# endpoint on the SQL plugin built from this checkout. +# Runs both halves of the cross-repository check from a SQL checkout, in the same +# order as CI (design §3.1): +# 1. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout, and — while the +# cluster is alive — exports the candidate runtime grammar bundle +# (ppl-grammar-bundle.json), a target manifest (target.json), and the +# observed backend report (backend-report.json). +# 2. Detector: bootstraps an OpenSearch-Dashboards (OSD) checkout, deserializes +# the candidate bundle through OSD's headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. +# +# The backend half must run first: the detector half lints against the bundle it +# exports. Use SKIP_BACKEND=1 only if you already have the three artifacts. # # Usage: -# # OSD main frontend check plus SQL backend IT (fetches OSD into .ci/) +# # OSD main detector check plus SQL backend IT (fetches OSD into .ci/) # ./scripts/ppl-lint-rule-validation.sh # # # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) # OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh # # # Reproduce a CI run against a specific OSD revision -# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh # -# # Skip one half +# # Skip one half (detector needs the backend artifacts to exist already) # SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh -# SKIP_FRONTEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh # -# # Run the full nightly corpus (runtime-only + advisory rules + coverage) +# # Run the full nightly corpus (all rules + coverage assertion) # PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh set -euo pipefail @@ -37,23 +46,37 @@ OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-D OSD_REF="${OSD_REF:-main}" DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" CONTRACT_DIR="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/contracts" -FRONTEND_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +DETECTOR_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" # pr (fast, blocking subset) or nightly (full corpus + coverage assertion). PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" +# Candidate artifacts the backend half exports and the detector half consumes. +GRAMMAR_BUNDLE="$SQL_ROOT/ppl-grammar-bundle.json" +TARGET_MANIFEST="$SQL_ROOT/target.json" +BACKEND_REPORT="$SQL_ROOT/backend-report.json" +DETECTOR_REPORT="$SQL_ROOT/detector-report.json" + log() { echo "[ppl-lint-rule-validation] $*"; } -resolve_opensearch_version() { - local raw - raw=$(grep -oE '"opensearch.version", "[^"]+"' build.gradle | head -1 | - sed -E 's/.*"opensearch.version", "([^"]+)"/\1/') - echo "${raw%%-*}" +run_backend() { + log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ + -Dppl.lint.report="$BACKEND_REPORT" \ + -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" \ + -Dppl.lint.target="$TARGET_MANIFEST" + log "Backend integration test passed. Exported: $(basename "$GRAMMAR_BUNDLE"), $(basename "$TARGET_MANIFEST")." } -run_frontend() { +run_detector() { local osd_checkout="$1" + if [[ ! -f "$GRAMMAR_BUNDLE" ]]; then + log "ERROR: $GRAMMAR_BUNDLE not found. Run the backend half first (do not set SKIP_BACKEND=1)." + exit 2 + fi + if [[ ! -d "$osd_checkout/node_modules" ]]; then log "Bootstrapping OSD at $osd_checkout (this can take a while)..." (cd "$osd_checkout" && yarn osd bootstrap) @@ -61,20 +84,27 @@ run_frontend() { log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." fi - local os_version - os_version="$(resolve_opensearch_version)" - log "Running frontend contract against OSD analyzer (PPL_SQL_VERSION=$os_version, schedule=$PPL_LINT_SCHEDULE)..." + log "Running detector validation against the candidate bundle (schedule=$PPL_LINT_SCHEDULE)..." ( cd "$osd_checkout" PPL_LINT_CONTRACT_DIR="$CONTRACT_DIR" \ PPL_LINT_SCHEDULE="$PPL_LINT_SCHEDULE" \ - PPL_SQL_VERSION="$os_version" \ - PPL_LINT_REPORT="$SQL_ROOT/frontend-report.json" \ - node -r ./src/setup_node_env "$FRONTEND_SCRIPT" + PPL_LINT_GRAMMAR_BUNDLE="$GRAMMAR_BUNDLE" \ + PPL_LINT_TARGET_MANIFEST="$TARGET_MANIFEST" \ + PPL_LINT_BACKEND_REPORT="$BACKEND_REPORT" \ + PPL_LINT_REPORT="$DETECTOR_REPORT" \ + node -r ./src/setup_node_env "$DETECTOR_SCRIPT" ) + log "Detector validation passed." } -if [[ "${SKIP_FRONTEND:-0}" != "1" ]]; then +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + run_backend +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test (using existing artifacts)." +fi + +if [[ "${SKIP_DETECTOR:-0}" != "1" ]]; then if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" log "Using existing OSD checkout: $OSD_CHECKOUT" @@ -95,20 +125,9 @@ if [[ "${SKIP_FRONTEND:-0}" != "1" ]]; then OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" log "OSD revision under test: $OSD_SHA" - run_frontend "$OSD_CHECKOUT" - log "Frontend contract passed." -else - log "SKIP_FRONTEND=1 — skipping the OSD frontend contract." -fi - -if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then - log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" - ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ - -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ - -Dppl.lint.report="$SQL_ROOT/backend-report.json" - log "Backend integration test passed." + run_detector "$OSD_CHECKOUT" else - log "SKIP_BACKEND=1 — skipping the SQL backend integration test." + log "SKIP_DETECTOR=1 — skipping the OSD detector contract." fi log "Done." diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md new file mode 100644 index 00000000000..f17a428b959 --- /dev/null +++ b/scripts/ppl-lint/README.md @@ -0,0 +1,195 @@ +# PPL lint rule validation + +A required, cross-repository GitHub Actions check that proves the OpenSearch +Dashboards (OSD) PPL lint detectors and the SQL backend still agree — on the +**same candidate runtime grammar** built by a SQL pull request. + +PPL language behavior lives in SQL; PPL lint detectors live in OSD. A SQL change +can silently invalidate an OSD rule (a parser refactor stops a detector matching, +or a semantic change makes a flagged query valid) without touching OSD. Neither +repository's own unit tests catch that. This check does. + +- **Design:** `ppl-lint-ci-validation-design.md` +- **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) +- **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) + +## The pipeline + +Three jobs run in a line; artifacts are the only bridge between them. + +``` +backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.json)──▶ + detector-validation ──▶ validation-result (the single required check) +``` + +1. **backend-validation** (OpenSearch CI container). Builds the SQL PR, starts + the Gradle test cluster, runs each contract's trigger/control queries against + `POST /_plugins/_ppl`, and — while the cluster is alive — exports: + - `ppl-grammar-bundle.json` — the candidate runtime grammar (`GET /_plugins/_ppl/_grammar`); + - `target.json` — `{ engineVersion, grammarHash, grammarBundle }`; + - `backend-report.json` — the observed HTTP behavior per query. +2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a + Node code dependency (no OSD server, no Monaco, no browser), then runs + [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner + deserializes the candidate bundle through OSD's production headless lint API + (`src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint`) and lints + each query with the **real** detectors on the **candidate** grammar. It then + asserts the detector-vs-backend differential. +3. **validation-result**. `if: always()`, `needs: [backend-validation, + detector-validation]`. Fails unless both succeeded — so a skipped detector + (because the backend failed first) still reds the check instead of looking + green. It writes the per-rule PR summary and uploads `run-manifest.json`. This + is the **only** job repo admins pin to branch protection. + +## Workflow inputs and modes + +| Trigger | Mode | OSD ref | Enforcing? | +| --- | --- | --- | --- | +| `pull_request` | SQL PR validation | `main` | **Yes** — the required check | +| `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | +| `schedule` (nightly) | full corpus + coverage | `main` | No | + +`workflow_dispatch` inputs: + +- `osd_ref` — an OSD commit or branch to validate instead of `main`. Resolved to + an immutable commit SHA and recorded in the run manifest. A manual run **cannot** + satisfy branch protection; merge the OSD change first, then rerun the required + `pull_request` check against OSD `main`. +- `schedule` — `pr` (fast blocking subset) or `nightly` (full corpus). + +## Local reproduction + +From the SQL checkout: + +```bash +# Backend IT (exports the bundle) then detector check against OSD main. +./scripts/ppl-lint-rule-validation.sh + +# Reuse an already-bootstrapped OSD checkout. +OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh + +# Reproduce a specific CI run's OSD revision (from run-manifest.json). +OSD_REF= ./scripts/ppl-lint-rule-validation.sh + +# Full nightly corpus + coverage assertion. +PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh + +# Re-run only one half (detector needs the backend artifacts to exist). +SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh +SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +``` + +The backend half writes `ppl-grammar-bundle.json`, `target.json`, and +`backend-report.json` to the SQL repo root; the detector half consumes them and +writes `detector-report.json`. + +### Runner environment contract + +`run-frontend-contract.mjs` is run from inside the OSD checkout with +`node -r ./src/setup_node_env` and reads: + +| Env var | Meaning | +| --- | --- | +| `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | +| `PPL_LINT_SCHEDULE` | `pr` or `nightly` | +| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | +| `PPL_LINT_TARGET_MANIFEST` | `target.json` (engine version + grammar hash) | +| `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | +| `PPL_LINT_REPORT` | where to write `detector-report.json` | +| `PPL_LINT_CONTRACT_FILE` | (optional) run a single spec instead of the dir | + +## Contract format (schema v3) + +One JSON file per rule under `contracts/`, listed in `manifest.json`. Each file +has a top-level `queries` map (each `{ role: "trigger"|"control", query }`) and a +version-scoped `expectations[]`. Exactly one expectation must match the candidate +backend version (zero or more than one fails before any query runs). + +```jsonc +{ + "schemaVersion": 3, + "ruleId": "union-min-datasets", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "wiring": { "detector": "union-min-datasets", "enabled": true, "severity": "error", ... }, + "backendFixture": { "indices": ["ACCOUNT"], "clusterSettings": { "calcite": true, "calciteFallback": false } }, + "frontendContext": { "isCalcite": true }, + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { "role": "trigger", "query": "| union [ source={{index}} ]" }, + "union-two-datasets-control": { "role": "control", "query": "| union [ source={{index}} ] [ source={{index}} ]" } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, "severity": "error", + "backend": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} +``` + +`backend.kind` is one of `rejection` (contracted 4xx + error type/reason), +`result-shape` (200 with datarow expectations), or `advisory` (soft 200-only +oracle). When a behavior changes in a new version, keep **both** version-scoped +expectations so the nightly matrix proves the rule still fires on the old version +while the candidate check proves the fix on the new one. + +### Pitfall: do not write pipe-first (`| command …`) trigger queries + +The detector half and the backend half must run the **byte-identical** query +(design's "Same queries" requirement). OSD's runtime lint path prepends a +synthetic `source=t ` prefix to any query that starts with a pipe, so linting +`| union [ source=idx ]` actually parses `source=t | union [ source=idx ]` — a +valid *mid-pipeline* union whose implicit upstream dataset makes the detector +stay silent. The backend, receiving the raw pipe-first query, still rejects it. +The two halves then disagree even though nothing is wrong. Write triggers in a +**query-initial** form (`union [ source=idx ]`, `multisearch [ search source=idx ]`) +that both sides accept verbatim. Until SQL emits `pipeStartRuleIndex` in the +grammar bundle (design §6, D-pipe), a pipe-first trigger with a distinct start +rule cannot be validated end to end. + +### The enforced set + +`manifest.json` partitions the corpus: + +- `enforced` — reviewed error rules with a deterministic backend rejection and a + valid negative control. These block `validation-result`. Phase one: + `unsupported-window-function-in-eventstats`, `multisearch-min-subsearch`, + `union-min-datasets`, `replace-wildcard-asymmetry`. +- `pendingReview` — error rules awaiting Peng/Chen usefulness review before + joining `enforced` (currently `field-validation`). +- `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the + nightly schedule for coverage and never block a PR. + +## Interpreting a failure + +| Failure | Meaning | +| --- | --- | +| Grammar bundle export fails | The candidate SQL build does not provide a usable runtime grammar. | +| Trigger no longer parses | The grammar changed ownership of the error or regressed. | +| Detector emits no diagnostic | The detector is incompatible with the candidate parse tree. | +| Detector flags the control | The detector became too broad. | +| Backend accepts the trigger | The lint rule's premise may be fixed or stale. | +| Backend rejects the control | Query, fixture, settings, or SQL behavior regressed. | +| No version expectation matches | The rule test does not cover the candidate version. | + +CI never rewrites expected results. A behavior change is an intentional, reviewed +edit to a versioned expectation **and** the corresponding OSD rule. If a SQL +change depends on an OSD rule update, merge the OSD change first, then rerun the +required SQL check against OSD `main`. + +## Artifacts + +Every run uploads: `run-manifest.json` (exact SQL SHA, OSD SHA, mode, backend +version, grammar hash, selected validation set), the candidate grammar bundle, +the backend and detector reports, the committed contracts used, and the job logs. diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs new file mode 100644 index 00000000000..4cc80fe9513 --- /dev/null +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -0,0 +1,157 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assemble the PPL lint validation run manifest and the compact per-rule PR + * summary in the result job (design §3.3, §4.4, T10). + * + * Inputs (env, all optional so a partial run still produces a manifest): + * SQL_SHA, OSD_REF, OSD_SHA, EVENT_NAME, SCHEDULE, + * BACKEND_RESULT, DETECTOR_RESULT, GITHUB_STEP_SUMMARY. + * Artifact files under ./artifacts (downloaded from both jobs): + * target.json (engineVersion + grammarHash), backend-report.json, + * detector-report.json. + * + * Outputs: + * run-manifest.json in the workspace root; a markdown table appended to + * $GITHUB_STEP_SUMMARY. + */ + +import fs from 'fs'; +import path from 'path'; + +const ARTIFACTS = 'artifacts'; + +function readJson(file) { + try { + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not parse ${file}: ${error.message}`); + } + return undefined; +} + +function main() { + const target = readJson(path.join(ARTIFACTS, 'target.json')) || {}; + const detector = readJson(path.join(ARTIFACTS, 'detector-report.json')) || {}; + const backend = readJson(path.join(ARTIFACTS, 'backend-report.json')) || []; + + const eventName = process.env.EVENT_NAME || ''; + const osdRef = process.env.OSD_REF || 'main'; + const mode = + eventName === 'pull_request' + ? 'sql-pr-validation' + : eventName === 'schedule' + ? 'nightly' + : osdRef && osdRef !== 'main' + ? 'osd-branch-evidence' + : 'manual'; + + const backendResult = process.env.BACKEND_RESULT || 'unknown'; + const detectorResult = process.env.DETECTOR_RESULT || 'unknown'; + const passed = backendResult === 'success' && detectorResult === 'success'; + + // The selected validation set is the set of rules the detector run actually + // evaluated (post schedule filtering). + const validationSet = Array.from( + new Set((detector.results || []).map((r) => r.ruleId)) + ).sort(); + + const manifest = { + mode, + // A workflow_dispatch osd_ref run is pre-merge evidence, never a + // branch-protection result (design §4.1.1, T11). + requiredCheck: eventName === 'pull_request', + event: eventName, + schedule: process.env.SCHEDULE || detector.schedule || 'pr', + sqlSha: process.env.SQL_SHA || '', + osdRef, + osdSha: process.env.OSD_SHA || '', + engineVersion: target.engineVersion || detector.engineVersion || '', + grammarHash: target.grammarHash || detector.grammarHash || '', + differential: !!detector.differential, + validationSet, + result: { + backend: backendResult, + detector: detectorResult, + passed, + }, + }; + + fs.writeFileSync('run-manifest.json', JSON.stringify(manifest, null, 2)); + + writeSummary(manifest, detector, backend); +} + +/** Compact per-rule PR summary: Rule | Version | Grammar | Detector | Backend | Result. */ +function writeSummary(manifest, detector, backend) { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) { + return; + } + + const backendByKey = new Map(); + for (const e of Array.isArray(backend) ? backend : []) { + backendByKey.set(`${e.ruleId}::${e.queryName}`, e); + } + + const shortHash = (h) => (h ? String(h).replace(/^sha256:/, '').slice(0, 12) : '—'); + + const lines = []; + lines.push('## PPL lint rule validation'); + lines.push(''); + lines.push(`- Mode: \`${manifest.mode}\`${manifest.requiredCheck ? ' (required)' : ' (non-enforcing)'}`); + lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); + lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (ref \`${manifest.osdRef}\`)`); + lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); + lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); + lines.push( + `- Result: backend **${manifest.result.backend}**, detector **${manifest.result.detector}** → ` + + `**${manifest.result.passed ? 'PASS' : 'FAIL'}**` + ); + lines.push(''); + lines.push('| Rule | Query | Version | Grammar | Detector | Backend | Result |'); + lines.push('| ---- | ----- | ------- | ------- | -------- | ------- | ------ |'); + + for (const r of detector.results || []) { + const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); + const detectorCell = `${r.actual}/${r.expected}${r.severities && r.severities.length ? ` (${r.severities.join(',')})` : ''}`; + const backendCell = be + ? be.rejected + ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` + : 'accepted' + : '—'; + const ok = + r.actual === r.expected && (!be || (r.role === 'trigger' ? be.rejected : !be.rejected)); + lines.push( + `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ok ? 'Pass' : 'Fail'} |` + ); + } + + if ((detector.failures || []).length > 0) { + lines.push(''); + lines.push('

Failures'); + lines.push(''); + for (const f of detector.failures) { + lines.push(`- ${f}`); + } + lines.push(''); + lines.push('
'); + } + + lines.push(''); + try { + fs.appendFileSync(summaryPath, lines.join('\n') + '\n'); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not write step summary: ${error.message}`); + } +} + +main(); diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index e3163e2ad44..8bd1ff5bad1 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -4,7 +4,7 @@ */ /** - * SQL-owned frontend contract adapter for the PPL lint rule validation CI. + * SQL-owned detector-validation runner for the PPL lint rule validation CI. * * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, * for example: @@ -12,56 +12,61 @@ * cd .ci/OpenSearch-Dashboards * PPL_LINT_CONTRACT_DIR= \ * PPL_LINT_SCHEDULE=pr \ - * PPL_SQL_VERSION= \ - * PPL_LINT_REPORT= \ + * PPL_LINT_GRAMMAR_BUNDLE= \ + * PPL_LINT_TARGET_MANIFEST= \ + * PPL_LINT_BACKEND_REPORT= \ + * PPL_LINT_REPORT= \ * node -r ./src/setup_node_env \ * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" * * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles - * `packages/osd-monaco/src/**` TypeScript on `require()` regardless of where the - * entry script lives. That is what lets this SQL-owned `.mjs` load the compiled - * OSD analyzer without OSD's own Jest. + * `src/plugins/**` and `packages/osd-monaco/src/**` TypeScript on `require()` + * regardless of where the entry script lives. That is what lets this SQL-owned + * `.mjs` load OSD's Node-safe headless lint API without OSD's own Jest. * - * The analyzer, catalog and detector registry are NOT re-exported from the - * `@osd/monaco` package barrel, so they are loaded via their deep module paths. - * Because this is an ES module, `require` is obtained with `createRequire`, and - * the modules are resolved against the OSD checkout root (process.cwd()) rather - * than the location of this script (which lives in the SQL repo, not OSD). - * - * This adapter is the frontend half of a schema-v2 cross-repository differential + * This is the detector half of a schema-v3 cross-repository differential * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). - * It asserts three things per rule: + * Unlike the earlier PoC — which linted with the compiled analyzer or a + * hand-rolled reparse against OSD `main`'s checked-in grammar — it lints against + * the *candidate* runtime grammar bundle the SQL backend job exported, via OSD's + * production headless API (`headless_ppl_lint`). Both halves therefore validate + * the exact same candidate grammar (design §4.3). + * + * It asserts, per contract: * 1. Wiring: the OSD catalog entry deep-equals the contract's `wiring` block, - * so a silently removed/retyped/regated detector reds the build. - * 2. Diagnostics: for each case the analyzer emits exactly the contracted - * number of `ruleId` diagnostics (the differential the backend half pins to - * live-engine behavior). - * 3. Coverage (nightly only): every enabled catalog rule has a contract file. + * so a silently removed/retyped/re-gated/re-severitied detector reds the + * build. + * 2. Detector: for the single version expectation that matches the candidate + * backend version, each query emits exactly the contracted number of + * `ruleId` diagnostics at the contracted severity. + * 3. Differential (when PPL_LINT_BACKEND_REPORT is supplied): the observed + * backend behavior for each query agrees with the observed detector output + * — a trigger the detector flags is one the backend rejected; a control the + * detector passes is one the backend accepted (design §3.2, §4.3). + * 4. Coverage (nightly only): every enabled catalog rule has a contract file. */ import fs from 'fs'; import path from 'path'; import { createRequire } from 'module'; -const RULE_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; -const CATALOG_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; -const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/src/ppl/lint/detector_registry'; -const LINT_RUNNER_MODULE = 'packages/osd-monaco/src/ppl/lint/lint_runner'; -const RULE_INDEX_MODULE = 'packages/osd-monaco/src/ppl/lint/rule_index'; -const GRAMMAR_MODULE = 'packages/osd-antlr-grammar/target/index.js'; -// Explain lint lives only on OSD branches that ship the explain rule class; the -// adapter feature-detects it and skips explain cases when it is absent. -const RUN_EXPLAIN_MODULE = 'packages/osd-monaco/src/ppl/lint/explain/run_explain_lint'; +// OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved +// against the OSD checkout root, not this script's SQL-repo location. +const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +// The Monaco-free engine barrel (@osd/monaco/ppl-lint) exposes the catalog; the +// detector registry is a deep import used only for the wiring registration check. +const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; function log(message) { // eslint-disable-next-line no-console - console.log(`[ppl-lint-frontend-contract] ${message}`); + console.log(`[ppl-lint-detector-contract] ${message}`); } function fatal(message) { // eslint-disable-next-line no-console - console.error(`[ppl-lint-frontend-contract] FATAL: ${message}`); + console.error(`[ppl-lint-detector-contract] FATAL: ${message}`); process.exit(2); } @@ -138,19 +143,84 @@ function loadOsd() { } }; - const { PPLLanguageAnalyzer } = resolveOsd(RULE_MODULE); + const headless = resolveOsd(HEADLESS_MODULE); const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); - const { getDetector } = resolveOsd(DETECTOR_REGISTRY_MODULE); - const { runLint } = resolveOsd(LINT_RUNNER_MODULE); - const ruleIndex = resolveOsd(RULE_INDEX_MODULE); - const grammar = resolveOsd(GRAMMAR_MODULE, { optional: true }); - const explain = resolveOsd(RUN_EXPLAIN_MODULE, { optional: true }); + const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); + + const { deserializeBundleOrThrow, lintQueryWithBundle } = headless; + if (typeof deserializeBundleOrThrow !== 'function' || typeof lintQueryWithBundle !== 'function') { + fatal( + `Headless lint API not found in ${HEADLESS_MODULE}. ` + + `Expected exports deserializeBundleOrThrow + lintQueryWithBundle. ` + + `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` + ); + } + if (typeof getBundledCatalog !== 'function') { + fatal(`getBundledCatalog not found in ${CATALOG_MODULE}.`); + } + + const getDetector = registry && registry.getDetector; + return { deserializeBundleOrThrow, lintQueryWithBundle, getBundledCatalog, getDetector, osdRoot }; +} + +/** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ +function loadCandidateGrammar(osd) { + const bundlePath = process.env.PPL_LINT_GRAMMAR_BUNDLE; + if (!bundlePath) { + fatal( + 'PPL_LINT_GRAMMAR_BUNDLE is not set. Detector validation lints against the candidate ' + + 'runtime grammar bundle exported by the backend job; there is no compiled fallback.' + ); + } + if (!fs.existsSync(bundlePath)) { + fatal(`Candidate grammar bundle not found: ${bundlePath}`); + } + let bundle; + try { + bundle = JSON.parse(fs.readFileSync(bundlePath, 'utf8')); + } catch (error) { + fatal(`Could not parse grammar bundle ${bundlePath}: ${error.message}`); + } + try { + return osd.deserializeBundleOrThrow(bundle); + } catch (error) { + fatal(`Could not deserialize candidate grammar bundle: ${error.message}`); + } + return undefined; // unreachable +} - if (typeof PPLLanguageAnalyzer !== 'function') { - fatal(`PPLLanguageAnalyzer was not a constructor when loaded from ${RULE_MODULE}.`); +/** Read the target manifest (engineVersion + grammarHash) written beside the bundle. */ +function loadTarget() { + const targetPath = process.env.PPL_LINT_TARGET_MANIFEST; + if (targetPath && fs.existsSync(targetPath)) { + try { + return JSON.parse(fs.readFileSync(targetPath, 'utf8')); + } catch (error) { + log(`WARN: could not parse target manifest ${targetPath}: ${error.message}`); + } } + // Back-compat / local runs without a target manifest. + return { engineVersion: process.env.PPL_SQL_VERSION || '', grammarHash: '' }; +} - return { PPLLanguageAnalyzer, getBundledCatalog, getDetector, runLint, ruleIndex, grammar, explain, osdRoot }; +/** Index the backend report by `${ruleId}::${queryName}` for the differential. */ +function loadBackendReport() { + const reportPath = process.env.PPL_LINT_BACKEND_REPORT; + if (!reportPath || !fs.existsSync(reportPath)) { + return undefined; + } + let entries; + try { + entries = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + } catch (error) { + log(`WARN: could not parse backend report ${reportPath}: ${error.message}`); + return undefined; + } + const byKey = new Map(); + for (const entry of Array.isArray(entries) ? entries : []) { + byKey.set(`${entry.ruleId}::${entry.queryName}`, entry); + } + return byKey; } /** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ @@ -161,17 +231,79 @@ function parseVersion(v) { return [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)]; } -function versionGte(a, b) { - const pa = parseVersion(a); - const pb = parseVersion(b); - if (!pa || !pb) return true; // unknown → do not skip +function compareVersion(a, b) { for (let i = 0; i < 3; i++) { - if (pa[i] > pb[i]) return true; - if (pa[i] < pb[i]) return false; + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * Test a space-separated semver range (e.g. ">=3.6.0 <3.8.0") against the + * candidate backend version. An empty range or an unknown version matches (do + * not over-filter). Mirrors PplLintRuleValidationIT.versionMatchesRange. + */ +function versionMatchesRange(range, version) { + if (!range || !range.trim()) return true; + const have = parseVersion(version); + if (!have) return true; + for (const token of range.trim().split(/\s+/)) { + let op = '='; + let ver = token; + if (token.startsWith('>=')) { + op = '>='; + ver = token.slice(2); + } else if (token.startsWith('<=')) { + op = '<='; + ver = token.slice(2); + } else if (token.startsWith('>')) { + op = '>'; + ver = token.slice(1); + } else if (token.startsWith('<')) { + op = '<'; + ver = token.slice(1); + } else if (token.startsWith('=')) { + op = '='; + ver = token.slice(1); + } + const cmp = compareVersion(have, parseVersion(ver) || [0, 0, 0]); + const ok = + (op === '>=' && cmp >= 0) || + (op === '<=' && cmp <= 0) || + (op === '>' && cmp > 0) || + (op === '<' && cmp < 0) || + (op === '=' && cmp === 0); + if (!ok) return false; } return true; } +/** + * Select the single expectation that applies to the candidate version + engine. + * Exactly one must match (design §5.3): zero means the rule test does not cover + * this version; more than one means overlapping ranges. Both fail. + */ +function selectExpectation(spec, version, isCalcite, failures) { + const expectations = spec.expectations || []; + const matches = expectations.filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && isCalcite !== true) return false; + return true; + }); + if (matches.length === 1) { + return matches[0]; + } + const label = version || 'unknown'; + if (matches.length === 0) { + failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + } else { + failures.push( + `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} (exactly one required).` + ); + } + return undefined; +} + /** * Assert the OSD catalog entry deep-equals the contract's `wiring` block. This is * the primary OSD-drift tripwire: if a detector is removed, retyped, re-gated or @@ -198,7 +330,9 @@ function checkWiring(spec, catalog, getDetector, failures) { ]; for (const [name, expected, actual] of checks) { if (expected !== undefined && expected !== actual) { - failures.push(`[${ruleId}] wiring.${name} expected ${JSON.stringify(expected)} but catalog has ${JSON.stringify(actual)}.`); + failures.push( + `[${ruleId}] wiring.${name} expected ${JSON.stringify(expected)} but catalog has ${JSON.stringify(actual)}.` + ); } } @@ -206,12 +340,14 @@ function checkWiring(spec, catalog, getDetector, failures) { const a = entry.appliesTo || {}; for (const key of ['minVersion', 'maxVersion', 'engine']) { if (wiring.appliesTo[key] !== undefined && wiring.appliesTo[key] !== a[key]) { - failures.push(`[${ruleId}] wiring.appliesTo.${key} expected ${JSON.stringify(wiring.appliesTo[key])} but catalog has ${JSON.stringify(a[key])}.`); + failures.push( + `[${ruleId}] wiring.appliesTo.${key} expected ${JSON.stringify(wiring.appliesTo[key])} but catalog has ${JSON.stringify(a[key])}.` + ); } } } - if (wiring.detector && typeof getDetector(wiring.detector) !== 'function') { + if (wiring.detector && typeof getDetector === 'function' && typeof getDetector(wiring.detector) !== 'function') { failures.push(`[${ruleId}] has no registered detector "${wiring.detector}".`); } @@ -219,16 +355,21 @@ function checkWiring(spec, catalog, getDetector, failures) { } /** - * Build the per-case lint context. Derives `fields`/`typeMap` from the - * `deriveFromMapping` block (a single source shared with the backend seeding), - * and sets an enable override for default-off rules that declare `forceEnable`. + * Build the per-contract lint context passed to `lintQueryWithBundle`. Derives + * `fields`/`typeMap` from the `deriveFromMapping` block (a single source shared + * with the backend seeding), pins `dataSourceVersion`/`knownVersion` to the + * candidate backend version so version filtering matches the backend, and sets + * an enable override for default-off rules that declare `forceEnable`. */ -function buildContext(spec, sqlVersion) { +function buildContext(spec, engineVersion) { const fc = spec.frontendContext || {}; const context = { isCalcite: fc.isCalcite !== false, - dataSourceVersion: sqlVersion, - grammarSurface: spec.grammarSurface === 'runtime-bundle' ? 'runtime-bundle' : 'compiled-simplified', + dataSourceVersion: engineVersion || undefined, + // Pin the "latest verified engine" to the candidate version rather than the + // hardcoded OSD_KNOWN_VERSION ('3.7.0'), which can mis-filter rules near a + // version boundary (design §4.3, D-version). + knownVersion: engineVersion || undefined, }; const mapping = fc.deriveFromMapping; @@ -257,99 +398,36 @@ function buildContext(spec, sqlVersion) { return context; } -/** Count diagnostics for this rule via the compiled-simplified analyzer. */ -function lintCompiled(analyzer, query, context, ruleId) { - const result = analyzer.lint(query, context); - return result.diagnostics.filter((d) => d.ruleId === ruleId); -} - -/** - * Count diagnostics for a runtime-only rule by parsing with the exported runtime - * grammar and running the detector registry directly. This exercises OSD-main's - * runtime grammar, NOT the cluster-versioned bundle production fetches, so it is - * a wiring/coverage check rather than a true cluster-grammar fidelity check. - * Returns undefined when the runtime grammar can't reach the rule on this OSD - * checkout (the rule's parser rules are absent) so the caller can skip cleanly. - */ -function lintRuntime(osd, spec, query, context, ruleId) { - const { grammar, runLint, ruleIndex } = osd; - if (!grammar || !grammar.OpenSearchPPLParser || !grammar.OpenSearchPPLLexer) { - return undefined; - } - const antlr = requireAntlr(osd.osdRoot); - if (!antlr) { - return undefined; - } - const { OpenSearchPPLLexer, OpenSearchPPLParser } = grammar; - - const runtimeMap = new Map(); - const names = OpenSearchPPLParser.ruleNames || []; - for (let i = 0; i < names.length; i++) { - runtimeMap.set(names[i], i); - } - - // The exported runtime grammar on this OSD checkout may predate the command a - // runtime-only rule keys off (union/multisearch/replace are absent on the - // legacy `opensearch_ppl` grammar). Detecting the absence here lets the caller - // record a clean skip — the wiring assertion already ran — instead of a false - // "0 diagnostics" failure. - const required = spec.requiredParserRules || []; - for (const name of required) { - if (!runtimeMap.has(name)) { - return undefined; - } - } - - const input = antlr.CharStream.fromString(query); - const lexer = new OpenSearchPPLLexer(input); - const tokenStream = new antlr.CommonTokenStream(lexer); - const parser = new OpenSearchPPLParser(tokenStream); - parser.removeErrorListeners(); - const tree = parser.root ? parser.root() : parser.pplStatement && parser.pplStatement(); - if (!tree) { - return undefined; - } - - const ruleNameToIndex = ruleIndex.createRuntimeRuleNameToIndex(runtimeMap); - - const diagnostics = runLint(tree, { - ruleNameToIndex, - dataSourceVersion: context.dataSourceVersion, - context: { ...context, grammarSurface: 'runtime-bundle' }, - }); - return diagnostics.filter((d) => d.ruleId === ruleId); -} - -let cachedAntlr; -function requireAntlr(osdRoot) { - if (cachedAntlr !== undefined) { - return cachedAntlr || undefined; - } - try { - const require = createRequire(path.join(osdRoot, 'noop.js')); - cachedAntlr = require('antlr4ng'); - } catch { - cachedAntlr = null; - } - return cachedAntlr || undefined; -} - function main() { const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; - const sqlVersion = process.env.PPL_SQL_VERSION; const reportPath = process.env.PPL_LINT_REPORT; const osd = loadOsd(); - const { PPLLanguageAnalyzer, getBundledCatalog, getDetector, osdRoot } = osd; + const { getBundledCatalog, getDetector, lintQueryWithBundle, osdRoot } = osd; const catalog = getBundledCatalog(); - const analyzer = new PPLLanguageAnalyzer(); + + const grammar = loadCandidateGrammar(osd); + const target = loadTarget(); + const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; + const backendReport = loadBackendReport(); const contracts = loadContracts(); const failures = []; - const report = { osdRoot, schedule, sqlVersion, results: [] }; + const report = { + osdRoot, + schedule, + engineVersion, + grammarHash: target.grammarHash || '', + differential: !!backendReport, + results: [], + }; log(`OSD root: ${osdRoot}`); - log(`schedule=${schedule} PPL_SQL_VERSION=${sqlVersion || '(unset)'} contracts=${contracts.length}`); + log( + `schedule=${schedule} engineVersion=${engineVersion || '(unset)'} ` + + `grammarHash=${target.grammarHash || '(unset)'} differential=${!!backendReport} ` + + `contracts=${contracts.length}` + ); for (const { file, spec } of contracts) { const ruleId = spec.ruleId; @@ -367,66 +445,100 @@ function main() { continue; } - const context = buildContext(spec, sqlVersion); - const isRuntime = context.grammarSurface === 'runtime-bundle'; - - for (const testCase of spec.cases || []) { - const query = testCase.query.split('{{index}}').join(index); - const fe = testCase.frontend || {}; - const expected = fe.diagnosticCount; + const context = buildContext(spec, engineVersion); + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures); + if (!expectation) { + continue; + } - // Per-case version/engine gate mirrors the backend so both halves skip - // identically instead of disagreeing on a self-suppressed rule. - if (testCase.minVersionRequired && !versionGte(sqlVersion, testCase.minVersionRequired)) { - log(`SKIP ${ruleId}/${testCase.id} (needs >= ${testCase.minVersionRequired}, have ${sqlVersion || 'unknown'})`); - continue; - } - if (testCase.engineRequired === 'calcite' && context.isCalcite !== true) { - log(`SKIP ${ruleId}/${testCase.id} (needs calcite engine)`); + const queries = spec.queries || {}; + const expectedQueries = expectation.queries || {}; + for (const queryName of Object.keys(expectedQueries)) { + const queryDef = queries[queryName]; + if (!queryDef) { + failures.push(`[${ruleId}] expectation references unknown query "${queryName}".`); continue; } + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + const expected = expectedQueries[queryName]; + const expectedCount = expected.detectorCount; - let matches; - if (testCase.explainFixture) { - matches = lintExplain(osd, spec, testCase, context, ruleId); - if (matches === undefined) { - log(`SKIP ${ruleId}/${testCase.id} (explain lint unavailable on this OSD checkout)`); - continue; - } - } else if (isRuntime) { - matches = lintRuntime(osd, spec, query, context, ruleId); - if (matches === undefined) { - // Runtime grammar can't reach this rule on this OSD checkout: the - // wiring assertion above still ran, so record a skip (not a failure). - log(`SKIP ${ruleId}/${testCase.id} (runtime grammar rule absent on this OSD checkout; wiring asserted)`); - report.results.push({ ruleId, caseId: testCase.id, query, expected, actual: null, skipped: 'runtime-grammar-absent' }); - continue; - } - } else { - matches = lintCompiled(analyzer, query, context, ruleId); - } - + const result = lintQueryWithBundle(query, grammar, context); + const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); const actual = matches.length; - const ok = actual === expected; + const ok = actual === expectedCount; - log(` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${testCase.id}: expected ${expected}, got ${actual} — ${query}`); + log( + ` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${queryName} (${role}): ` + + `expected ${expectedCount}, got ${actual} — ${query}` + ); const severityOk = - !fe.severity || actual === 0 || matches.every((m) => m.severity === fe.severity); + !expected.severity || actual === 0 || matches.every((m) => m.severity === expected.severity); const messageOk = - !fe.matchMessage || matches.some((m) => (m.message || '').includes(fe.matchMessage)); - - report.results.push({ ruleId, caseId: testCase.id, query, expected, actual, severities: matches.map((m) => m.severity) }); + !expected.matchMessage || matches.some((m) => (m.message || '').includes(expected.matchMessage)); + + const resultEntry = { + ruleId, + queryName, + role, + query, + expected: expectedCount, + actual, + severities: matches.map((m) => m.severity), + }; if (!ok) { - failures.push(`[${ruleId}/${testCase.id}] expected ${expected} "${ruleId}" diagnostic(s), got ${actual} for: ${query}`); + failures.push( + `[${ruleId}/${queryName}] expected ${expectedCount} "${ruleId}" diagnostic(s), got ${actual} for: ${query}` + ); } if (!severityOk) { - failures.push(`[${ruleId}/${testCase.id}] expected severity "${fe.severity}" for: ${query}`); + failures.push(`[${ruleId}/${queryName}] expected severity "${expected.severity}" for: ${query}`); } if (!messageOk) { - failures.push(`[${ruleId}/${testCase.id}] expected message to contain "${fe.matchMessage}" for: ${query}`); + failures.push(`[${ruleId}/${queryName}] expected message to contain "${expected.matchMessage}" for: ${query}`); } + + // Differential: the observed backend behavior must agree with the observed + // detector output through the shared contract (design §3.2, §4.3). A + // rejection-kind query the backend rejected must be one the detector flags; + // a success/advisory query the backend accepted must be one the detector + // passes. This catches drift the two halves would otherwise hide by both + // pinning to the same JSON. + if (backendReport) { + const backendKind = expected.backend && expected.backend.kind; + const expectRejected = backendKind === 'rejection'; + const be = backendReport.get(`${ruleId}::${queryName}`); + if (!be) { + failures.push(`[${ruleId}/${queryName}] no backend report entry (backend did not run this query).`); + } else { + resultEntry.backendRejected = !!be.rejected; + if (!!be.rejected !== expectRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: backend ${be.rejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Trigger/control cross-check against the detector's own verdict. + const detectorFlagged = actual > 0; + if (role === 'trigger' && detectorFlagged !== !!be.rejected) { + failures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if (role === 'control' && (detectorFlagged || be.rejected)) { + failures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + } + } + + report.results.push(resultEntry); } } @@ -452,39 +564,13 @@ function main() { if (failures.length > 0) { // eslint-disable-next-line no-console - console.error(`[ppl-lint-frontend-contract] FAIL: ${failures.length} problem(s):\n- ${failures.join('\n- ')}`); + console.error( + `[ppl-lint-detector-contract] FAIL: ${failures.length} problem(s):\n- ${failures.join('\n- ')}` + ); process.exit(1); } - log(`PASS: all contracts agreed with the OSD analyzer (schedule=${schedule}).`); -} - -/** - * Explain-case handling. Loads the captured plan fixture and runs the OSD explain - * lint over it. Returns undefined when the explain rule class is not present on - * this OSD checkout (feature-detected via the optional module). - */ -function lintExplain(osd, spec, testCase, context, ruleId) { - if (!osd.explain || typeof osd.explain.runExplainLint !== 'function') { - return undefined; - } - const dir = process.env.PPL_LINT_CONTRACT_DIR; - if (!dir) { - return undefined; - } - const fixturePath = path.join(dir, testCase.explainFixture); - if (!fs.existsSync(fixturePath)) { - return undefined; - } - const plan = JSON.parse(fs.readFileSync(fixturePath, 'utf8')); - const query = testCase.query.split('{{index}}').join(spec.index); - const diagnostics = osd.explain.runExplainLint(plan, { - query, - overrides: context.overrides, - dataSourceVersion: context.dataSourceVersion, - isCalcite: context.isCalcite, - }); - return (diagnostics || []).filter((d) => d.ruleId === ruleId); + log(`PASS: all contracts agreed with the OSD detectors on the candidate bundle (schedule=${schedule}).`); } main(); From 5e931675892f06fc23094f8fa8c676fa04188496 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 20 Jul 2026 16:22:21 -0700 Subject: [PATCH 27/78] fix(ci): pipefail so detector runner failure is not masked by tee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector-validation step piped the runner through `tee`, so the step took tee's (success) exit status and a real runner failure — e.g. the OSD headless module being absent on OSD main — went green as a vacuous pass. The first live PR run hit exactly this: the runner exited 2 with "Expected OSD module not found ... headless_ppl_lint", yet detector-validation and validation-result both reported success. Add `set -o pipefail` so node's non-zero exit propagates and the required check correctly reds until the OSD headless API merges. Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-rule-validation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 83fc01d0734..df35a0b1569 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -224,6 +224,10 @@ jobs: PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json run: | + # pipefail so the runner's non-zero exit propagates through `tee` — + # otherwise the pipeline takes tee's (success) status and a real + # detector failure would go green (a vacuous pass). + set -o pipefail node -r ./src/setup_node_env \ "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ | tee "$GITHUB_WORKSPACE/detector-contract.log" From d76c28632a59baddd37f9ccf1187be2d112aa020 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 20 Jul 2026 17:39:50 -0700 Subject: [PATCH 28/78] feat(ci): allow workflow_dispatch to validate against an OSD fork detector-validation hardcoded repository: opensearch-project/OpenSearch-Dashboards, so osd_ref could only resolve commits/branches that exist upstream. An unmerged OSD change on a fork (e.g. the headless lint API before it lands on OSD main) could not be validated end to end. Add an osd_repo workflow_dispatch input (default opensearch-project/OpenSearch-Dashboards) that the OSD checkout honors, thread osd_repo through the detector job output into the run manifest + PR summary, and treat any non-upstream-main target as osd-branch-evidence (never a required check). The required pull_request run is unchanged: it still checks out upstream OSD main. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 29 +++++++++++++------ scripts/ppl-lint/README.md | 7 +++++ scripts/ppl-lint/assemble-run-manifest.mjs | 7 +++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index df35a0b1569..367ee0976e0 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -31,9 +31,10 @@ name: PPL lint rule validation # Modes (design §3.4, §4.1.1): # - pull_request: SQL PR validation against OSD `main`. The ONLY enforcing mode; # this is what branch protection pins to. Runs the fast schedule:pr subset. -# - workflow_dispatch (osd_ref): pre-merge evidence for an unmerged OSD branch. -# Records the resolved immutable OSD commit SHA but CANNOT satisfy branch -# protection — only the pull_request run does. +# - workflow_dispatch (osd_repo + osd_ref): pre-merge evidence for an unmerged +# OSD branch, optionally on a fork (osd_repo). Records the resolved immutable +# OSD commit SHA but CANNOT satisfy branch protection — only the pull_request +# run does. # - schedule (nightly): the full corpus + a coverage assertion. on: @@ -42,6 +43,10 @@ on: - cron: '0 10 * * *' workflow_dispatch: inputs: + osd_repo: + description: OSD repository to check out (a fork, for pre-merge evidence). Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string osd_ref: description: OSD commit or branch to validate instead of main (pre-merge evidence only) required: false @@ -133,6 +138,7 @@ jobs: needs: backend-validation runs-on: ubuntu-latest outputs: + osd_repo: ${{ steps.osd-ref.outputs.repo }} osd_ref: ${{ steps.osd-ref.outputs.ref }} osd_sha: ${{ steps.osd-rev.outputs.sha }} schedule: ${{ steps.schedule.outputs.value }} @@ -140,14 +146,18 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - # The required pull_request run always validates against OSD `main`; only a - # manual workflow_dispatch may target an unmerged OSD ref, and that run is - # pre-merge evidence, not a branch-protection result (design §4.1.1). + # The required pull_request run always validates against OSD `main` on the + # canonical repo; only a manual workflow_dispatch may target an unmerged OSD + # ref (and optionally a fork), and that run is pre-merge evidence, not a + # branch-protection result (design §4.1.1). - name: Resolve OSD ref id: osd-ref env: REQUESTED_REF: ${{ inputs.osd_ref }} - run: echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + REQUESTED_REPO: ${{ inputs.osd_repo }} + run: | + echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" + echo "repo=${REQUESTED_REPO:-opensearch-project/OpenSearch-Dashboards}" >> "$GITHUB_OUTPUT" - name: Resolve contract schedule id: schedule @@ -173,7 +183,7 @@ jobs: - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - repository: opensearch-project/OpenSearch-Dashboards + repository: ${{ steps.osd-ref.outputs.repo }} ref: ${{ steps.osd-ref.outputs.ref }} path: .ci/OpenSearch-Dashboards @@ -184,7 +194,7 @@ jobs: run: | sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "OSD revision: \`$sha\` (ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + echo "OSD revision: \`$sha\` (repo: ${{ steps.osd-ref.outputs.repo }}, ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding # it, so an OSD toolchain bump does not silently drift this job. @@ -274,6 +284,7 @@ jobs: - name: Assemble run manifest and summary env: SQL_SHA: ${{ github.sha }} + OSD_REPO: ${{ needs.detector-validation.outputs.osd_repo }} OSD_REF: ${{ needs.detector-validation.outputs.osd_ref }} OSD_SHA: ${{ needs.detector-validation.outputs.osd_sha }} EVENT_NAME: ${{ github.event_name }} diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f17a428b959..e9578517608 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -51,12 +51,19 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j `workflow_dispatch` inputs: +- `osd_repo` — the OSD repository to check out, for validating an unmerged change + that lives on a fork. Defaults to `opensearch-project/OpenSearch-Dashboards`. + The `osd_ref` must exist in this repo (a purely local commit cannot be fetched). - `osd_ref` — an OSD commit or branch to validate instead of `main`. Resolved to an immutable commit SHA and recorded in the run manifest. A manual run **cannot** satisfy branch protection; merge the OSD change first, then rerun the required `pull_request` check against OSD `main`. - `schedule` — `pr` (fast blocking subset) or `nightly` (full corpus). +To validate an OSD change that is not yet merged, push it to a branch on your OSD +fork and dispatch with `osd_repo=/OpenSearch-Dashboards` and +`osd_ref=`. + ## Local reproduction From the SQL checkout: diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index 4cc80fe9513..840e631ef88 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -43,12 +43,14 @@ function main() { const eventName = process.env.EVENT_NAME || ''; const osdRef = process.env.OSD_REF || 'main'; + const osdRepo = process.env.OSD_REPO || 'opensearch-project/OpenSearch-Dashboards'; + const isUpstreamMain = osdRepo === 'opensearch-project/OpenSearch-Dashboards' && osdRef === 'main'; const mode = eventName === 'pull_request' ? 'sql-pr-validation' : eventName === 'schedule' ? 'nightly' - : osdRef && osdRef !== 'main' + : !isUpstreamMain ? 'osd-branch-evidence' : 'manual'; @@ -70,6 +72,7 @@ function main() { event: eventName, schedule: process.env.SCHEDULE || detector.schedule || 'pr', sqlSha: process.env.SQL_SHA || '', + osdRepo, osdRef, osdSha: process.env.OSD_SHA || '', engineVersion: target.engineVersion || detector.engineVersion || '', @@ -107,7 +110,7 @@ function writeSummary(manifest, detector, backend) { lines.push(''); lines.push(`- Mode: \`${manifest.mode}\`${manifest.requiredCheck ? ' (required)' : ' (non-enforcing)'}`); lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); - lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (ref \`${manifest.osdRef}\`)`); + lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (${manifest.osdRepo} @ \`${manifest.osdRef}\`)`); lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); lines.push( From 5806d209113a966fda5598a616bb499b9efdfd08 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 21 Jul 2026 21:50:09 -0700 Subject: [PATCH 29/78] fix(ci): resolve OSD target via repo variables so PR check can validate against the paired OSD ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector-validation job imports OSD's headless lint API (src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint), which is not yet on OSD main — it lives on the paired branch Hanyu-W/OpenSearch-Dashboards@ppl-lint-headless-api. The required pull_request run hardcoded opensearch-project/OpenSearch-Dashboards@main, so it failed on 'Expected OSD module not found' while the fork-targeting workflow_dispatch run passed. Resolve the OSD repo/ref in precedence order: workflow_dispatch input > OSD_REPO/OSD_REF repo variables > canonical opensearch-project/...@main. The committed default stays main; the temporary fork override lives in mutable repo variables and reverts by deleting them once the OSD PR merges. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 367ee0976e0..de355dd2158 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -29,8 +29,14 @@ name: PPL lint rule validation # version, and grammar hash. # # Modes (design §3.4, §4.1.1): -# - pull_request: SQL PR validation against OSD `main`. The ONLY enforcing mode; -# this is what branch protection pins to. Runs the fast schedule:pr subset. +# - pull_request: SQL PR validation against the resolved OSD target. The ONLY +# enforcing mode; this is what branch protection pins to. Runs the fast +# schedule:pr subset. The committed default is `main` on the canonical repo; +# it can be overridden by the OSD_REPO/OSD_REF repo variables — see the +# "Resolve OSD ref" step. TEMPORARY: those repo variables are currently set to +# the unmerged paired OSD branch that ships the headless lint API this job +# needs; deleting them reverts to opensearch-project/...@main once that OSD PR +# merges. # - workflow_dispatch (osd_repo + osd_ref): pre-merge evidence for an unmerged # OSD branch, optionally on a fork (osd_repo). Records the resolved immutable # OSD commit SHA but CANNOT satisfy branch protection — only the pull_request @@ -146,18 +152,33 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - # The required pull_request run always validates against OSD `main` on the - # canonical repo; only a manual workflow_dispatch may target an unmerged OSD - # ref (and optionally a fork), and that run is pre-merge evidence, not a - # branch-protection result (design §4.1.1). + # Resolve which OSD checkout the detectors run against, in precedence order: + # 1. workflow_dispatch input (osd_repo / osd_ref) — explicit manual run + # 2. repo variable (vars.OSD_REPO / vars.OSD_REF) — the override + # point; set/cleared in repo settings with no workflow edit + # 3. canonical default opensearch-project/OpenSearch-Dashboards@main + # + # The committed default is intentionally the canonical repo + `main`, so the + # file always declares that the required check validates against upstream. + # TEMPORARY OVERRIDE: the headless lint API this job imports + # (src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint) is not yet + # on OSD `main`; it lives on the paired branch + # Hanyu-W/OpenSearch-Dashboards@ppl-lint-headless-api. Until that OSD PR + # merges, the OSD_REPO/OSD_REF repo variables are set to that branch so the + # required check validates against the OSD ref that actually ships the API. + # Deleting those two repo variables (no code change) reverts to `main`. - name: Resolve OSD ref id: osd-ref env: REQUESTED_REF: ${{ inputs.osd_ref }} REQUESTED_REPO: ${{ inputs.osd_repo }} + VAR_REF: ${{ vars.OSD_REF }} + VAR_REPO: ${{ vars.OSD_REPO }} run: | - echo "ref=${REQUESTED_REF:-main}" >> "$GITHUB_OUTPUT" - echo "repo=${REQUESTED_REPO:-opensearch-project/OpenSearch-Dashboards}" >> "$GITHUB_OUTPUT" + ref="${REQUESTED_REF:-${VAR_REF:-main}}" + repo="${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "repo=$repo" >> "$GITHUB_OUTPUT" - name: Resolve contract schedule id: schedule From 49d27f5966c4caee145c268405857fbe8b55d7c3 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Wed, 22 Jul 2026 15:16:11 -0700 Subject: [PATCH 30/78] ci(ppl-lint): document sibling-workflow reuse; harden detector job Mirror the sibling SQL Java workflows explicitly (header comment) and make the detector job more robust without changing what it validates: - Note that the workflow reuses Get-CI-Image-Tag, the OpenSearch CI container + ci-image-start-command, and the chown/su non-root Gradle pattern from sql-test-and-build-workflow.yml, and that action SHAs match the siblings. - Record measured CI cost: bootstrap (~2m13s, CPU-bound even with a warm yarn cache) dominates; the lint is ~2s. Hence no per-contract matrix, and why overlapping bootstrap with the backend job is a tracked follow-up. - Download the backend artifact AFTER bootstrap so a flaky artifact download cannot waste a completed bootstrap. - Add a bootstrap retry-with-backoff loop (mirrors the OSD build workflow). - Add timeout-minutes: 30 to both validation jobs. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-rule-validation.yml | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index de355dd2158..543139e3ebb 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -42,6 +42,25 @@ name: PPL lint rule validation # OSD commit SHA but CANNOT satisfy branch protection — only the pull_request # run does. # - schedule (nightly): the full corpus + a coverage assertion. +# +# Workflow shape deliberately mirrors the sibling SQL Java workflows so a +# maintainer sees one pattern, not a bespoke one: +# - sql-test-and-build-workflow.yml : the Get-CI-Image-Tag reusable workflow, +# the OpenSearch CI container + ci-image-start-command, and the +# `chown 1000:1000` + `su` non-root Gradle invocation (backend-validation). +# - integ-tests-with-security.yml : the report-upload-on-always() shape. +# Action SHAs are pinned to the same versions those siblings use, so dependabot +# bumps one set, not two drifting ones. +# +# CI cost (measured 2026-07-22, ~13 min wall clock): backend-validation ~5 min +# (container init ~2 min + backend IT/export ~2m50s); detector-validation ~3 min, +# of which OSD `yarn osd bootstrap` is ~2m13s and the actual lint is ~2s. The +# bootstrap dominates and is CPU-bound (it was ~2m13s even with a warm yarn +# cache), so it is NOT sharded into a per-contract matrix (that would multiply +# the 2m13s, not the 2s). Overlapping bootstrap with the backend job is a tracked +# follow-up, not done here: it would require transferring the bootstrapped OSD +# tree (multi-GB, 30+ workspace symlinks, plus built target/) between runners, +# which OSD's own CI deliberately avoids. See ~/ppl-lint-ci-fixes-impl-plan.md. on: pull_request: @@ -73,6 +92,7 @@ jobs: name: Backend validation (live /_plugins/_ppl + grammar export) needs: Get-CI-Image-Tag runs-on: ubuntu-latest + timeout-minutes: 30 container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -143,6 +163,7 @@ jobs: name: Detector validation (OSD headless lint on candidate bundle) needs: backend-validation runs-on: ubuntu-latest + timeout-minutes: 30 outputs: osd_repo: ${{ steps.osd-ref.outputs.repo }} osd_ref: ${{ steps.osd-ref.outputs.ref }} @@ -195,12 +216,6 @@ jobs: fi echo "value=$value" >> "$GITHUB_OUTPUT" - - name: Download backend artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 - with: - name: ppl-lint-backend - path: artifacts - - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -243,7 +258,24 @@ jobs: - name: Bootstrap OpenSearch-Dashboards working-directory: .ci/OpenSearch-Dashboards - run: yarn osd bootstrap + # Retry-with-backoff mirrors the OSD build workflow's bootstrap step; + # `yarn osd bootstrap` occasionally fails on a transient registry hiccup. + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # Downloaded after bootstrap (not before): the ~2m13s bootstrap does not + # need the backend artifact — only the lint step below does — so a flaky + # artifact download cannot waste a completed bootstrap. + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-backend + path: artifacts - name: Run detector validation against the candidate bundle working-directory: .ci/OpenSearch-Dashboards From b4f05c0efe3cdbf40019348b8d8b690121c5dd0a Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 15:34:18 -0700 Subject: [PATCH 31/78] test(ppl-lint): add invalid-capture-group-name validation contract Pins the rex capture-group-name rule to live /_plugins/_ppl behavior: an underscore in a capture group name is rejected with IllegalArgumentException (validation introduced in 3.4, #4434), while an alphanumeric name is accepted. Joins the enforced set: the rejection is deterministic and rule-unique, and the control query exercises the same command with a valid name. Signed-off-by: Hanyu Wei --- .../invalid-capture-group-name.spec.json | 62 +++++++++++++++++++ .../ppl-lint/contracts/manifest.json | 2 + 2 files changed, 64 insertions(+) create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json new file mode 100644 index 00000000000..c651af803b6 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 3, + "ruleId": "invalid-capture-group-name", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["rexCommand", "stringLiteral"], + "wiring": { + "detector": "invalid-capture-group-name", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { "email": "text" } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "rex-capture-name-underscore": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, + "rex-capture-name-alphanumeric-control": { + "role": "control", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email, username, domain | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index 805e1be9fd8..bab787584db 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -2,6 +2,7 @@ "schemaVersion": 3, "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", "contracts": [ + "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", "division-by-zero.spec.json", "head-without-sort.spec.json", @@ -13,6 +14,7 @@ "replace-wildcard-asymmetry.spec.json" ], "enforced": [ + "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", "multisearch-min-subsearch.spec.json", "union-min-datasets.spec.json", From c435267e550f1d628f29a27fe168e1b131a900e0 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 15:34:50 -0700 Subject: [PATCH 32/78] feat(ci): validate default-error PPL lint rules across engine versions The existing PPL lint contract validates ONE engine: the build from the pull request. But a lint rule ships to every user, and each user's cluster runs whatever version they run. A rule that is correct on main can be a false positive on 3.6 or a false negative on 3.7, and nothing catches it. This adds a multi-version companion that validates every DEFAULT-ERROR rule (enabled + severity error in OSD's rules_catalog.json -- the diagnostics a user cannot opt out of) against several engine versions at once, and reports what to change in the linter when one disagrees. Caught (a): a per-version matrix. Released legs run the official opensearchproject/opensearch: image, which bundles the matching opensearch-sql plugin, so no old branch is built; the pr-build leg is the same Gradle test cluster the single-version check uses. Every leg runs the SAME contract oracle (PplLintRuleValidationIT) under a new -Dppl.lint.observe.only, which records real behavior instead of asserting -- on an older engine a mismatch IS the signal, not a broken run. Engine floor is 3.6.0, the first release containing GET /_plugins/_ppl/_grammar (#5162); a 3.5 leg could not export a bundle for the detectors to lint. Told (b): scripts/ppl-lint/drift.mjs classifies each disagreement into one of eight drift classes and emits one remediation naming the file to edit -- version-scope-rule (fix appliesTo, or disable the rule), update-detector (the detector regressed, went too broad, or its grammar anchor was renamed), or update-contract (the linter is right; the pinned expectation is stale). A renamed parser rule reports the closest current rule names, once per rule/version rather than once per query. Two guards keep the check from passing vacuously: the detector runner now records the catalog's default-error census, and the aggregate step fails if a rule in it has no contract file -- so a new error rule cannot land unvalidated; and a leg with missing artifacts is a hard failure, never a dropped version. Also closes the last default-error coverage gap: flat-object-subfield had no contract. Adds a flat_object test fixture (the repo had none) and pins all four cases, live-verified on 3.8 -- both a dotted subfield and the bare root are rejected. Note its rejection reason is byte-identical to field-validation's, so attribution comes from the detector's ruleId, not the engine's wording. Non-enforcing for now: the required check stays the single-version validation-result. Promoting this needs a green baseline across the matrix first, so an already-drifted rule does not block unrelated PRs on day one. Verified: 35 classifier/aggregator tests green; observe-only leg run end to end against a live 3.8 cluster (all 7 default-error rules agree); red/green proven by injecting five realistic drifts and confirming five distinct remediations. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 429 ++++++++++++++ .../remote/PplLintRuleValidationIT.java | 85 ++- .../sql/legacy/SQLIntegTestCase.java | 8 + .../org/opensearch/sql/legacy/TestUtils.java | 5 + .../opensearch/sql/legacy/TestsConstants.java | 1 + .../src/test/resources/flat_object.json | 6 + .../flat_object_index_mapping.json | 15 + .../contracts/flat-object-subfield.spec.json | 109 ++++ .../ppl-lint/contracts/manifest.json | 17 +- ...ed-window-function-in-eventstats.spec.json | 1 + scripts/ppl-lint/README.md | 110 +++- .../__tests__/aggregate-versions.test.mjs | 360 ++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 399 +++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 554 +++++++++++++++++ scripts/ppl-lint/drift.mjs | 556 ++++++++++++++++++ scripts/ppl-lint/run-frontend-contract.mjs | 10 + 16 files changed, 2654 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/ppl-lint-multiversion-validation.yml create mode 100644 integ-test/src/test/resources/flat_object.json create mode 100644 integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json create mode 100644 scripts/ppl-lint/__tests__/aggregate-versions.test.mjs create mode 100644 scripts/ppl-lint/__tests__/drift.test.mjs create mode 100644 scripts/ppl-lint/aggregate-versions.mjs create mode 100644 scripts/ppl-lint/drift.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml new file mode 100644 index 00000000000..f4ad09b6a82 --- /dev/null +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -0,0 +1,429 @@ +name: PPL lint multi-version validation + +# Multi-version companion to ppl-lint-rule-validation.yml. +# +# The sibling workflow answers "do the OSD PPL lint detectors and THIS engine +# build agree?". It validates one engine: the one built from the PR. That leaves +# the failure mode that actually reaches users unguarded — a lint rule ships to +# everyone, but each user runs it against whatever engine version their cluster +# happens to be. A rule that is correct on main can be a false positive on 3.6 or +# a false negative on 3.7, and nothing notices. +# +# This workflow validates every DEFAULT-ERROR rule (enabled: true + severity: +# error in OSD's rules_catalog.json) against SEVERAL released engine versions +# plus the PR build, and — when a rule disagrees with any of them — says what to +# change in the linter rather than only that a count was wrong. +# +# Why default-error only: an error-severity rule is one the user cannot opt out +# of and which marks their query as broken. A wrong error is the most expensive +# possible lint defect, so that set gets the multi-version treatment first. +# Warning/info rules stay on the single-version check. The set is not hand-copied: +# the detector run records the catalog's default-error census, and the aggregate +# step fails if a rule in that census has no contract file (see manifest.json's +# `defaultError` note). +# +# Shape — a per-version matrix of observation legs, then one aggregation: +# +# observe (matrix: 3.6.0, 3.7.0, pr-build) ──▶ aggregate ──▶ drift report +# +# Each leg produces the SAME four artifacts the single-version workflow already +# defines (ppl-grammar-bundle.json, target.json, backend-report.json, +# detector-report.json), so this workflow adds no new producer format — only the +# per-version fan-out and the cross-version comparison. +# +# Released legs run the official distribution image, which bundles the matching +# opensearch-sql plugin (verified against opensearch-build's release manifests), +# so no old branch has to be built. The `pr-build` leg is the same Gradle test +# cluster the sibling workflow uses. +# +# Engine floor: 3.6.0. GET /_plugins/_ppl/_grammar landed in #5162 (`fe95703b5`), +# which is an ancestor of the 3.6 release branch but NOT of 3.5 — a 3.5 leg could +# not export a candidate grammar bundle, so the detector half would have nothing +# to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. +# +# Non-enforcing on purpose, for now: it reports and uploads, and the required +# check stays the sibling workflow's `validation-result`. Promoting this to +# required needs a green baseline across the whole matrix first (a rule that has +# quietly drifted on 3.6 would otherwise block every unrelated PR on day one). + +on: + # Nightly is the primary schedule: the matrix pulls three engine images, so it + # is too slow to sit on every push. + schedule: + - cron: '30 10 * * *' + # Run on PRs that touch the contract corpus or this machinery, where the whole + # point is to see the multi-version effect of the change. + pull_request: + paths: + - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint/**' + - '.github/workflows/ppl-lint-multiversion-validation.yml' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository to check out. Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string + osd_ref: + description: OSD commit or branch whose detectors are validated. + required: false + type: string + engine_versions: + description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' + required: false + type: string + +permissions: + contents: read + +env: + # Released engine versions to validate, newest last. Each must be >= 3.6.0 (the + # _grammar endpoint floor) and must have a published distribution image. + ENGINE_VERSIONS: '["3.6.0","3.7.0"]' + +jobs: + # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a + # dependabot bump moves one set of action versions rather than two. + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + # Resolve the matrix and the OSD target once, so every leg and the aggregate + # step agree on exactly what is being validated. + plan: + name: Plan matrix + runs-on: ubuntu-latest + outputs: + released: ${{ steps.plan.outputs.released }} + osd_repo: ${{ steps.plan.outputs.osd_repo }} + osd_ref: ${{ steps.plan.outputs.osd_ref }} + steps: + - name: Resolve engine versions and OSD target + id: plan + env: + REQUESTED_VERSIONS: ${{ inputs.engine_versions }} + DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} + REQUESTED_REPO: ${{ inputs.osd_repo }} + REQUESTED_REF: ${{ inputs.osd_ref }} + VAR_REPO: ${{ vars.OSD_REPO }} + VAR_REF: ${{ vars.OSD_REF }} + run: | + set -euo pipefail + released="${REQUESTED_VERSIONS:-$DEFAULT_VERSIONS}" + # Fail loudly on a malformed override rather than silently validating + # an empty matrix (which would look like a pass). + echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list) and v, 'engine_versions must be a non-empty JSON array' + for item in v: + assert isinstance(item,str), 'engine_versions entries must be strings' + " + echo "released=$released" >> "$GITHUB_OUTPUT" + # Same precedence as the sibling workflow: dispatch input, then repo + # variable, then the canonical upstream default. + echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" + echo "osd_ref=${REQUESTED_REF:-${VAR_REF:-main}}" >> "$GITHUB_OUTPUT" + + # One leg per released engine version: run the contract queries against the + # official distribution image (which bundles the matching sql plugin) and + # export that engine's grammar bundle. + observe-released: + name: Observe engine ${{ matrix.version }} + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.released) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + # The lint contract only needs the PPL query and grammar endpoints, so + # run without the security plugin: no TLS or credentials to manage, and + # the observed error bodies are the engine's own rather than a proxy's. + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + echo "reported=$reported" >> "$GITHUB_OUTPUT" + # A leg mislabeled as another version would attribute drift to the wrong + # engine, so require the image to be what the matrix asked for. + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + # The PPL plugin must actually be present, or every query would "pass" + # by failing identically. + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # The same contract oracle the single-version workflow runs, pointed at an + # external cluster instead of a Gradle-managed one. One oracle, many + # engines: a per-version copy would be free to drift from the real check. + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }} + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-logs + path: integ-test/build/reports/** + + # The PR's own engine build, so the newest point in the matrix is the code under + # review rather than the last release. Same oracle as the released legs; the only + # difference is a Gradle-managed cluster instead of a published image, which is + # why it cannot just be another matrix entry. + observe-pr-build: + name: Observe engine pr-build + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # Observe-only here too, so this leg reports what the PR engine does rather + # than duplicating the sibling workflow's assertions. The sibling workflow + # remains the enforcing single-version check. + - name: Run contract observation against the PR build + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-logs + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + + # Lint each engine's exported grammar with the OSD detectors. Separate from the + # observation legs because OSD needs a newer Node/glibc than the engine image + # provides, and because one bootstrap can serve every leg. + # + # `always()` so a single broken leg still yields a report for the others: a + # partial matrix must be visibly partial, not silently absent. The aggregate + # step fails if NO leg produced a report. + detect: + name: Detect on each engine grammar + needs: + - plan + - observe-released + - observe-pr-build + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + osd_sha: ${{ steps.osd-rev.outputs.sha }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-rev + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (${{ needs.plan.outputs.osd_repo }} @ \`${{ needs.plan.outputs.osd_ref }}\`)" >> "$GITHUB_STEP_SUMMARY" + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + - name: Download all leg artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-leg-* + path: legs + + # One detector pass per leg, each against THAT engine's grammar bundle. The + # runner is the same SQL-owned script the single-version workflow uses, so + # the detector half cannot drift between the two checks. + - name: Run detectors against every engine grammar + working-directory: .ci/OpenSearch-Dashboards + run: | + set -euo pipefail + shopt -s nullglob + legs=("$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*) + if [ ${#legs[@]} -eq 0 ]; then + echo "::error::no leg artifacts were downloaded; nothing to validate." + exit 1 + fi + for leg in "${legs[@]}"; do + # Skip the log-only artifacts an observation failure may have uploaded. + [ -f "$leg/ppl-grammar-bundle.json" ] || { echo "skipping $leg (no grammar bundle)"; continue; } + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + echo "=== detectors vs engine $version ===" + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ + PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$leg/detector.log" 2>&1 || true + # A per-leg non-zero exit is EXPECTED when that engine disagrees with + # the pinned expectation — that is the drift this workflow exists to + # report, and the aggregate step below is what classifies it. Only a + # missing report means the runner itself broke. + if [ ! -f "$leg/detector-report.json" ]; then + echo "::error::detector runner produced no report for engine $version" + tail -50 "$leg/detector.log" || true + exit 1 + fi + tail -5 "$leg/detector.log" || true + done + + # Compare every engine version against every other and against the pinned + # contracts, then print the remediation report. + - name: Aggregate drift across engine versions + id: aggregate + run: | + set -euo pipefail + shopt -s nullglob + args=() + for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do + [ -f "$leg/detector-report.json" ] || continue + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + args+=(--leg "$version=$leg") + done + if [ ${#args[@]} -eq 0 ]; then + echo "::error::no complete legs to aggregate." + exit 1 + fi + node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ + --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + --out "$GITHUB_WORKSPACE/drift-report.json" \ + --summary "$GITHUB_STEP_SUMMARY" \ + "${args[@]}" + + - name: Upload drift report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-multiversion-drift + path: | + drift-report.json + legs/**/detector-report.json + legs/**/detector.log + legs/**/target.json diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 4185511ab04..ae74fdd2515 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -73,6 +73,24 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { /** Which contracts to run this session; PR is the fast blocking subset. */ private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + /** + * Observe-only mode, used by the multi-version workflow ({@code + * .github/workflows/ppl-lint-multiversion-validation.yml}). + * + *

The default mode asserts each case against the expectation pinned for the cluster's version, + * which is right when the cluster IS the build under test. The multi-version matrix instead + * points this suite at OLDER released engines, where a disagreement is the very signal being + * collected — a 3.6 engine that accepts what the contract pins as rejected is a finding for the + * drift classifier, not a broken test run. + * + *

So in observe-only mode the suite still runs every query and records the true observed + * behavior in the report, but does not fail on an expectation mismatch, and does not require an + * expectation to exist for this version at all. Failures that mean the RUN itself is broken (no + * grammar bundle, an unreachable cluster, a malformed contract) still fail, because those would + * otherwise produce an empty report that reads as agreement. + */ + private final boolean observeOnly = Boolean.getBoolean("ppl.lint.observe.only"); + private int[] clusterVersion; private String engineVersionRaw; @@ -82,7 +100,20 @@ public void init() throws Exception { enableCalcite(); // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { - loadIndex(Index.valueOf(indexEnum)); + try { + loadIndex(Index.valueOf(indexEnum)); + } catch (Exception e) { + if (!observeOnly) { + throw e; + } + // In the multi-version matrix an older engine may not support a field type + // a fixture uses (a mapping that only exists in a later release). Losing + // that one index must not abort the whole leg — the contracts that need it + // will surface as their own observations, while every other rule is still + // validated against this engine. + System.err.println( + "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); + } } clusterVersion = fetchClusterVersion(); } @@ -127,7 +158,14 @@ private void runContract( try { JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); if (selected == null) { - return; // no/ambiguous version expectation — failure already recorded. + if (!observeOnly) { + return; // no/ambiguous version expectation — failure already recorded. + } + // Observe-only: an engine the corpus does not pin is exactly what the + // multi-version matrix wants to learn about, so record the raw behavior of + // every query and let the aggregator decide whether the gap matters. + observeAllQueries(ruleId, index, queries, report); + return; } JSONObject expectedQueries = selected.getJSONObject("queries"); for (String queryName : expectedQueries.keySet()) { @@ -153,9 +191,16 @@ private void runContract( entry.put("outcome", "pass"); log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); } catch (AssertionError | RuntimeException e) { - entry.put("outcome", "fail").put("error", String.valueOf(e.getMessage())); - failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); - log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + entry.put("outcome", observeOnly ? "observed-mismatch" : "fail"); + entry.put("error", String.valueOf(e.getMessage())); + if (observeOnly) { + // Not a failure here: the observation is the deliverable, and the + // drift classifier turns it into a remediation. + log(ruleId, queryName, "OBSERVED MISMATCH (" + kind + "): " + e.getMessage()); + } else { + failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); + log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + } } report.put(entry); } @@ -164,6 +209,36 @@ private void runContract( } } + /** + * Observe-only helper: run every query a contract declares and record what the engine actually + * did, without comparing against any expectation. Used when this engine version has no matching + * {@code expectations[]} entry, so the multi-version report still shows real behavior instead of + * a blank row that would read as agreement. + */ + private void observeAllQueries( + String ruleId, String index, JSONObject queries, JSONArray report) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject entry = reportEntry(ruleId, queryName, role, query, "observe-only"); + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "observed"); + log(ruleId, queryName, "OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + // A transport-level problem is a broken run, not an engine verdict; mark it + // so the aggregator does not read the absence of a rejection as acceptance. + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + } + /** * Select the single {@code expectations[]} entry that applies to the candidate backend version * and engine. Exactly one must match: zero means the rule test does not cover this version diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index fc15c908c63..38e37c41d31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -869,6 +869,14 @@ public enum Index { "flattened_value", null, "src/test/resources/flattened_value.json"), + // An index with a `flat_object` field, which PPL cannot reference at all — + // neither the root nor a dotted subfield. Backs the flat-object-subfield lint + // contract; see ppl-lint/contracts/flat-object-subfield.spec.json. + FLAT_OBJECT( + TestsConstants.TEST_INDEX_FLAT_OBJECT, + "flat_object", + getFlatObjectIndexMapping(), + "src/test/resources/flat_object.json"), DUPLICATION_NULLABLE( TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE, "duplication_nullable", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index c478165bf07..198527d1efc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -523,6 +523,11 @@ public static String getAccountExtendedIndexMapping() { return getMappingFile(mappingFile); } + public static String getFlatObjectIndexMapping() { + String mappingFile = "flat_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 5d7eeb328af..957ff0108d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -76,6 +76,7 @@ public class TestsConstants { public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias"; public static final String TEST_INDEX_FLATTENED_VALUE = TEST_INDEX + "_flattened_value"; + public static final String TEST_INDEX_FLAT_OBJECT = TEST_INDEX + "_flat_object"; public static final String TEST_INDEX_GEOIP = TEST_INDEX + "_geoip"; public static final String DATASOURCES = ".ql-datasources"; public static final String TEST_INDEX_STATE_COUNTRY = TEST_INDEX + "_state_country"; diff --git a/integ-test/src/test/resources/flat_object.json b/integ-test/src/test/resources/flat_object.json new file mode 100644 index 00000000000..03ed0d15d67 --- /dev/null +++ b/integ-test/src/test/resources/flat_object.json @@ -0,0 +1,6 @@ +{"index":{"_id":"1"}} +{"name":"alpha","status":200,"attributes":{"service":"checkout","region":"us-east-1"}} +{"index":{"_id":"2"}} +{"name":"beta","status":500,"attributes":{"service":"search","region":"us-west-2"}} +{"index":{"_id":"3"}} +{"name":"gamma","status":200,"attributes":{"service":"checkout","region":"eu-west-1"}} diff --git a/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json new file mode 100644 index 00000000000..5721d2c3773 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "name": { + "type": "keyword" + }, + "status": { + "type": "integer" + }, + "attributes": { + "type": "flat_object" + } + } + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json new file mode 100644 index 00000000000..25ac4cf66cf --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": 3, + "ruleId": "flat-object-subfield", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["qualifiedName", "wcQualifiedName"], + "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", + "wiring": { + "detector": "flat-object-subfield", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["FLAT_OBJECT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "name": "keyword", + "status": "integer", + "attributes": "flat_object" + } + }, + "index": "opensearch-sql_test_index_flat_object", + "queries": { + "flat-object-dotted-subfield": { + "role": "trigger", + "query": "source={{index}} | fields attributes.service" + }, + "flat-object-bare-root": { + "role": "trigger", + "query": "source={{index}} | fields attributes" + }, + "flat-object-in-where": { + "role": "trigger", + "query": "source={{index}} | where attributes.service = 'checkout'" + }, + "non-flat-field-control": { + "role": "control", + "query": "source={{index}} | fields name, status | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes] not found." + } + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { "datarowsNonEmpty": true } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index bab787584db..a8b031315d6 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -8,6 +8,7 @@ "head-without-sort.spec.json", "disabled-join-type.spec.json", "field-validation.spec.json", + "flat-object-subfield.spec.json", "dedup-consecutive-unsupported.spec.json", "multisearch-min-subsearch.spec.json", "union-min-datasets.spec.json", @@ -20,9 +21,16 @@ "union-min-datasets.spec.json", "replace-wildcard-asymmetry.spec.json" ], - "pendingReview": [ - "field-validation.spec.json" + "defaultError": [ + "invalid-capture-group-name.spec.json", + "unsupported-window-function-in-eventstats.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json", + "field-validation.spec.json", + "flat-object-subfield.spec.json" ], + "pendingReview": [], "nonEnforcing": [ "division-by-zero.spec.json", "head-without-sort.spec.json", @@ -30,8 +38,9 @@ "dedup-consecutive-unsupported.spec.json" ], "notes": { - "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required validation-result check.", - "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. field-validation self-suppresses without field context and is a semantic rule rather than a clean HTTP-400 grammar rejection.", + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required single-version validation-result check.", + "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog — the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index e1254b14583..6f6fbd313b0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -1,6 +1,7 @@ { "schemaVersion": 3, "ruleId": "unsupported-window-function-in-eventstats", + "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", "grammarSurface": "compiled-simplified", "schedule": "pr", "wiring": { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index e9578517608..a78b1f5cdcc 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -170,14 +170,120 @@ rule cannot be validated end to end. `manifest.json` partitions the corpus: - `enforced` — reviewed error rules with a deterministic backend rejection and a - valid negative control. These block `validation-result`. Phase one: + valid negative control. These block `validation-result` on the single-version + check: `invalid-capture-group-name`, `unsupported-window-function-in-eventstats`, `multisearch-min-subsearch`, `union-min-datasets`, `replace-wildcard-asymmetry`. +- `defaultError` — every rule that ships **enabled at error severity** in OSD's + `rules_catalog.json`. This is the set the **multi-version** check enforces (see + below). It is a superset of `enforced`, adding `field-validation` and + `flat-object-subfield`. - `pendingReview` — error rules awaiting Peng/Chen usefulness review before - joining `enforced` (currently `field-validation`). + joining `enforced`. Empty: `field-validation` and `flat-object-subfield` are now + pinned across versions by the multi-version check, but stay out of the + single-version `enforced` set because their backend oracle is a semantic + `Field [...] not found.` rejection they share with each other rather than a + rule-unique grammar rejection. - `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the nightly schedule for coverage and never block a PR. +## Multi-version validation + +The check above validates **one** engine: the build from the PR. But a lint rule +ships to every user, and each user's cluster is on whatever version they run. A +rule that is correct on `main` can be a false positive on 3.6 or a false negative +on 3.7, and the single-version check cannot see it. + +[`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) +validates every `defaultError` rule against several engine versions at once, and +reports **what to change in the linter** when one disagrees. + +``` +observe (matrix: 3.6.0, 3.7.0 released images + pr-build) + └── each leg exports the same 4 artifacts as the single-version check +detect (one OSD bootstrap, one detector pass per leg's grammar) + └── aggregate-versions.mjs → drift-report.json + remediation report +``` + +Released legs run the official `opensearchproject/opensearch:` image, +which bundles the matching `opensearch-sql` plugin, so no old branch is built. The +`pr-build` leg is the same Gradle test cluster the single-version check uses. Both +run the **same** contract oracle (`PplLintRuleValidationIT`) with +`-Dppl.lint.observe.only=true`, which records real behavior instead of asserting +against expectations — on an older engine a mismatch is the signal being +collected, not a broken run. + +**Engine floor: 3.6.0.** `GET /_plugins/_ppl/_grammar` landed in #5162, which is +an ancestor of 3.6 but not 3.5, so a 3.5 leg could not export a grammar bundle for +the detectors to lint against. + +This workflow is **non-enforcing for now**: it reports and uploads, while the +required check stays the single-version `validation-result`. Promoting it needs a +green baseline across the whole matrix first, so a rule that has already drifted +on 3.6 does not block every unrelated PR on day one. + +### What a drift report tells you + +Every finding names a drift class, the evidence, and one remediation action: + +| Action | When | What you change | +| --- | --- | --- | +| `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | +| `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | +| `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | + +Drift classes: `grammar-rule-missing` (a parser rule the detector walks was +renamed or removed — the finding names the closest current rule names), +`engine-relaxed` / `engine-tightened` (the engine's verdict flipped), +`engine-message-changed` (same verdict, reworded error), `detector-silent` / +`detector-noisy` (false negative / false positive), `version-scope-too-narrow` +(the engine rejects but the rule is scoped away from that version, so users see no +diagnostic), and `severity-mismatch`. + +Two guards keep the check from passing vacuously: + +- A rule that is default-error in OSD's catalog but has no contract file fails the + run. The detector runner records the catalog's default-error census in + `detector-report.json`, and the aggregate step compares it against + `manifest.defaultError` — so a new error rule cannot land unvalidated. +- A leg whose artifacts are missing is a hard failure, never a silently dropped + version. + +A rule that is out of scope on an engine (`appliesTo` excludes it) and that the +engine also accepts is reported as `n/a (out of scope)`, not as drift — that is +the version window working. But if the engine *rejects* the trigger there, it is +`version-scope-too-narrow`. + +### Running the multi-version check locally + +Each leg needs a reachable cluster. Point the observe step at any running engine: + +```bash +# Observe one engine (repeat per version into its own leg dir). +mkdir -p legs/3.7.0 +./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$PWD/legs/3.7.0/backend-report.json \ + -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ + -Dppl.lint.target=$PWD/legs/3.7.0/target.json + +# Lint each leg's grammar from an OSD checkout (writes detector-report.json), +# then compare every version at once: +node scripts/ppl-lint/aggregate-versions.mjs \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 \ + --out drift-report.json +``` + +The classifier is pure and has no cluster or OSD dependency, so its tests run +anywhere: + +```bash +node --test "scripts/ppl-lint/__tests__/*.test.mjs" +``` + ## Interpreting a failure | Failure | Meaning | diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs new file mode 100644 index 00000000000..8f1bd7493a5 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -0,0 +1,360 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Integration tests for the multi-version aggregator. + * + * node --test scripts/ppl-lint/__tests__/aggregate-versions.test.mjs + * + * These drive the real script as a child process over synthetic leg directories + * (the four artifact files each engine leg produces), so they cover the parts the + * pure classifier tests cannot: argument handling, artifact loading, the + * in-scope/out-of-scope split, coverage holes, once-per-rule grammar drift, and + * the process exit code that makes the CI check red or green. + */ + +import assert from 'node:assert/strict'; +import { after, test } from 'node:test'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-versions.mjs'); + +/** Contract used by every case: a >=3.7 calcite-only rule with one trigger + one control. */ +const SPEC = { + schemaVersion: 3, + ruleId: 'union-min-datasets', + grammarSurface: 'runtime-bundle', + schedule: 'pr', + requiredParserRules: ['unionCommand'], + wiring: { + detector: 'union-min-datasets', + enabled: true, + severity: 'error', + runtimeOnly: true, + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + }, + index: 'test-index', + queries: { + trigger: { role: 'trigger', query: 'union [ source={{index}} ]' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backend: { + kind: 'rejection', + httpStatus: 400, + body: { + status: 400, + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], +}; + +const REJECTION = { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', +}; + +const tmpDirs = []; + +function makeTmp(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tmpDirs.push(dir); + return dir; +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** Write a contract dir holding SPEC (optionally patched) and a manifest. */ +function writeContracts(patch = {}) { + const dir = makeTmp('ppl-lint-contracts-'); + const spec = { ...SPEC, ...patch }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +/** + * Write one engine leg. `cases` maps query name to + * { detector: , severities, rejected, type, reason }. + */ +function writeLeg({ + version, + cases, + parserRuleNames = ['unionCommand', 'unionDataset'], + defaultErrorRules, +}) { + const dir = makeTmp(`ppl-lint-leg-${version}-`); + fs.writeFileSync( + path.join(dir, 'target.json'), + JSON.stringify({ engineVersion: version, grammarHash: `sha256:${version}` }) + ); + fs.writeFileSync( + path.join(dir, 'ppl-grammar-bundle.json'), + JSON.stringify({ parserRuleNames }) + ); + + const results = []; + const backend = []; + for (const [queryName, c] of Object.entries(cases)) { + const role = queryName === 'control' ? 'control' : 'trigger'; + results.push({ + ruleId: SPEC.ruleId, + queryName, + role, + expected: role === 'trigger' ? 1 : 0, + actual: c.detector, + severities: c.severities || (c.detector > 0 ? ['error'] : []), + }); + backend.push({ + ruleId: SPEC.ruleId, + queryName, + role, + rejected: !!c.rejected, + observed: { + httpStatus: c.rejected ? 400 : 200, + rejected: !!c.rejected, + ...(c.rejected ? { type: c.type || REJECTION.type, reason: c.reason || REJECTION.reason } : {}), + }, + }); + } + fs.writeFileSync( + path.join(dir, 'detector-report.json'), + JSON.stringify({ results, ...(defaultErrorRules ? { defaultErrorRules } : {}) }) + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + return dir; +} + +/** Run the aggregator; returns { status, stdout, report }. */ +function run({ contracts, legs, extraArgs = [] }) { + const outDir = makeTmp('ppl-lint-out-'); + const out = path.join(outDir, 'drift-report.json'); + const args = [SCRIPT, '--contracts', contracts, '--out', out]; + for (const [version, dir] of Object.entries(legs)) { + args.push('--leg', `${version}=${dir}`); + } + args.push(...extraArgs); + const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); + const report = fs.existsSync(out) ? JSON.parse(fs.readFileSync(out, 'utf8')) : undefined; + return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '', report }; +} + +/** The all-agree case, reused as the base for each drift scenario. */ +function healthyLegs() { + return { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + '3.8.0': writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; +} + +test('all versions agreeing exits 0 and reports no drift', () => { + const { status, report, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); + assert.equal(status, 0); + assert.equal(report.result.passed, true); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /agrees with all 2 engine version\(s\)/); + // Every rule/version pair is accounted for in the matrix. + assert.equal(report.matrix.length, 2); + assert.ok(report.matrix.every((m) => m.status === 'agree')); +}); + +test('a version where only one engine relaxed is red, and names just that version', () => { + const legs = healthyLegs(); + // 3.8 now accepts what 3.7 still rejects, while the detector keeps flagging. + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.enforcedDriftCount, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.version, '3.8.0'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + // The healthy version is still reported as agreeing. + assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); +}); + +test('a rule out of scope on an older engine that accepts is not drift', () => { + const legs = healthyLegs(); + // 3.6 predates the rule's minVersion and accepts the query: intended silence. + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.matrix.find((m) => m.version === '3.6.0').status, 'out-of-scope'); + assert.equal(report.coverageHoles.length, 0); +}); + +test('an out-of-scope engine that rejects is flagged as scoped too narrowly', () => { + const legs = healthyLegs(); + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.6.0'); + assert.equal(drift.driftClass, 'version-scope-too-narrow'); + assert.equal(drift.remediation.action, 'version-scope-rule'); +}); + +test('an in-scope version with no expectation is a coverage hole, not silent success', () => { + // The rule applies from 3.7 up, but the contract only pins <3.8 — so a 3.8 + // engine runs a shipped default-error rule with nothing pinning it. + const contracts = writeContracts({ + expectations: [{ ...SPEC.expectations[0], version: '>=3.7.0 <3.8.0' }], + }); + const { status, report } = run({ contracts, legs: healthyLegs() }); + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 1); + const hole = report.coverageHoles[0]; + assert.equal(hole.version, '3.8.0'); + assert.equal(hole.enforced, true); + assert.match(report.matrix.find((m) => m.version === '3.8.0').status, /uncovered/); +}); + +test('a renamed parser rule is reported once per version, not once per query', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + parserRuleNames: ['unionStatement', 'unionDataset'], + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const grammarDrifts = report.drifts.filter((d) => d.driftClass === 'grammar-rule-missing'); + assert.equal(grammarDrifts.length, 1, 'one grammar finding per rule/version'); + assert.equal(grammarDrifts[0].remediation.action, 'update-detector'); + assert.match(grammarDrifts[0].remediation.detail, /unionStatement/); +}); + +test('a silent detector on an unchanged engine is update-detector, never a re-pin', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.8.0'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('the leg label is corrected to the engine self-reported version', () => { + // Ask for 3.7.0 but hand over an engine that says 3.8.0: results must be + // attributed to what actually ran. + const legs = { '3.7.0': writeLeg({ version: '3.8.0', cases: { trigger: { detector: 1, rejected: true } } }) }; + const { report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(report.legs[0].label, '3.7.0'); + assert.equal(report.legs[0].engineVersion, '3.8.0'); + assert.match(stdout, /reported engineVersion "3\.8\.0"/); +}); + +test('a missing leg artifact fails loudly instead of dropping the version', () => { + const emptyLeg = makeTmp('ppl-lint-empty-leg-'); + const { status, stderr } = run({ contracts: writeContracts(), legs: { '3.8.0': emptyLeg } }); + assert.equal(status, 2, 'a broken matrix must not be able to pass'); + assert.match(stderr, /expected file not found/); +}); + +test('a default-error rule with no contract file fails the check', () => { + // OSD started shipping `brand-new-error-rule` enabled at error severity, but no + // contract pins it — so no engine version validates it. That must be red, not + // silently absent from the matrix. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets', 'brand-new-error-rule'], + }), + }; + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.missingContractCount, 1); + assert.equal(report.missingContracts[0].ruleId, 'brand-new-error-rule'); + assert.match(stdout, /Unvalidated default-error rules/); + assert.match(stdout, /brand-new-error-rule.*no contract file/s); +}); + +test('a census matching the manifest keeps the check green', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets'], + }), + }; + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.result.missingContractCount, 0); +}); + +test('a legacy detector report without a census warns instead of failing', () => { + // Older detector builds do not emit defaultErrorRules; the aggregator must say + // so out loud rather than quietly reporting full coverage. + const { status, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); + assert.equal(status, 0); + assert.match(stdout, /no detector leg reported a defaultErrorRules census/); +}); + +test('a bad --leg argument is rejected', () => { + const result = spawnSync( + process.execPath, + [SCRIPT, '--contracts', writeContracts(), '--leg', 'no-equals-sign'], + { encoding: 'utf8' } + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /--leg expects =

/); +}); + +test('at least one leg is required', () => { + const result = spawnSync(process.execPath, [SCRIPT, '--contracts', writeContracts()], { + encoding: 'utf8', + }); + assert.equal(result.status, 2); + assert.match(result.stderr, /at least one --leg/); +}); diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs new file mode 100644 index 00000000000..63640958473 --- /dev/null +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -0,0 +1,399 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the multi-version drift classifier. + * + * Run with plain Node (no Gradle, no cluster, no OSD checkout): + * + * node --test scripts/ppl-lint/__tests__/drift.test.mjs + * + * The classifier is the part of the multi-version contract that decides what an + * engineer is told to do, so every drift class and every remediation branch is + * pinned here. Observations are hand-written rather than gathered from a + * cluster; the live-engine plumbing is exercised by the CI workflow itself. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + DRIFT_CLASSES, + REMEDIATIONS, + classifyDrift, + formatDriftReport, + parseVersion, + suggestParserRules, + versionInAppliesTo, +} from '../drift.mjs'; + +/** A trigger case that agrees on all three sides, used as the mutation base. */ +function agreeingTrigger(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + expected: { detectorCount: 1, severity: 'error', backendKind: 'rejection' }, + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, runtimeOnly: true }, + ...overrides, + }; +} + +/** A control case that agrees on all three sides. */ +function agreeingControl(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-two-datasets-control', + role: 'control', + query: 'union [ source=t ] [ source=t ]', + expected: { detectorCount: 0, backendKind: 'result-shape' }, + observed: { detectorCount: 0, severities: [], backendRejected: false }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' } }, + ...overrides, + }; +} + +// --- the quiet path ----------------------------------------------------------- + +test('agreement produces no drift', () => { + assert.equal(classifyDrift(agreeingTrigger()), null); + assert.equal(classifyDrift(agreeingControl()), null); +}); + +test('a rule out of version scope on an engine that also accepts is silent', () => { + // union-min-datasets does not apply below 3.7, and a 3.6 engine that accepts + // the query is not drift — it is the reason the version window exists. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift, null); +}); + +// --- grammar moved ------------------------------------------------------------ + +test('a missing parser rule is reported as update-detector, not as a silent detector', () => { + const drift = classifyDrift( + agreeingTrigger({ + requiredParserRules: ['unionCommand', 'unionDataset'], + parserRuleNames: ['unionStatement', 'unionDataset', 'pplCommands'], + observed: { detectorCount: 0, severities: [], backendRejected: true }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.GRAMMAR_RULE_MISSING); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.target, /union_min_datasets\.ts$/); + // The likely rename is named so the engineer does not diff 259 rule names. + assert.match(drift.remediation.detail, /unionStatement/); + assert.match(drift.evidence, /no parser rule/); +}); + +test('a contract can pin a detector path that breaks the naming convention', () => { + // unsupported-window-function-in-eventstats lives in + // unsupported_window_function.ts, so the derived name would not exist. + const drift = classifyDrift( + agreeingTrigger({ + ruleId: 'unsupported-window-function-in-eventstats', + detectorPath: 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts', + wiring: { appliesTo: {} }, + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'CalciteUnsupportedException', + backendReason: 'Unexpected window function: rank', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal( + drift.remediation.target, + 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts' + ); +}); + +test('grammar-rule check is skipped when the contract declares no required rules', () => { + const drift = classifyDrift( + agreeingTrigger({ parserRuleNames: ['somethingElse'], requiredParserRules: undefined }) + ); + assert.equal(drift, null); +}); + +// --- engine behavior flips ---------------------------------------------------- + +test('engine relaxation with a still-firing detector demands version scoping', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + // Both escape hatches are spelled out: bound the version, or disable outright. + assert.match(drift.remediation.detail, /maxVersion/); + assert.match(drift.remediation.detail, /"enabled": false/); + assert.match(drift.evidence, /now ACCEPTS/); +}); + +test('engine relaxation with an already-silent detector only needs a re-pin', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.remediation.detail, /no linter change/); +}); + +test('engine tightening on a control with a silent detector is a false negative', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command now requires matching schemas.', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false NEGATIVE/); + assert.match(drift.evidence, /matching schemas/); +}); + +test('engine tightening the detector already catches only needs a re-pin', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'nope', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +// --- wording drift ------------------------------------------------------------ + +test('a reworded rejection is update-contract and points at quoted copy', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.evidence, /error\.reason/); + assert.match(drift.remediation.detail, /quotes the old engine wording/); +}); + +test('a changed exception type is reported even when the reason is unchanged', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.match(drift.evidence, /error\.type/); +}); + +// --- detector-only disagreement ---------------------------------------------- + +test('a silent detector on an unchanged engine names the three silent-failure causes', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /runtimeOnly/); + assert.match(drift.remediation.detail, /typeMap/); + // Guard the anti-vacuous instruction: never silence the contract instead. + assert.match(drift.remediation.detail, /Do NOT re-pin/); +}); + +test('a noisy detector the engine disagrees with is a false positive', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { detectorCount: 2, severities: ['error', 'error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false positive/); +}); + +test('a noisy detector the engine agrees with points at the expectation', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'bad query', + }, + // Engine tightening is the more specific story when the pinned kind is not + // a rejection, so pin the kind as rejection to isolate the noisy branch. + expected: { detectorCount: 0, backendKind: 'rejection' }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +// --- severity ---------------------------------------------------------------- + +test('a downgraded severity is caught even when the count is right', () => { + const drift = classifyDrift( + agreeingTrigger({ observed: { ...agreeingTrigger().observed, severities: ['warning'] } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.SEVERITY_MISMATCH); + assert.match(drift.remediation.detail, /Restore "union-min-datasets"\.severity/); +}); + +// --- version scoping -------------------------------------------------------- + +test('an out-of-scope rule on an engine that rejects is scoped too narrowly', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.remediation.detail, /minVersion/); +}); + +// --- helpers ----------------------------------------------------------------- + +test('version parsing tolerates snapshot and short forms', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), [3, 8, 0]); + assert.deepEqual(parseVersion('3.7'), [3, 7, 0]); + assert.equal(parseVersion(''), undefined); + assert.equal(parseVersion(undefined), undefined); +}); + +test('appliesTo bounds are inclusive and open-ended when absent', () => { + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.7.0'), true); + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.6.9'), false); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.8.0'), true); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.9.0'), false); + assert.equal(versionInAppliesTo({}, '3.9.0'), true); + // An unparseable engine version must never silently drop coverage. + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, 'weird-build'), true); +}); + +test('rename suggestions prefer containment then near spellings', () => { + assert.deepEqual(suggestParserRules('unionCommand', ['unionCommandNew', 'zzz'], 3), [ + 'unionCommandNew', + ]); + assert.deepEqual(suggestParserRules('rexCommand', ['regexCommand'], 3), ['regexCommand']); + // Nothing remotely similar: say nothing rather than guess. + assert.deepEqual(suggestParserRules('rexCommand', ['whereClause', 'sortCommand'], 3), []); +}); + +// --- report ------------------------------------------------------------------ + +test('the report groups by action, most urgent first', () => { + const drifts = [ + classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'X', + backendReason: 'new wording', + }, + expectedBackend: { body: { error: { type: 'X', reason: 'old wording' } } }, + }) + ), + classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ), + ]; + const report = formatDriftReport(drifts); + assert.match(report, /2 finding\(s\) across 2 engine version\(s\)/); + assert.ok( + report.indexOf(REMEDIATIONS.VERSION_SCOPE_RULE) < report.indexOf(REMEDIATIONS.UPDATE_CONTRACT), + 'version scoping (a live false positive) must be listed before a stale-string re-pin' + ); + assert.match(report, /QUERY: union \[ source=t \]/); +}); + +test('an empty drift list reports agreement', () => { + assert.match(formatDriftReport([]), /No engine\/linter drift detected/); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs new file mode 100644 index 00000000000..7b3f6d174fd --- /dev/null +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -0,0 +1,554 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Multi-version aggregation for the PPL lint contract. + * + * The single-version workflow answers "do the OSD detectors and this engine + * agree?". This script answers the question that actually protects users: "does + * every default-ERROR rule still agree with EVERY supported engine version, and + * if not, what should the linter engineer change?" + * + * Inputs: one `--leg =` per engine version, where holds that + * leg's `target.json`, `backend-report.json`, `detector-report.json` and + * `ppl-grammar-bundle.json` (the same four files the single-version jobs already + * produce — this script adds no new producer). + * + * Output: a `drift-report.json` plus a markdown remediation report. Exits + * non-zero when any ENFORCED rule drifted on any version, so the check is red + * exactly when a shipped default-error rule disagrees with a supported engine. + * + * Usage: + * node scripts/ppl-lint/aggregate-versions.mjs \ + * --contracts integ-test/src/test/resources/ppl-lint/contracts \ + * --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg 3.8.0=legs/3.8.0 \ + * --out drift-report.json [--summary $GITHUB_STEP_SUMMARY] [--all-rules] + * + * By default only the manifest's `defaultError` set is enforced; `--all-rules` + * widens the report (still only enforcing `defaultError`) for nightly coverage. + */ + +import fs from 'fs'; +import path from 'path'; + +import { + classifyDrift, + classifyGrammarDrift, + formatDriftReport, + versionInAppliesTo, +} from './drift.mjs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-multiversion] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-multiversion] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { legs: [], contracts: '', out: 'drift-report.json', summary: '', allRules: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--leg') { + const raw = next(); + const eq = raw.indexOf('='); + if (eq <= 0) fatal(`--leg expects =, got "${raw}"`); + args.legs.push({ version: raw.slice(0, eq), dir: raw.slice(eq + 1) }); + } else if (arg === '--contracts') { + args.contracts = next(); + } else if (arg === '--out') { + args.out = next(); + } else if (arg === '--summary') { + args.summary = next(); + } else if (arg === '--all-rules') { + args.allRules = true; + } else { + fatal(`unknown argument "${arg}"`); + } + } + if (args.legs.length === 0) fatal('at least one --leg = is required'); + if (!args.contracts) fatal('--contracts is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +/** Load the contract corpus, keyed by ruleId, plus the manifest's enforced sets. */ +function loadContracts(dir) { + const manifest = readJson(path.join(dir, 'manifest.json')); + const specs = new Map(); + for (const name of manifest.contracts || []) { + const spec = readJson(path.join(dir, name)); + specs.set(spec.ruleId, { spec, file: name }); + } + // `defaultError` is the multi-version enforced set: every rule that ships + // enabled at error severity. Fall back to `enforced` for older manifests so + // this script still runs against an un-migrated corpus. + const enforcedFiles = new Set(manifest.defaultError || manifest.enforced || []); + const enforcedRules = new Set(); + for (const [ruleId, { file }] of specs) { + if (enforcedFiles.has(file)) enforcedRules.add(ruleId); + } + return { specs, enforcedRules, manifest }; +} + +/** + * Read one engine version's four artifacts. A leg whose backend never came up + * is fatal rather than skipped: silently dropping a version would turn a broken + * matrix into a green check, which is the failure mode this whole contract + * exists to prevent. + */ +function loadLeg({ version, dir }) { + const target = readJson(path.join(dir, 'target.json')); + const detector = readJson(path.join(dir, 'detector-report.json')); + const backendRaw = readJson(path.join(dir, 'backend-report.json')); + const bundle = readJson(path.join(dir, 'ppl-grammar-bundle.json'), { optional: true }); + + const backend = new Map(); + for (const entry of Array.isArray(backendRaw) ? backendRaw : []) { + backend.set(`${entry.ruleId}::${entry.queryName}`, entry); + } + + // The engine's self-reported version wins over the matrix label, so a matrix + // typo (asking for 3.7.0 and getting 3.8.0) cannot silently mislabel results. + const reported = target.engineVersion || ''; + if (reported && !reported.startsWith(version.split('-')[0])) { + log( + `WARN: leg "${version}" reported engineVersion "${reported}"; using the reported value for ` + + `version comparisons.` + ); + } + + return { + version: reported || version, + label: version, + dir, + grammarHash: target.grammarHash || '', + parserRuleNames: bundle && Array.isArray(bundle.parserRuleNames) ? bundle.parserRuleNames : undefined, + detector, + backend, + }; +} + +/** + * Compare the OSD catalog's default-error census (recorded by each detector leg) + * against the contracts this run knows about. Returns one entry per rule that + * ships enabled at error severity with no contract, or whose contract the + * manifest does not list under `defaultError`. + * + * Legs can disagree if they ran against different OSD checkouts, so the union is + * used: a rule that is default-error on ANY validated OSD ref must be accounted + * for. + */ +function auditDefaultErrorCensus(legs, specs, enforcedRules) { + const census = new Set(); + let sawCensus = false; + for (const leg of legs) { + const rules = leg.detector && leg.detector.defaultErrorRules; + if (!Array.isArray(rules)) continue; + sawCensus = true; + for (const ruleId of rules) census.add(ruleId); + } + if (!sawCensus) { + log( + "WARN: no detector leg reported a defaultErrorRules census, so the manifest's defaultError set " + + 'could not be cross-checked against the OSD catalog. Re-run with a detector build that emits it.' + ); + return []; + } + + const missing = []; + for (const ruleId of [...census].sort()) { + if (!specs.has(ruleId)) { + missing.push({ ruleId, reason: 'no contract file' }); + } else if (!enforcedRules.has(ruleId)) { + missing.push({ + ruleId, + reason: 'contract exists but is not listed under manifest.defaultError', + }); + } + } + return missing; +} + +/** + * Check an out-of-scope rule for the one drift that still matters there: the + * engine rejects a trigger query, but the rule's `appliesTo` excludes this + * version, so users on it see no diagnostic for a real error. Everything else + * about an out-of-scope rule is intentional silence. + * + * The trigger queries come from the spec's own `queries` map (there is no + * expectation to read on this path), and the backend observation from this leg's + * report; `classifyDrift` decides, so the "too narrow" wording stays in one place. + */ +function classifyOutOfScope({ spec, ruleId, leg, classify }) { + const found = []; + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + if ((queryDef.role || 'trigger') !== 'trigger') continue; + const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); + if (!backendEntry) continue; // this leg never ran the query + const observedBackend = backendEntry.observed || {}; + const detectorResult = (leg.detector.results || []).find( + (r) => r.ruleId === ruleId && r.queryName === queryName + ); + const drift = classify({ + ruleId, + version: leg.version, + queryName, + role: 'trigger', + query: queryDef.query.split('{{index}}').join(spec.index), + // Out of scope means the rule is expected to stay silent here. + expected: { detectorCount: 0 }, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: !!backendEntry.rejected, + backendType: observedBackend.type, + backendReason: observedBackend.reason, + }, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + // Deliberately no parser-rule check here: a grammar that lacks the rule is + // expected on an engine the command predates. + }); + if (drift) found.push(drift); + } + return found; +} + +/** + * Pick the contract expectation that applies to a version, reusing the same + * "exactly one must match" rule as the two single-version halves. Returns + * undefined when the corpus does not cover this version — reported separately as + * a coverage hole, not as behavioral drift. + */ +function selectExpectation(spec, version, versionMatchesRange) { + const matches = (spec.expectations || []).filter((exp) => versionMatchesRange(exp.version, version)); + return matches.length === 1 ? matches[0] : undefined; +} + +/** Minimal semver-range test, kept byte-compatible with the other two halves. */ +function makeRangeMatcher() { + const parse = (v) => { + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v || '')); + return m ? [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)] : undefined; + }; + const cmp = (a, b) => { + for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + return 0; + }; + return (range, version) => { + if (!range || !String(range).trim()) return true; + const have = parse(version); + if (!have) return true; + for (const token of String(range).trim().split(/\s+/)) { + let op = '='; + let ver = token; + for (const candidate of ['>=', '<=', '>', '<', '=']) { + if (token.startsWith(candidate)) { + op = candidate; + ver = token.slice(candidate.length); + break; + } + } + const c = cmp(have, parse(ver) || [0, 0, 0]); + const ok = + (op === '>=' && c >= 0) || + (op === '<=' && c <= 0) || + (op === '>' && c > 0) || + (op === '<' && c < 0) || + (op === '=' && c === 0); + if (!ok) return false; + } + return true; + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const versionMatchesRange = makeRangeMatcher(); + const { specs, enforcedRules, manifest } = loadContracts(args.contracts); + const legs = args.legs.map(loadLeg); + + log(`contracts=${specs.size} enforced(default-error)=${enforcedRules.size} legs=${legs.length}`); + for (const leg of legs) { + log( + ` leg ${leg.label}: engine=${leg.version} grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + + `detectorResults=${(leg.detector.results || []).length} backendCases=${leg.backend.size}` + ); + } + + const drifts = []; + const coverageHoles = []; + const matrix = []; // one row per rule × version, for the summary table + + // A rule that ships enabled at error severity but has no contract file is + // invisible to this whole check. Compare the manifest's declared set against + // the census each detector leg recorded from the OSD catalog it linted with, so + // a new default-error rule cannot land unvalidated. + const missingContracts = auditDefaultErrorCensus(legs, specs, enforcedRules); + + for (const [ruleId, { spec, file }] of specs) { + const isEnforced = enforcedRules.has(ruleId); + if (!isEnforced && !args.allRules) continue; + + // A rule the catalog does not apply to an engine version ships nothing to + // users there, so it needs no expectation for it. Only a rule that IS in + // scope and has no expectation is a genuine hole. + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + + for (const leg of legs) { + const inScope = versionInAppliesTo(appliesTo, leg.version); + + // A parser rule that vanished from the grammar is one fact about this + // rule on this engine, not one per query — raise it once and move on, so + // the report shows the single edit to make instead of the same paragraph + // repeated for every case. + if (inScope) { + const grammarDrift = classifyGrammarDrift({ + ruleId, + version: leg.version, + requiredParserRules: spec.requiredParserRules, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + }); + if (grammarDrift) { + drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); + matrix.push({ ruleId, version: leg.version, status: 'drift', drifts: 1 }); + continue; + } + } + + const expectation = selectExpectation(spec, leg.version, versionMatchesRange); + if (!expectation) { + if (!inScope) { + // Deliberately out of scope on this engine. Still run the classifier + // for the one case that matters — an engine that rejects a trigger the + // rule has been scoped away from (a missed diagnostic). + const outOfScopeDrifts = classifyOutOfScope({ + spec, + ruleId, + leg, + classify: classifyDrift, + }); + for (const drift of outOfScopeDrifts) { + drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + } + matrix.push({ + ruleId, + version: leg.version, + status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', + drifts: outOfScopeDrifts.length, + }); + continue; + } + // In scope on this engine but nothing pins its behavior there. + coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); + matrix.push({ ruleId, version: leg.version, status: 'uncovered', drifts: 0 }); + continue; + } + + let ruleDrifts = 0; + for (const [queryName, expected] of Object.entries(expectation.queries || {})) { + const queryDef = (spec.queries || {})[queryName]; + if (!queryDef) continue; // the single-version halves already fail on this + const query = queryDef.query.split('{{index}}').join(spec.index); + const role = queryDef.role || 'trigger'; + + const detectorResult = (leg.detector.results || []).find( + (r) => r.ruleId === ruleId && r.queryName === queryName + ); + const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); + const observedBackend = backendEntry && backendEntry.observed; + + const drift = classifyDrift({ + ruleId, + version: leg.version, + queryName, + role, + query, + expected: { + detectorCount: expected.detectorCount, + severity: expected.severity, + backendKind: expected.backend && expected.backend.kind, + }, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: backendEntry ? !!backendEntry.rejected : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + }, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + requiredParserRules: spec.requiredParserRules, + expectedBackend: expected.backend, + }); + + if (drift) { + drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + ruleDrifts++; + } + } + matrix.push({ + ruleId, + version: leg.version, + status: ruleDrifts === 0 ? 'agree' : 'drift', + drifts: ruleDrifts, + }); + } + } + + const enforcedDrifts = drifts.filter((d) => d.enforced); + const enforcedHoles = coverageHoles.filter((h) => h.enforced); + + const report = { + schemaVersion: 1, + legs: legs.map((l) => ({ + label: l.label, + engineVersion: l.version, + grammarHash: l.grammarHash, + })), + enforcedRules: [...enforcedRules].sort(), + missingContracts, + manifestDescription: manifest.description || '', + matrix, + drifts, + coverageHoles, + result: { + driftCount: drifts.length, + enforcedDriftCount: enforcedDrifts.length, + enforcedCoverageHoles: enforcedHoles.length, + missingContractCount: missingContracts.length, + passed: + enforcedDrifts.length === 0 && + enforcedHoles.length === 0 && + missingContracts.length === 0, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + const markdown = renderMarkdown(report, drifts, coverageHoles, legs); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + if (!report.result.passed) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + + `${enforcedHoles.length} coverage hole(s) and ${missingContracts.length} unvalidated ` + + `default-error rule(s).` + ); + process.exit(1); + } + log( + `PASS: every default-error rule agrees with all ${legs.length} engine version(s)` + + (drifts.length > 0 ? ` (${drifts.length} non-enforced finding(s) reported)` : '') + + '.' + ); +} + +/** Rule × version agreement matrix followed by the grouped remediation report. */ +function renderMarkdown(report, drifts, coverageHoles, legs) { + const lines = []; + lines.push('## PPL lint multi-version validation'); + lines.push(''); + lines.push( + `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + + `**${report.result.passed ? 'PASS' : 'FAIL'}** ` + + `(${report.result.enforcedDriftCount} enforced drift(s), ` + + `${report.result.enforcedCoverageHoles} coverage hole(s))` + ); + lines.push(''); + + const versions = legs.map((l) => l.version); + const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); + lines.push(`| Rule | ${versions.map((v) => `\`${v}\``).join(' | ')} |`); + lines.push(`| ---- | ${versions.map(() => '----').join(' | ')} |`); + const cell = { + agree: 'agree', + drift: 'DRIFT', + uncovered: 'not covered', + 'out-of-scope': 'n/a (out of scope)', + }; + for (const ruleId of rules) { + const cells = versions.map((version) => { + const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); + if (!row) return '—'; + return row.status === 'drift' ? `**DRIFT** (${row.drifts})` : cell[row.status]; + }); + lines.push(`| \`${ruleId}\` | ${cells.join(' | ')} |`); + } + lines.push(''); + + if ((report.missingContracts || []).length > 0) { + lines.push('### Unvalidated default-error rules'); + lines.push(''); + for (const entry of report.missingContracts) { + lines.push( + `- \`${entry.ruleId}\` ships enabled at error severity but ${entry.reason}, so no engine ` + + `version validates it. FIX: add \`${entry.ruleId}.spec.json\` under ` + + `integ-test/src/test/resources/ppl-lint/contracts/ with a trigger + control query and list ` + + `it in manifest.json under \`defaultError\`. If the rule should not be default-error, lower ` + + `its severity or disable it in packages/osd-monaco/src/ppl/lint/rules_catalog.json.` + ); + } + lines.push(''); + } + + if (coverageHoles.length > 0) { + lines.push('### Coverage holes'); + lines.push(''); + for (const hole of coverageHoles) { + lines.push( + `- \`${hole.ruleId}\` has no expectation matching engine \`${hole.version}\`` + + `${hole.enforced ? ' (ENFORCED — this rule ships to users on that engine unpinned)' : ''}. ` + + `FIX (${hole.file}): add an \`expectations[]\` entry whose \`version\` range covers ` + + `\`${hole.version}\`, or narrow the rule's \`appliesTo\` so it does not apply there.` + ); + } + lines.push(''); + } + + lines.push('### Remediation'); + lines.push(''); + lines.push('```'); + lines.push(formatDriftReport(drifts)); + lines.push('```'); + return lines.join('\n'); +} + +main(); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs new file mode 100644 index 00000000000..b343778ac3e --- /dev/null +++ b/scripts/ppl-lint/drift.mjs @@ -0,0 +1,556 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Drift classification for the multi-version PPL lint contract. + * + * The contract runs every default-error lint rule against SEVERAL engine + * versions. Detecting that a rule disagrees with an engine is only half the + * job — a bare "expected 1 diagnostic, got 0" tells an engineer that something + * moved but not what to do about it. This module turns one observed + * (expected vs detector vs backend) triple into: + * + * 1. a drift CLASS — which of the known ways engine/linter can diverge; and + * 2. a REMEDIATION — the concrete linter-side action, one of: + * disable-rule the engine no longer has the behavior; stop shipping + * the diagnostic (or scope it to older versions) + * version-scope-rule the behavior is version-bounded; fix appliesTo + * update-detector the detector's own logic/anchor is now wrong + * update-contract the linter is right; the pinned expectation is stale + * + * The classifier is deliberately pure and side-effect free so it can be unit + * tested without a cluster (see __tests__/drift.test.mjs). The runner supplies + * observations; this module decides nothing about how they were gathered. + * + * Naming: a "trigger" query is one the rule is supposed to flag; a "control" is + * a near-identical valid query it must stay silent on. `role` distinguishes them. + */ + +/** Every drift class this module can emit, with a stable one-line meaning. */ +export const DRIFT_CLASSES = { + GRAMMAR_RULE_MISSING: 'grammar-rule-missing', + ENGINE_RELAXED: 'engine-relaxed', + ENGINE_TIGHTENED: 'engine-tightened', + ENGINE_MESSAGE_CHANGED: 'engine-message-changed', + DETECTOR_SILENT: 'detector-silent', + DETECTOR_NOISY: 'detector-noisy', + VERSION_SCOPE_TOO_NARROW: 'version-scope-too-narrow', + SEVERITY_MISMATCH: 'severity-mismatch', +}; + +/** Remediation actions, phrased as what the linter engineer changes. */ +export const REMEDIATIONS = { + DISABLE_RULE: 'disable-rule', + VERSION_SCOPE_RULE: 'version-scope-rule', + UPDATE_DETECTOR: 'update-detector', + UPDATE_CONTRACT: 'update-contract', +}; + +/** OSD paths an engineer edits, kept in one place so a move is a one-line fix. */ +const OSD_PATHS = { + catalog: 'packages/osd-monaco/src/ppl/lint/rules_catalog.json', + ruleDir: 'packages/osd-monaco/src/ppl/lint/rules/', + ruleIndex: 'packages/osd-monaco/src/ppl/lint/rule_index.ts', +}; + +/** + * Path of the detector implementation for a rule. + * + * Most rules follow the snake_case-of-the-id convention, but not all: the + * catalog id `unsupported-window-function-in-eventstats` lives in + * `unsupported_window_function.ts`. A remediation that names a file the engineer + * cannot open is worse than one that names a directory, so a contract may pin the + * real path via `detectorPath` and we fall back to the convention otherwise. + */ +function detectorFile(ruleId, detectorPath) { + if (detectorPath) { + return detectorPath; + } + return `${OSD_PATHS.ruleDir}${String(ruleId).replace(/-/g, '_')}.ts`; +} + +/** + * Cheap edit-distance, used only to suggest "did the grammar rename X to Y?". + * Bounded by the shorter string, so it is O(n*m) on short identifiers. + */ +function editDistance(a, b) { + const m = a.length; + const n = b.length; + if (m === 0 || n === 0) return Math.max(m, n); + let prev = Array.from({ length: n + 1 }, (_, j) => j); + for (let i = 1; i <= m; i++) { + const row = [i]; + for (let j = 1; j <= n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost); + } + prev = row; + } + return prev[n]; +} + +/** Split a camelCase parser rule name into lower-case tokens. */ +function camelTokens(name) { + return String(name) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +/** + * Best candidates for a parser rule that vanished from the candidate grammar. + * + * Ranked by how a real ANTLR rename tends to look, strongest signal first: + * 0. containment `unionCommand` -> `unionCommandNew` + * 1. same leading token `unionCommand` -> `unionStatement` + * 2. near spelling `rexCommand` -> `regexCommand` + * + * The leading token carries the identity of the rule; the trailing one is + * usually a generic suffix (`Command`, `Clause`, `Expression`) shared by most of + * the grammar, so matching on it alone would suggest dozens of unrelated rules. + * That is why only the FIRST token counts, and why an unrelated rule returns + * nothing rather than a plausible-looking wrong guess. + */ +export function suggestParserRules(missingRule, availableRules, limit = 3) { + const missing = String(missingRule); + const lower = missing.toLowerCase(); + const missingHead = camelTokens(missing)[0]; + const scored = []; + for (const candidate of availableRules) { + const cl = String(candidate).toLowerCase(); + let score; + if (cl.includes(lower) || lower.includes(cl)) { + score = 0; // containment: strongest signal of a rename + } else if (missingHead && camelTokens(candidate)[0] === missingHead) { + score = 1; // same subject, renamed suffix + } else { + // Only very near spellings survive this tier. A looser budget scaled to + // name length lets long names match unrelated same-suffix rules + // (`unionCommand` vs `binCommand` differ by 4 edits but are unrelated), so + // the cap is absolute: typo-or-insertion distance, nothing more. + const distance = editDistance(lower, cl); + if (distance > 2) continue; + score = 1 + distance; + } + scored.push({ candidate, score }); + } + scored.sort((a, b) => a.score - b.score || String(a.candidate).localeCompare(String(b.candidate))); + return scored.slice(0, limit).map((s) => s.candidate); +} + +/** Parse "3.8.0-SNAPSHOT" / "3.7" into [major, minor, patch]; undefined if unparseable. */ +export function parseVersion(value) { + if (!value) return undefined; + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value)); + if (!match) return undefined; + return [Number(match[1]), Number(match[2] || 0), Number(match[3] || 0)]; +} + +export function compareVersion(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * True when `version` falls inside the catalog's `appliesTo` window. An absent + * bound is open-ended, and an unparseable version is treated as in-range so an + * unrecognized engine build never silently drops coverage. + */ +export function versionInAppliesTo(appliesTo, version) { + const have = parseVersion(version); + if (!have) return true; + const min = parseVersion(appliesTo && appliesTo.minVersion); + const max = parseVersion(appliesTo && appliesTo.maxVersion); + if (min && compareVersion(have, min) < 0) return false; + // maxVersion is treated as inclusive, matching the OSD catalog's own reading. + if (max && compareVersion(have, max) > 0) return false; + return true; +} + +/** Human-readable one-liner for the observed pair, reused across messages. */ +function describeObservation(observed) { + const detector = observed.detectorCount > 0 ? `flagged (${observed.detectorCount})` : 'silent'; + let backend = 'accepted'; + if (observed.backendRejected) { + const type = observed.backendType ? ` ${observed.backendType}` : ''; + backend = `rejected${type}`; + } else if (observed.backendRejected === undefined) { + backend = 'not observed'; + } + return `detector ${detector}, engine ${backend}`; +} + +/** + * Report a parser rule the detector walks that the candidate grammar no longer + * defines. Exported so a caller can raise it ONCE per rule/version — the fact is + * a property of the grammar, not of any single query, and repeating it per query + * buries the one edit an engineer has to make. `classifyDrift` still calls it so + * a caller that does not hoist the check keeps the diagnosis. + */ +export function classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed = {}, + queryName, + role = 'trigger', + query, + detectorPath, +}) { + if (!Array.isArray(requiredParserRules) || !Array.isArray(parserRuleNames)) { + return null; + } + const available = new Set(parserRuleNames); + const missing = requiredParserRules.filter((rule) => !available.has(rule)); + if (missing.length === 0) { + return null; + } + const missingList = missing.map((r) => `"${r}"`).join(', '); + const suggestions = [...new Set(missing.flatMap((rule) => suggestParserRules(rule, parserRuleNames)))]; + const at = queryName ? ` [${queryName}]` : ''; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + driftClass: DRIFT_CLASSES.GRAMMAR_RULE_MISSING, + evidence: + `${ruleId} @ ${version}${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + + `which this rule's detector walks.` + + (observed && observed.detectorCount !== undefined ? ` ${describeObservation(observed)}.` : ''), + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine grammar renamed or removed ${missingList}. ` + + (suggestions.length > 0 + ? `Closest rule(s) now in the grammar: ${suggestions.map((r) => `"${r}"`).join(', ')}. ` + : '') + + `Re-anchor the detector (and ${OSD_PATHS.ruleIndex} if the name is listed there) onto the ` + + `current rule name, then update this contract's requiredParserRules. If the command itself ` + + `is gone from the engine, disable the rule instead.`, + }, + }; +} + +/** + * Classify one query's outcome on one engine version. + * + * Returns `null` when the detector, the engine and the pinned expectation all + * agree — the overwhelmingly common case. Otherwise returns a single drift + * object; checks run most-specific-first so the reported cause is the root one + * (a renamed grammar rule explains a silent detector, not the other way round). + * + * @param {object} input + * @param {string} input.ruleId + * @param {string} input.version engine version under test, e.g. "3.7.0" + * @param {string} input.queryName + * @param {string} input.role 'trigger' | 'control' + * @param {string} input.query the query as sent to both halves + * @param {object} input.expected { detectorCount, severity, backendKind } + * @param {object} input.observed { detectorCount, severities, backendRejected, backendType, backendReason } + * @param {object} [input.wiring] OSD catalog entry (appliesTo, runtimeOnly, ...) + * @param {string[]} [input.parserRuleNames] candidate grammar's parser rule names + * @param {string[]} [input.requiredParserRules] grammar rules the detector walks + * @param {object} [input.expectedBackend] contract's pinned rejection body + */ +export function classifyDrift(input) { + const { + ruleId, + version, + queryName, + role = 'trigger', + query, + expected = {}, + observed = {}, + wiring, + parserRuleNames, + requiredParserRules, + expectedBackend, + detectorPath, + } = input; + + const detectorFlagged = (observed.detectorCount || 0) > 0; + const expectFlagged = (expected.detectorCount || 0) > 0; + const backendRejected = observed.backendRejected; + const where = `${ruleId} @ ${version} [${queryName}]`; + const base = { ruleId, version, queryName, role, query, driftVersion: version }; + + // --- 1. Did the grammar move out from under the detector? ------------------- + // A detector that walks a parser rule the candidate grammar no longer defines + // cannot fire at all. This is the root cause of an otherwise baffling silent + // detector, so it is checked before any behavioral comparison. + const grammarDrift = classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed, + queryName, + role, + query, + detectorPath, + }); + if (grammarDrift) { + return grammarDrift; + } + + // --- 2. Is the rule even in scope for this engine version? ------------------ + // A rule whose appliesTo excludes this version is intentionally inert here. + // That is only correct if the engine also does not exhibit the behavior; if the + // engine rejects the trigger, the version window is too narrow and users on + // this version get no diagnostic. + const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); + if (!inScope) { + if (role === 'trigger' && backendRejected === true) { + return { + ...base, + driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, + evidence: + `${where}: engine ${version} rejects this trigger, but the rule's appliesTo ` + + `(${JSON.stringify((wiring && wiring.appliesTo) || {})}) excludes ${version}, so no diagnostic ` + + `is shown to users on that version.`, + remediation: { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Widen "${ruleId}".appliesTo to include ${version} (lower minVersion / raise maxVersion) so the ` + + `diagnostic reaches users on engines that actually reject the query.`, + }, + }; + } + // Out of scope and the engine agrees it is a non-issue: nothing to report. + return null; + } + + // --- 3. Behavioral flips: the engine changed its verdict -------------------- + const expectRejection = expected.backendKind === 'rejection'; + + // 3a. The engine now ACCEPTS what the contract pinned as a rejection. Any + // diagnostic the linter still emits is a false positive shipped to users — + // the single most damaging drift, so it is reported even when the detector + // count happens to match the stale expectation. + if (role === 'trigger' && expectRejection && backendRejected === false) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS a query the contract pinned as rejected ` + + `(${describeObservation(observed)}). The engine gained support for this construct.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `"${ruleId}" is now a FALSE POSITIVE on ${version}. Bound it to the versions that still ` + + `reject: set appliesTo.maxVersion just below ${version}. If no supported engine rejects it ` + + `any more, set "enabled": false (disable-rule) and drop the detector. Then re-pin this ` + + `contract's ${version} expectation to detectorCount 0 / backend.kind "result-shape".`, + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectation to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; + } + + // 3b. The engine now REJECTS what the contract pinned as valid. A control that + // started failing means the linter is silently missing a real error. + if (!expectRejection && backendRejected === true) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_TIGHTENED, + evidence: + `${where}: engine ${version} now REJECTS a query the contract pinned as valid ` + + `(${observed.backendType || 'error'}: ${observed.backendReason || 'no reason'}). ` + + `${describeObservation(observed)}.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already flags this, so the linter is correct and only the pinned expectation ` + + `is stale. Re-pin the ${version} expectation to backend.kind "rejection" with the observed ` + + `error.type/reason, and pick a genuinely valid query for the control.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine rejects this but the linter is silent — a false NEGATIVE on ${version}. Extend ` + + `the detector to cover this shape (or, if the rejection belongs to a different rule, add a ` + + `contract case under that rule). Then re-pin this expectation.`, + }, + }; + } + + // --- 4. Same verdict, different wording ------------------------------------ + // The engine still rejects, but the error type/reason moved. The detector is + // still right; the pinned body — and any detector text that quotes the engine + // wording — is stale. Worth flagging because linter messages and quick-fix + // copy are written against these strings. + if (backendRejected === true && expectRejection && expectedBackend) { + const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; + const typeChanged = + expectedError.type !== undefined && + observed.backendType !== undefined && + expectedError.type !== observed.backendType; + const reasonChanged = + expectedError.reason !== undefined && + observed.backendReason !== undefined && + expectedError.reason !== observed.backendReason; + if (typeChanged || reasonChanged) { + const parts = []; + if (typeChanged) parts.push(`error.type "${expectedError.type}" -> "${observed.backendType}"`); + if (reasonChanged) + parts.push(`error.reason "${expectedError.reason}" -> "${observed.backendReason}"`); + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED, + evidence: + `${where}: engine ${version} still rejects the query but reworded the failure — ${parts.join('; ')}.`, + remediation: { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector's verdict is unaffected, so no rule change is required. Update the ${version} ` + + `expectation's backend.body to the observed wording. Also check whether "${ruleId}"'s message ` + + `or quick-fix copy in ${detectorFile(ruleId, detectorPath)} quotes the old engine wording.`, + }, + }; + } + } + + // --- 5. Detector-only disagreements ---------------------------------------- + // The engine behaved as pinned, so any mismatch is on the linter side. + if (expectFlagged && !detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_SILENT, + evidence: + `${where}: expected ${expected.detectorCount} diagnostic(s) but the detector produced none, ` + + `while the engine behaved as pinned (${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine still exhibits the behavior, so the rule is still wanted — the detector regressed. ` + + `Check, in order: (1) appliesTo/minVersion vs engine ${version}; (2) runtimeOnly — a runtimeOnly ` + + `rule only fires when the lint context's grammarSurface is "runtime-bundle"; (3) required lint ` + + `context (fields/typeMap) that the detector self-suppresses without; (4) the detector's own ` + + `traversal. Do NOT re-pin the expectation to 0 — that would hide a false negative.`, + }, + }; + } + + if (!expectFlagged && detectorFlagged) { + const engineAgrees = backendRejected === true; + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: expected no diagnostic but the detector emitted ${observed.detectorCount} ` + + `(${describeObservation(observed)}).`, + remediation: engineAgrees + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The engine rejects this query too, so the diagnostic is arguably correct and the ` + + `expectation is what is wrong. Re-pin the ${version} expectation, or choose a control query ` + + `the engine actually accepts.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine ACCEPTS this query, so the diagnostic is a false positive on ${version}. Narrow ` + + `the detector so it stops matching this shape; if the whole rule no longer applies to any ` + + `supported engine, disable it in ${OSD_PATHS.catalog}.`, + }, + }; + } + + // --- 6. Right verdict, wrong severity -------------------------------------- + if ( + expected.severity && + detectorFlagged && + Array.isArray(observed.severities) && + observed.severities.length > 0 && + !observed.severities.every((s) => s === expected.severity) + ) { + return { + ...base, + driftClass: DRIFT_CLASSES.SEVERITY_MISMATCH, + evidence: + `${where}: expected severity "${expected.severity}" but the detector emitted ` + + `${JSON.stringify(observed.severities)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: OSD_PATHS.catalog, + detail: + `Restore "${ruleId}".severity to "${expected.severity}" in the catalog, or — if the downgrade was ` + + `deliberate — re-pin this contract and note that the rule left the enforced default-error set.`, + }, + }; + } + + return null; +} + +/** + * Render drifts as the PR-facing remediation report. Grouped by remediation + * action so the reader sees the decision first ("two rules need version + * scoping") rather than a flat wall of query failures. + */ +export function formatDriftReport(drifts) { + if (drifts.length === 0) { + return 'No engine/linter drift detected.'; + } + const byAction = new Map(); + for (const drift of drifts) { + const action = drift.remediation.action; + if (!byAction.has(action)) byAction.set(action, []); + byAction.get(action).push(drift); + } + + const lines = [ + `PPL lint drift: ${drifts.length} finding(s) across ${new Set(drifts.map((d) => d.version)).size} engine version(s).`, + '', + ]; + // Most urgent action first: a false positive already reaching users outranks a + // stale pinned string. + const order = [ + REMEDIATIONS.DISABLE_RULE, + REMEDIATIONS.VERSION_SCOPE_RULE, + REMEDIATIONS.UPDATE_DETECTOR, + REMEDIATIONS.UPDATE_CONTRACT, + ]; + for (const action of order) { + const group = byAction.get(action); + if (!group || group.length === 0) continue; + lines.push(`## ${action} (${group.length})`); + for (const drift of group) { + lines.push(`- [${drift.driftClass}] ${drift.evidence}`); + lines.push(` FIX (${drift.remediation.target}): ${drift.remediation.detail}`); + // A rule-level finding (e.g. a grammar rename) has no single query behind it. + if (drift.query) { + lines.push(` QUERY: ${drift.query}`); + } + } + lines.push(''); + } + return lines.join('\n'); +} diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 8bd1ff5bad1..e45b4a48b04 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -419,6 +419,16 @@ function main() { engineVersion, grammarHash: target.grammarHash || '', differential: !!backendReport, + // Census of the rules that ship enabled at ERROR severity, read from the OSD + // catalog this run linted with. The multi-version aggregator enforces its + // `defaultError` manifest set against this list, so a rule that becomes + // default-error in OSD without a contract file cannot slip through + // unvalidated — and the aggregator does not need its own OSD checkout to + // notice (design: default-error is the set users cannot opt out of). + defaultErrorRules: catalog + .filter((rule) => rule.enabled && rule.severity === 'error') + .map((rule) => rule.id) + .sort(), results: [], }; From f825f3fed3c7bd4a1fedef45cdc3d29b944ff098 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 16:33:14 -0700 Subject: [PATCH 33/78] fix(ci): stop the multi-version lint check from passing vacuously Self-review of the multi-version drift check found five ways it could either pass while a rule had actually drifted, or hand an engineer advice that would make things worse. All five are fixed with regression tests. 1. A transport failure was read as engine ACCEPTANCE. The IT marks an unanswered query outcome:"error" and writes no `rejected` field; the aggregator coerced that absence to false. A socket timeout therefore looked like an engine that had gained support for the construct, and the report told the engineer the rule was "now a FALSE POSITIVE ... set enabled:false and drop the detector". Observations now distinguish "engine accepted" from "no verdict received", and an uncomparable case is not classified at all -- a dead leg can no longer manufacture linter advice. 2. A reworded engine message masked a detector that had gone silent. The engine-message-changed branch returned before the detector-silent check without verifying the detector still agreed, reporting "the detector's verdict is unaffected, so no rule change is required". Re-pinning the string would have turned the check green over a rule that no longer fired. It now requires the detector count to match, so when both moved the silent detector is reported instead. 3. A rule whose every case was uncomparable reported "agree". "agree" now means "we compared something and it matched"; otherwise the pair is reported inconclusive and FAILS, with advice to check that leg's logs and re-run rather than to edit anything. 4. A dead observe job silently shrank the matrix. The aggregate step dropped legs with no report and then printed "agrees with all N versions" for the survivors. It now requires every planned version to have produced a leg. 5. Two narrower cases: a detector that fires on a version its appliesTo excludes was unreportable (reachable in production, since OSD's version filter runs a rule when the cluster version is unknown), and selectExpectation was missing the engine:"calcite" filter both single-version halves apply. Also: a fixture index that fails to seed on an older engine is now recorded, so its contracts report as unusable instead of turning IndexNotFoundException into a fake engine verdict -- which would otherwise have advised pinning the contract to that exception. Verified: 43 tests green (up from 35); the healthy 7-rule x 3-version matrix still passes and the five-injected-drift scenario still yields five correct remediations. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 20 +++ .../remote/PplLintRuleValidationIT.java | 71 ++++++++- scripts/ppl-lint/README.md | 16 +- .../__tests__/aggregate-versions.test.mjs | 110 +++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 52 ++++++ scripts/ppl-lint/aggregate-versions.mjs | 149 +++++++++++++++--- scripts/ppl-lint/drift.mjs | 46 +++++- 7 files changed, 436 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f4ad09b6a82..e9ee6849111 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -397,19 +397,39 @@ jobs: # contracts, then print the remediation report. - name: Aggregate drift across engine versions id: aggregate + env: + RELEASED: ${{ needs.plan.outputs.released }} run: | set -euo pipefail shopt -s nullglob args=() + present=() for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do [ -f "$leg/detector-report.json" ] || continue version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') args+=(--leg "$version=$leg") + present+=("$version") done if [ ${#args[@]} -eq 0 ]; then echo "::error::no complete legs to aggregate." exit 1 fi + # Every version the plan asked for must have produced a leg. Aggregating + # only the survivors would report "PASS: agrees with all N versions" over + # a matrix that silently lost one — the exact vacuous pass this workflow + # exists to prevent. A dead leg is a failure, not a smaller matrix. + missing=() + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do + found=no + for have in "${present[@]}"; do + [ "$have" = "$want" ] && found=yes && break + done + [ "$found" = yes ] || missing+=("$want") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." + exit 1 + fi node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index ae74fdd2515..065bc4e78eb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -94,6 +94,13 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + /** + * Index fixtures that could not be created on this engine (observe-only mode only). Contracts + * that need one are reported as {@code outcome: "error"} instead of as engine behavior, because + * an IndexNotFoundException from a missing fixture is not a verdict about the query. + */ + private final Set unseededIndices = new LinkedHashSet<>(); + @Override public void init() throws Exception { super.init(); @@ -108,9 +115,16 @@ public void init() throws Exception { } // In the multi-version matrix an older engine may not support a field type // a fixture uses (a mapping that only exists in a later release). Losing - // that one index must not abort the whole leg — the contracts that need it - // will surface as their own observations, while every other rule is still + // that one index must not abort the whole leg — every other rule is still // validated against this engine. + // + // But it must not be silent either: without the index, every query against + // it fails with IndexNotFoundException, which looks exactly like a real + // engine verdict. Left unmarked, the drift report would advise pinning the + // contract to IndexNotFoundException, or "extending the detector" for a + // control the engine only rejected because its data was missing. Record the + // failure so those cases are reported as unusable rather than as behavior. + unseededIndices.add(indexEnum); System.err.println( "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); } @@ -154,6 +168,16 @@ private void runContract( JSONObject fixture = contract.optJSONObject("backendFixture"); boolean calciteOn = fixtureCalciteEnabled(fixture); + // A contract whose fixture index never got created cannot produce a meaningful + // observation: every query would fail with IndexNotFoundException regardless of + // the rule. Report each case as an error so the aggregator counts it as + // inconclusive rather than as the engine's verdict. + String missingIndex = missingFixtureIndex(fixture); + if (missingIndex != null) { + recordUnusableContract(ruleId, index, queries, report, missingIndex); + return; + } + List applied = applyClusterSettings(fixture); try { JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); @@ -209,6 +233,49 @@ private void runContract( } } + /** + * The first index fixture this contract needs that failed to seed, or null when every index it + * declares is present. Only ever non-null in observe-only mode, where a seeding failure is + * tolerated instead of aborting the leg. + */ + private String missingFixtureIndex(JSONObject fixture) { + if (unseededIndices.isEmpty() || fixture == null) { + return null; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + return unseededIndices.contains("ACCOUNT") ? "ACCOUNT" : null; + } + for (int i = 0; i < declared.length(); i++) { + String name = declared.getString(i); + if (unseededIndices.contains(name)) { + return name; + } + } + return null; + } + + /** + * Record every case of a contract whose fixture index is missing as {@code outcome: "error"}, so + * the multi-version aggregator treats them as inconclusive. Writing nothing at all would be + * worse: absent rows are indistinguishable from a detector that never ran. + */ + private void recordUnusableContract( + String ruleId, String index, JSONObject queries, JSONArray report, String missingIndex) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + report.put( + reportEntry(ruleId, queryName, role, query, "observe-only") + .put("outcome", "error") + .put( + "error", + "fixture index " + missingIndex + " could not be created on this engine")); + log(ruleId, queryName, "SKIPPED (fixture index " + missingIndex + " unavailable)"); + } + } + /** * Observe-only helper: run every query a contract declares and record what the engine actually * did, without comparing against any expectation. Used when this engine version has no matching diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index a78b1f5cdcc..76b3d56a0e0 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -240,14 +240,26 @@ renamed or removed — the finding names the closest current rule names), (the engine rejects but the rule is scoped away from that version, so users see no diagnostic), and `severity-mismatch`. -Two guards keep the check from passing vacuously: +Four guards keep the check from passing vacuously. Each exists because "we could +not check" must never render as "it is fine": - A rule that is default-error in OSD's catalog but has no contract file fails the run. The detector runner records the catalog's default-error census in `detector-report.json`, and the aggregate step compares it against `manifest.defaultError` — so a new error rule cannot land unvalidated. - A leg whose artifacts are missing is a hard failure, never a silently dropped - version. + version. The aggregate step also checks that every version the plan asked for + produced a report, so a dead observe job cannot shrink the matrix into a green + "agrees with all N versions". +- A case with no engine verdict (a transport failure, recorded by the IT as + `outcome: "error"`) is **not** read as acceptance. Coercing it would report a + timeout as an engine that now accepts the query — and advise disabling a + perfectly good rule. Likewise, a contract whose fixture index failed to seed is + reported as unusable rather than as a stream of `IndexNotFoundException` + verdicts. +- A rule whose every case was uncomparable is reported `inconclusive` and **fails** + — it proved nothing. Inconclusive findings say "check that leg's logs and re-run", + never "edit the rule", because the linter is not what went wrong. A rule that is out of scope on an engine (`appliesTo` excludes it) and that the engine also accepts is reported as `n/a (out of scope)`, not as drift — that is diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 8f1bd7493a5..76e7b558181 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -341,6 +341,116 @@ test('a legacy detector report without a census warns instead of failing', () => assert.match(stdout, /no detector leg reported a defaultErrorRules census/); }); +// --- "we don't know" must never render as "it's fine" ------------------------- + +/** Write a leg where a named query produced no engine verdict (transport failure). */ +function writeLegWithTransportError({ version, erroredQuery, cases }) { + const dir = writeLeg({ version, cases }); + const file = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(file, 'utf8')).map((entry) => + entry.queryName === erroredQuery + ? // Exactly what the IT writes on a transport failure: an `error` outcome and + // NO `rejected` field, because no verdict was ever received. + { ruleId: entry.ruleId, queryName: entry.queryName, role: entry.role, outcome: 'error', error: 'connect timeout' } + : { ...entry, outcome: 'observed' } + ); + fs.writeFileSync(file, JSON.stringify(backend)); + return dir; +} + +test('a transport error is not read as engine acceptance', () => { + // Regression: coercing a missing `rejected` to false made a timeout look like an + // engine that now ACCEPTS the trigger, and advised disabling a healthy rule. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'trigger', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; + const { report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-relaxed').length, + 0, + 'a timeout must never be reported as the engine relaxing' + ); + assert.ok( + !/FALSE POSITIVE/.test(stdout), + 'a timeout must never advise disabling or version-scoping a rule' + ); + assert.match(stdout, /1 not compared: trigger \(no engine verdict\)/); +}); + +test('a leg where nothing could be compared is inconclusive, not agreement', () => { + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + // Both cases lose their verdict: the whole leg proved nothing. + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', outcome: 'error', error: 'timeout' }, + ]) + ); + const { status, report, stdout } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'inconclusive must be red — "could not check" is not "passed"'); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.drifts.length, 0, 'a dead leg must not manufacture linter advice'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(stdout, /Inconclusive \(leg problem, not a linter problem\)/); +}); + +test('a reworded engine message does not mask a detector that went silent', () => { + // Regression: ENGINE_MESSAGE_CHANGED returned before the detector-silent check, + // so the report said "no rule change is required" while the rule had stopped + // firing. Re-pinning the string would have gone green over a dead rule. + const dir = writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 0, rejected: true, reason: 'union needs >= 2 datasets' }, + control: { detector: 0, rejected: false }, + }, + }); + const { report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + const drift = report.drifts.find((d) => d.queryName === 'trigger'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('a detector firing on a version its appliesTo excludes is reported', () => { + // OSD's version filter runs a rule when the cluster version is unknown, so an + // out-of-scope rule CAN reach users. Silence here would hide that false positive. + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.6.0'); + assert.equal(drift.driftClass, 'detector-noisy'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('a calcite-scoped expectation is selected rather than counted twice', () => { + // Both single-version halves drop `engine: "calcite"` entries when Calcite is + // off. Without that filter here, a per-engine pair for one range matches twice + // and is misreported as an uncovered version. + const contracts = writeContracts({ + expectations: [ + SPEC.expectations[0], + { ...SPEC.expectations[0], engine: undefined, queries: SPEC.expectations[0].queries }, + ], + }); + const { report } = run({ contracts, legs: healthyLegs() }); + // Two matching entries is genuinely ambiguous and must not silently pick one. + assert.ok( + report.coverageHoles.length > 0 || report.matrix.some((m) => m.status === 'uncovered'), + 'an ambiguous pair of expectations must be surfaced, not resolved arbitrarily' + ); +}); + test('a bad --leg argument is rejected', () => { const result = spawnSync( process.execPath, diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 63640958473..2ecb55d9964 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -228,6 +228,58 @@ test('a reworded rejection is update-contract and points at quoted copy', () => assert.match(drift.remediation.detail, /quotes the old engine wording/); }); +test('a reworded rejection does not mask a detector that stopped firing', () => { + // Regression: this branch used to return before the detector-silent check, so a + // simultaneous rewording + detector regression reported "the detector's verdict + // is unaffected, no rule change is required". Re-pinning the string would have + // turned the check green over a rule that no longer fires at all. + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); +}); + +test('a detector firing where appliesTo excludes the version is a false positive', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + // The version filter's unknown-version behavior is why this reaches users. + assert.match(drift.remediation.detail, /version is unknown/); +}); + +test('an unobserved engine verdict is never treated as acceptance', () => { + // backendRejected: undefined means "we never got an answer". It must not select + // the engine-relaxed branch, which would advise disabling a healthy rule. + const drift = classifyDrift( + agreeingTrigger({ + observed: { detectorCount: 1, severities: ['error'], backendRejected: undefined }, + }) + ); + assert.equal(drift, null); +}); + test('a changed exception type is reported even when the reason is unchanged', () => { const drift = classifyDrift( agreeingTrigger({ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 7b3f6d174fd..4ec33300975 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -194,6 +194,43 @@ function auditDefaultErrorCensus(legs, specs, enforcedRules) { return missing; } +/** + * Read one backend report entry into an observation, distinguishing "the engine + * accepted this" from "we never got an answer". + * + * This distinction is load-bearing. The IT marks a transport-level failure + * `outcome: "error"` and, having never received a verdict, writes no `rejected` + * field. Coercing that absence to `false` would report a timeout as an engine + * that now ACCEPTS a query it used to reject — which reads as an engine + * relaxation and would advise disabling a perfectly good rule. Anything that is + * not a real observed verdict becomes `undefined`, which the classifier treats as + * "not observed" rather than as acceptance. + * + * Returns `{ observed, usable }`: `usable` is false when this case produced no + * comparable engine verdict, so the caller can refuse to call it agreement. + */ +function readBackendObservation(backendEntry, detectorResult) { + const observedBackend = (backendEntry && backendEntry.observed) || undefined; + const outcome = backendEntry && backendEntry.outcome; + // `observed`/`error` are the observe-only outcomes; `pass`/`fail` come from the + // asserting mode. Only those carry a real verdict. + const hasVerdict = + !!backendEntry && + outcome !== 'error' && + (typeof backendEntry.rejected === 'boolean' || !!observedBackend); + + return { + usable: hasVerdict && !!detectorResult, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: hasVerdict ? !!backendEntry.rejected : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + }, + }; +} + /** * Check an out-of-scope rule for the one drift that still matters there: the * engine rejects a trigger query, but the rule's `appliesTo` excludes this @@ -244,9 +281,23 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { * "exactly one must match" rule as the two single-version halves. Returns * undefined when the corpus does not cover this version — reported separately as * a coverage hole, not as behavioral drift. + * + * The `engine` filter matters as much as the version range: both single-version + * halves (`PplLintRuleValidationIT.selectExpectation` and + * `run-frontend-contract.mjs`) drop `engine: "calcite"` entries when Calcite is + * off. Omitting it here would make a contract that pins one range per engine match + * TWICE and be misreported as an uncovered version. Every leg in this workflow + * runs with Calcite enabled (the observation legs do not disable it), so a + * calcite-scoped expectation is in play; a contract's `frontendContext.isCalcite: + * false` opts out. */ function selectExpectation(spec, version, versionMatchesRange) { - const matches = (spec.expectations || []).filter((exp) => versionMatchesRange(exp.version, version)); + const isCalcite = !((spec.frontendContext || {}).isCalcite === false); + const matches = (spec.expectations || []).filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && !isCalcite) return false; + return true; + }); return matches.length === 1 ? matches[0] : undefined; } @@ -303,6 +354,10 @@ function main() { const drifts = []; const coverageHoles = []; + // Rule/version pairs where no case could actually be compared (a leg that lost + // its engine verdicts or its detector rows). Tracked separately from drift + // because the answer is "re-run / fix the leg", not "edit the linter". + const inconclusive = []; const matrix = []; // one row per rule × version, for the summary table // A rule that ships enabled at error severity but has no contract file is @@ -372,9 +427,17 @@ function main() { } let ruleDrifts = 0; + let compared = 0; + const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; - if (!queryDef) continue; // the single-version halves already fail on this + if (!queryDef) { + // The contract references a query it does not define. The single-version + // halves fail on this, but skipping it silently here would shrink the + // compared set without saying so. + unusable.push(`${queryName} (not defined in the contract's queries map)`); + continue; + } const query = queryDef.query.split('{{index}}').join(spec.index); const role = queryDef.role || 'trigger'; @@ -382,7 +445,19 @@ function main() { (r) => r.ruleId === ruleId && r.queryName === queryName ); const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); - const observedBackend = backendEntry && backendEntry.observed; + const { observed, usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + // No comparable pair, so there is nothing to classify. Attempting it + // anyway would turn a dead leg into linter advice: a case with no engine + // verdict and no detector row looks exactly like "the detector went + // silent", and the report would tell the engineer to go fix a detector + // that is fine. Record it as not compared and move on. + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + continue; + } + compared++; const drift = classifyDrift({ ruleId, @@ -395,13 +470,7 @@ function main() { severity: expected.severity, backendKind: expected.backend && expected.backend.kind, }, - observed: { - detectorCount: detectorResult ? detectorResult.actual : 0, - severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: backendEntry ? !!backendEntry.rejected : undefined, - backendType: observedBackend ? observedBackend.type : undefined, - backendReason: observedBackend ? observedBackend.reason : undefined, - }, + observed, wiring: spec.wiring, detectorPath: spec.detectorPath, parserRuleNames: leg.parserRuleNames, @@ -414,17 +483,39 @@ function main() { ruleDrifts++; } } - matrix.push({ - ruleId, - version: leg.version, - status: ruleDrifts === 0 ? 'agree' : 'drift', - drifts: ruleDrifts, - }); + // "agree" has to mean "we compared something and it matched". A rule whose + // every case lost its engine verdict (a timed-out leg) or its detector row + // (a runner that died mid-corpus) has proven nothing, and calling that + // agreement is exactly the vacuous pass this check exists to prevent. + if (compared === 0) { + inconclusive.push({ + ruleId, + file, + version: leg.version, + enforced: isEnforced, + reasons: unusable, + }); + matrix.push({ ruleId, version: leg.version, status: 'inconclusive', drifts: ruleDrifts }); + } else { + if (unusable.length > 0) { + log( + `WARN: ${ruleId} @ ${leg.version} compared ${compared} case(s); ` + + `${unusable.length} not compared: ${unusable.join(', ')}` + ); + } + matrix.push({ + ruleId, + version: leg.version, + status: ruleDrifts === 0 ? 'agree' : 'drift', + drifts: ruleDrifts, + }); + } } } const enforcedDrifts = drifts.filter((d) => d.enforced); const enforcedHoles = coverageHoles.filter((h) => h.enforced); + const enforcedInconclusive = inconclusive.filter((i) => i.enforced); const report = { schemaVersion: 1, @@ -439,15 +530,20 @@ function main() { matrix, drifts, coverageHoles, + inconclusive, result: { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, enforcedCoverageHoles: enforcedHoles.length, missingContractCount: missingContracts.length, + enforcedInconclusive: enforcedInconclusive.length, + // An inconclusive default-error rule fails too: "we could not check" must + // never render as "it is fine". passed: enforcedDrifts.length === 0 && enforcedHoles.length === 0 && - missingContracts.length === 0, + missingContracts.length === 0 && + enforcedInconclusive.length === 0, }, }; @@ -469,8 +565,8 @@ function main() { // eslint-disable-next-line no-console console.error( `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + - `${enforcedHoles.length} coverage hole(s) and ${missingContracts.length} unvalidated ` + - `default-error rule(s).` + `${enforcedHoles.length} coverage hole(s), ${missingContracts.length} unvalidated ` + + `default-error rule(s) and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` ); process.exit(1); } @@ -514,6 +610,21 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { } lines.push(''); + if ((report.inconclusive || []).length > 0) { + lines.push('### Inconclusive (leg problem, not a linter problem)'); + lines.push(''); + for (const entry of report.inconclusive) { + lines.push( + `- \`${entry.ruleId}\` on engine \`${entry.version}\`: no case could be compared — ` + + `${entry.reasons.join('; ')}. This is NOT a lint finding: the engine or the detector run ` + + `did not answer, so nothing was validated. Check that leg's job logs (an unreachable ` + + `cluster, an index that failed to seed, or a detector runner that died mid-corpus) and ` + + `re-run. Do not edit the rule or the contract on the strength of this.` + ); + } + lines.push(''); + } + if ((report.missingContracts || []).length > 0) { lines.push('### Unvalidated default-error rules'); lines.push(''); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index b343778ac3e..62f4ad5b7d9 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -327,6 +327,34 @@ export function classifyDrift(input) { }, }; } + // Out of scope but the detector fired anyway. `appliesTo` is applied by OSD's + // version filter, which runs a rule when the cluster version is UNKNOWN — so a + // user whose version could not be resolved sees a diagnostic the catalog says + // does not apply to them. If the engine accepts the query, that is a false + // positive reaching exactly the users the version window was meant to protect. + if (detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: the rule's appliesTo (${JSON.stringify((wiring && wiring.appliesTo) || {})}) ` + + `excludes ${version}, yet the detector emitted ${observed.detectorCount} diagnostic(s) ` + + `(${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `A rule out of scope for ${version} must stay silent there. Check that the detector honors ` + + `the version context rather than deciding on its own, and remember OSD's version filter runs ` + + `a rule when the cluster version is unknown — so this also fires for users whose version ` + + `could not be resolved.` + + (backendRejected === true + ? ` The engine does reject this query, so widening appliesTo in ${OSD_PATHS.catalog} may be` + + ` the right fix instead.` + : ''), + }, + }; + } // Out of scope and the engine agrees it is a non-issue: nothing to report. return null; } @@ -397,11 +425,19 @@ export function classifyDrift(input) { } // --- 4. Same verdict, different wording ------------------------------------ - // The engine still rejects, but the error type/reason moved. The detector is - // still right; the pinned body — and any detector text that quotes the engine - // wording — is stale. Worth flagging because linter messages and quick-fix - // copy are written against these strings. - if (backendRejected === true && expectRejection && expectedBackend) { + // The engine still rejects, but the error type/reason moved. The pinned body — + // and any detector text that quotes the engine wording — is stale. Worth + // flagging because linter messages and quick-fix copy are written against these + // strings. + // + // Requires the detector to still agree with the expectation (`detectorMatches`). + // Without that condition a reworded message would MASK a detector that went + // silent at the same time: the report would say "the detector's verdict is + // unaffected, no rule change required", the engineer would re-pin the string, + // and the check would go green over a rule that no longer fires. When both moved + // at once, the silent detector is the more serious story and step 5 tells it. + const detectorMatches = (observed.detectorCount || 0) === (expected.detectorCount || 0); + if (backendRejected === true && expectRejection && expectedBackend && detectorMatches) { const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; const typeChanged = expectedError.type !== undefined && From 809c031c3935a3a0cd6e0b996c0e975667442ec4 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:01:41 -0700 Subject: [PATCH 34/78] fix(ci): forward ppl.lint.* system properties to the test JVM by prefix The multi-version matrix's first real run failed on the 3.6.0 leg with nine contract assertion failures. Root cause: `-Dppl.lint.observe.only=true` never reached the test JVM. integ-test/build.gradle forwards ppl.lint.* properties via a hand-maintained allowlist, and the new property was added to the IT and to the workflow but not to that list -- so the leg ASSERTED expectations pinned for 3.8 against a 3.6 engine, which is precisely the failure observe-only exists to prevent. Forward by prefix instead of by list. A hand-maintained list is a silent trap: an omitted property produces no error anywhere, it just quietly does nothing, and the resulting failure looks like a product bug rather than a plumbing bug. Any `ppl.lint.*` property the invoker sets now reaches the test JVM, so adding a knob to the IT is sufficient. Verified: `:integ-test:integTestRemote --dry-run` configures cleanly with -Dppl.lint.observe.only and -Dppl.lint.report set. Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index bdb126a0090..822b44cde2b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -169,18 +169,25 @@ tasks.withType(licenseHeaders.class) { } // Forward the PPL lint rule validation contract knobs to every integ test JVM -// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), an -// optional path to write the observed backend report, and — while the cluster is -// alive — optional paths to export the candidate runtime grammar bundle and its -// target manifest for the detector-validation job. Applied globally so every +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), +// whether to observe rather than assert (the multi-version matrix), an optional +// path to write the observed backend report, and — while the cluster is alive — +// optional paths to export the candidate runtime grammar bundle and its target +// manifest for the detector-validation job. Applied globally so every // RestIntegTestTask that runs the class picks it up without per-task edits. +// +// Forwarded by PREFIX rather than by an explicit list. A hand-maintained list is +// a silent trap: a property the IT reads but the list omits simply never reaches +// the test JVM, with no error anywhere. That already cost one CI run — +// `ppl.lint.observe.only` was added to the IT and the workflow but not to the +// list, so a multi-version leg asserted expectations pinned for a DIFFERENT +// engine version and failed instead of observing. Forwarding every `ppl.lint.*` +// property the invoker set means adding a knob to the IT is enough. tasks.withType(Test).configureEach { systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") - ["ppl.lint.report", "ppl.lint.grammar.bundle", "ppl.lint.target"].each { prop -> - if (System.getProperty(prop) != null) { - systemProperty prop, System.getProperty(prop) - } - } + System.properties.stringPropertyNames() + .findAll { it.startsWith("ppl.lint.") && it != "ppl.lint.schedule" } + .each { prop -> systemProperty prop, System.getProperty(prop) } } validateNebulaPom.enabled = false From 935a8f1e7ad2e82ee8a1f416c8f6cb6704983422 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:17:06 -0700 Subject: [PATCH 35/78] test(ppl-lint): pin per-version rejection shapes observed on 3.6 and 3.7 The first real multi-version run reported 5 drifts, all engine-message-changed. They were genuine version differences, not linter faults: the DETECTOR verdict is identical on 3.6/3.7/3.8 for every rule (each count matched, live-observed), but older engines describe the same rejection differently. Two epochs, both observed rather than guessed: - 3.6 masks the cause entirely. Every rejection collapses to HTTP 400 IllegalArgumentException / "Invalid Query" -- for capture-group names, unknown fields, flat_object references and replace wildcard mismatches alike. Pinning that string buys no discriminating power on its own, so on 3.6 these rules rely on the detector count for attribution; the note in each contract says so. - 3.7 already carries the specific wording 3.8 uses, so >=3.7.0 covers both. eventstats needs three epochs because its status also moved: 3.6 gives HTTP 500 UnsupportedOperationException / "There was internal problem at backend", 3.7 keeps the 500 but names the function, and 3.8 turned it into a proper HTTP 400 CalciteUnsupportedException. Keeping both epochs (rather than relaxing the newer expectation) is the point: the matrix now proves each rule still fires on older engines instead of reporting the wording difference as drift. Two reporting fixes found while reading the real output: an `inconclusive` matrix cell rendered BLANK because the status had no label (a blank cell reads as "nothing to see here", the opposite of what it means), and the headline counted only drifts and holes -- so a run failing solely on inconclusive rules displayed "0 drifts, 0 holes -- FAIL", which looks like a reporting bug rather than the actual cause. Unmapped statuses now render visibly and every failure reason is named in the headline. Verified by replaying all three real CI leg artifacts through the aggregator: 5 drifts -> 0, every version agreeing on all 5 rules whose detectors CI could load. 43 tests green. Signed-off-by: Hanyu Wei --- .../contracts/field-validation.spec.json | 84 +++++++++++++++++-- .../contracts/flat-object-subfield.spec.json | 81 ++++++++++++++++-- .../invalid-capture-group-name.spec.json | 59 +++++++++++-- .../replace-wildcard-asymmetry.spec.json | 60 +++++++++++-- ...ed-window-function-in-eventstats.spec.json | 51 ++++++++++- scripts/ppl-lint/aggregate-versions.mjs | 25 +++++- 6 files changed, 331 insertions(+), 29 deletions(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index c67343ef23a..45f85e3b769 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -13,12 +13,19 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, - "visibleIndices": ["{{index}}"], + "visibleIndices": [ + "{{index}}" + ], "deriveFromMapping": { "account_number": "long", "balance": "long", @@ -50,7 +57,52 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "queries": { "unknown-field-existence": { "detectorCount": 1, @@ -59,7 +111,13 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } + } } }, "grok-field-slot-shape-typo": { @@ -68,12 +126,24 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } + } } }, "known-field-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json index 25ac4cf66cf..440c55e27ac 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -3,7 +3,10 @@ "ruleId": "flat-object-subfield", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["qualifiedName", "wcQualifiedName"], + "requiredParserRules": [ + "qualifiedName", + "wcQualifiedName" + ], "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", "wiring": { "detector": "flat-object-subfield", @@ -15,8 +18,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["FLAT_OBJECT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "FLAT_OBJECT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, @@ -47,7 +55,68 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "flat-object-dotted-subfield": { @@ -100,7 +169,9 @@ "backend": { "kind": "result-shape", "httpStatus": 200, - "expect": { "datarowsNonEmpty": true } + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index c651af803b6..3d39fa2e3dc 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -3,7 +3,10 @@ "ruleId": "invalid-capture-group-name", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["rexCommand", "stringLiteral"], + "requiredParserRules": [ + "rexCommand", + "stringLiteral" + ], "wiring": { "detector": "invalid-capture-group-name", "enabled": true, @@ -14,12 +17,19 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, - "deriveFromMapping": { "email": "text" } + "deriveFromMapping": { + "email": "text" + } }, "index": "opensearch-sql_test_index_account", "queries": { @@ -34,7 +44,38 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "rex-capture-name-underscore": { @@ -54,7 +95,13 @@ }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 8946dad60a9..593bb92023d 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -3,7 +3,10 @@ "ruleId": "replace-wildcard-asymmetry", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["replacePair", "stringLiteral"], + "requiredParserRules": [ + "replacePair", + "stringLiteral" + ], "wiring": { "detector": "replace-wildcard-asymmetry", "enabled": true, @@ -11,11 +14,19 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.4.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -33,7 +44,38 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "replace-symmetric-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "replace-wildcard-count-mismatch": { @@ -53,7 +95,13 @@ }, "replace-symmetric-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 6f6fbd313b0..d6821273175 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -31,9 +31,58 @@ "query": "source={{index}} | eventstats avg(age) as avg_age" } }, + "notes": "The DETECTOR verdict is identical on 3.6/3.7/3.8 (1 diagnostic on the trigger, 0 on the control, live-observed). Only the engine's rejection shape moves, in three epochs: 3.6 masks the cause entirely (HTTP 500 UnsupportedOperationException / 'There was internal problem at backend'); 3.7 keeps the 500 but names the function; 3.8 turned it into a proper HTTP 400 CalciteUnsupportedException. Each epoch is pinned separately so the multi-version check proves the rule still fires on older engines instead of reporting the wording difference as drift. A 500 is a poor rejection oracle (it cannot distinguish this failure from an unrelated crash), which is why the <3.8 entries additionally rely on the detector count for discrimination.", "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + }, + { + "version": ">=3.7.0 <3.8.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + }, + { + "version": ">=3.8.0", "queries": { "eventstats-rank": { "detectorCount": 1, diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 4ec33300975..9efc55bd850 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -582,11 +582,23 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { const lines = []; lines.push('## PPL lint multi-version validation'); lines.push(''); + // Every reason the run can be red belongs in the headline. Reporting only + // drifts and holes made a FAIL caused solely by inconclusive rules read as + // "0 drifts, 0 holes — FAIL", which looks like a reporting bug rather than the + // real cause. + const reasons = [ + `${report.result.enforcedDriftCount} enforced drift(s)`, + `${report.result.enforcedCoverageHoles} coverage hole(s)`, + ]; + if (report.result.enforcedInconclusive) { + reasons.push(`${report.result.enforcedInconclusive} inconclusive`); + } + if (report.result.missingContractCount) { + reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); + } lines.push( `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + - `**${report.result.passed ? 'PASS' : 'FAIL'}** ` + - `(${report.result.enforcedDriftCount} enforced drift(s), ` + - `${report.result.enforcedCoverageHoles} coverage hole(s))` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); @@ -599,12 +611,17 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { drift: 'DRIFT', uncovered: 'not covered', 'out-of-scope': 'n/a (out of scope)', + inconclusive: '**inconclusive**', }; for (const ruleId of rules) { const cells = versions.map((version) => { const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); if (!row) return '—'; - return row.status === 'drift' ? `**DRIFT** (${row.drifts})` : cell[row.status]; + if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; + // An unmapped status must still render as something visible. A blank cell + // reads as "nothing to see here", which is the opposite of what an + // unrecognized state means. + return cell[row.status] || `**${row.status}**`; }); lines.push(`| \`${ruleId}\` | ${cells.join(' | ')} |`); } From 458e8dedffa08440b0f512632345a67c49368992 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:50:18 -0700 Subject: [PATCH 36/78] fix(ci): treat a missing version expectation as an observation, not a failure With the property plumbing fixed, the 3.6 leg went from nine contract failures to one: "[union-min-datasets] no version expectation matches backend version 3.6.0". union-min-datasets is a >=3.7 rule, so having no 3.6 expectation is correct -- the aggregator already classifies that as out-of-scope. But observe-only mode could not suppress it, because selectExpectation records into whatever failure list it is handed and was handed the real one. The leg therefore recorded the observation AND failed on it, which is precisely the "a disagreement is the signal, not a broken run" contract that observe-only exists to honor. Hand selectExpectation a scratch list in observe-only mode and discard it. The two other failure sites are deliberately left failing even in observe-only: a contract referencing an undefined query, and a grammar bundle that could not be exported, are both broken-run problems rather than engine behavior, and silencing them would produce an empty report that reads as agreement. The same run confirmed the earlier work landed: invalid-capture-group-name now PASSes on 3.6 against its new generic-rejection epoch, and the 3.7.0 and pr-build legs both went green. Signed-off-by: Hanyu Wei --- .../calcite/remote/PplLintRuleValidationIT.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 065bc4e78eb..7ae505e9b35 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -180,14 +180,20 @@ private void runContract( List applied = applyClusterSettings(fixture); try { - JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); + // In observe-only mode, "no expectation matches this version" is information, + // not a failure — a rule the corpus does not pin for THIS engine is exactly + // what the multi-version matrix is here to learn. selectExpectation records + // into whatever list it is handed, so hand it a scratch list we discard; + // otherwise the leg both records the observation AND fails, which is what + // kept union-min-datasets (a >=3.7 rule) failing the 3.6 leg. + List selectionFailures = observeOnly ? new ArrayList<>() : failures; + JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, selectionFailures); if (selected == null) { if (!observeOnly) { return; // no/ambiguous version expectation — failure already recorded. } - // Observe-only: an engine the corpus does not pin is exactly what the - // multi-version matrix wants to learn about, so record the raw behavior of - // every query and let the aggregator decide whether the gap matters. + // Record the raw behavior of every query and let the aggregator decide + // whether the gap matters (out-of-scope rule vs a real coverage hole). observeAllQueries(ruleId, index, queries, report); return; } From cbaf909e2f44e78966087db60c90827e28bee7bd Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:55:36 -0700 Subject: [PATCH 37/78] fix(ci): do not report an unsupported command as a too-narrow version window With all 7 default-error rules finally loading, the matrix came back 20/21 cells agreeing, with one drift: version-scope-too-narrow for union-min-datasets on 3.6, advising "widen appliesTo to include 3.6.0". That advice was wrong. On 3.6 the `union` command does not exist at all -- the rule's CONTROL query (a valid two-dataset union) is rejected with the same SyntaxCheckException as the single-dataset trigger. The classifier judged the trigger in isolation, so a rejection it read as "the engine has this behavior, your rule just is not scoped to it" was really "this command is unsupported here". Following it would ship a diagnostic claiming a precise cause ("union requires at least two datasets") for a query that fails simply because the command is unknown. Pass whether the control was also rejected. When it was, the version window is doing its job and the pair is correctly reported out-of-scope rather than as drift. This is the same failure mode as the earlier vacuous-pass fixes, in the opposite direction: not a missed finding, but a confidently WRONG remediation. A rejection only means what the rule claims if a comparable valid query succeeds. Verified by replaying all three real CI leg artifacts: the matrix is now fully green -- 7 default-error rules x 3 engine versions (3.6.0, 3.7.0, 3.8.0-SNAPSHOT), 0 drifts, 0 coverage holes, 0 inconclusive. 44 tests green. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/__tests__/drift.test.mjs | 22 ++++++++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 12 ++++++++++++ scripts/ppl-lint/drift.mjs | 12 +++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 2ecb55d9964..4c8d4e6f4dc 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -368,6 +368,28 @@ test('a downgraded severity is caught even when the count is right', () => { // --- version scoping -------------------------------------------------------- +test('an unsupported command is not mistaken for a too-narrow version window', () => { + // Real case from CI: on 3.6 the `union` command does not exist, so BOTH the + // trigger and the control fail with SyntaxCheckException. Judging the trigger + // alone said "widen appliesTo to 3.6" — which would ship a diagnostic claiming a + // precise cause ("requires at least two datasets") for what is really + // "unsupported command". A rejected control means the version window is right. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Invalid Query', + }, + controlAlsoRejected: true, + }) + ); + assert.equal(drift, null); +}); + test('an out-of-scope rule on an engine that rejects is scoped too narrowly', () => { const drift = classifyDrift( agreeingTrigger({ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 9efc55bd850..5ff5c513a22 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -243,6 +243,17 @@ function readBackendObservation(backendEntry, detectorResult) { */ function classifyOutOfScope({ spec, ruleId, leg, classify }) { const found = []; + + // Did this rule's CONTROL query — a valid use of the same command — also get + // rejected on this engine? If so the command itself is unsupported here, and a + // rejected trigger says nothing about the rule's specific condition. Computed + // once, since it is a property of the rule on this engine. + const controlAlsoRejected = Object.entries(spec.queries || {}).some(([name, def]) => { + if ((def.role || 'trigger') !== 'control') return false; + const entry = leg.backend.get(`${ruleId}::${name}`); + return !!(entry && entry.rejected); + }); + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); @@ -268,6 +279,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { }, wiring: spec.wiring, detectorPath: spec.detectorPath, + controlAlsoRejected, // Deliberately no parser-rule check here: a grammar that lacks the rule is // expected on an engine the command predates. }); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 62f4ad5b7d9..43c9979bb6d 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -261,6 +261,8 @@ export function classifyGrammarDrift({ * @param {string[]} [input.parserRuleNames] candidate grammar's parser rule names * @param {string[]} [input.requiredParserRules] grammar rules the detector walks * @param {object} [input.expectedBackend] contract's pinned rejection body + * @param {boolean} [input.controlAlsoRejected] true when this rule's control query was + * ALSO rejected on this engine, i.e. the command itself is unsupported here */ export function classifyDrift(input) { const { @@ -276,6 +278,7 @@ export function classifyDrift(input) { requiredParserRules, expectedBackend, detectorPath, + controlAlsoRejected, } = input; const detectorFlagged = (observed.detectorCount || 0) > 0; @@ -310,7 +313,14 @@ export function classifyDrift(input) { // this version get no diagnostic. const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); if (!inScope) { - if (role === 'trigger' && backendRejected === true) { + // A trigger the engine rejects normally means the version window is too + // narrow. But if the rule's CONTROL — a valid query using the same command — + // is rejected too, the command itself does not exist on this engine yet, and + // the rejection says nothing about the rule's specific condition. Widening + // appliesTo there would ship a diagnostic that claims a precise cause for + // what is really "unsupported command", so that case is correctly silent: + // the version window is doing its job. + if (role === 'trigger' && backendRejected === true && controlAlsoRejected !== true) { return { ...base, driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, From 6988ece3b11e7fe20e5250891234d8d8c6b037aa Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 12:38:40 -0700 Subject: [PATCH 38/78] fix(ci): apply the unobserved-verdict rule to the out-of-scope path too Code review of this branch found three defects, all with one root cause: the "an absent verdict is not a verdict" protection added earlier lives in readBackendObservation, which only the IN-SCOPE path used. The out-of-scope path re-read `entry.rejected` inline and so lost it. 1. An errored trigger on an out-of-scope rule coerced to `rejected: false`, and the version-scope-too-narrow check needs `=== true` -- so a genuinely mis-scoped rule (3.6 rejects the trigger, 3.6 users get no diagnostic) rendered as `out-of-scope` with exit 0. Reachable via the new recordUnusableContract path: an unseeded fixture marks every case errored, so a version-scoped rule validates nothing and still shows green. 2. `controlAlsoRejected` failed OPEN for the same reason, producing exactly the wrong advice it was written to prevent -- "lower union-min-datasets' minVersion to 3.6", shipping "union requires two datasets" to users whose query fails only because the command is absent. Fixed by making the control verdict THREE-state: rejected (command unsupported -> suppress), accepted (command works -> report), unknown (no verdict -> cannot tell, so stay quiet). Collapsing it to a boolean was the bug. 3. `compared === 0` let a partially-observed rule render `agree`. flat-object-subfield has three triggers and one control; if all three triggers lost their verdict and only the control survived, `compared === 1` and the cell said `agree`. Triggers ARE the rule's behavioral claim, so they are now counted separately and losing all of them is inconclusive. Verified: the genuine version-scope-too-narrow finding still fires (only unobserved cases are suppressed, not real ones), and replaying all three real CI leg artifacts is still fully green -- 7 default-error rules x 3 engine versions. 47 tests, up from 44; one existing test's tail assertion was updated because losing a single-trigger spec's only trigger now correctly reports inconclusive rather than passing with a warning. Signed-off-by: Hanyu Wei --- .../__tests__/aggregate-versions.test.mjs | 97 ++++++++++++++++++- scripts/ppl-lint/aggregate-versions.mjs | 77 ++++++++++----- 2 files changed, 149 insertions(+), 25 deletions(-) diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 76e7b558181..65606dbb926 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -368,7 +368,7 @@ test('a transport error is not read as engine acceptance', () => { cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, }), }; - const { report, stdout } = run({ contracts: writeContracts(), legs }); + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); assert.equal( report.drifts.filter((d) => d.driftClass === 'engine-relaxed').length, 0, @@ -378,7 +378,39 @@ test('a transport error is not read as engine acceptance', () => { !/FALSE POSITIVE/.test(stdout), 'a timeout must never advise disabling or version-scoping a rule' ); - assert.match(stdout, /1 not compared: trigger \(no engine verdict\)/); + // This spec has a single trigger, so losing it means the rule's behavioral + // claim went unchecked: inconclusive and red, not a passing WARN. + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.match(stdout, /trigger \(no engine verdict\)/); +}); + +test('losing every trigger is inconclusive even when a control still compares', () => { + // flat-object-subfield's real shape: several triggers plus one control. The + // triggers ARE the rule's claim, so a leg that kept only the control has proven + // nothing — but `compared > 0`, so a naive count would have rendered `agree`. + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { status, report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'a rule whose triggers all went unobserved must not read as agreement'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(report.result.enforcedInconclusive, 1); }); test('a leg where nothing could be compared is inconclusive, not agreement', () => { @@ -451,6 +483,67 @@ test('a calcite-scoped expectation is selected rather than counted twice', () => ); }); +test('an errored trigger on an out-of-scope rule does not silently pass', () => { + // The out-of-scope path used to read `entry.rejected` directly. An errored + // observation has no such field, so it coerced to false, the + // version-scope-too-narrow check (which needs `=== true`) never fired, and a + // genuinely mis-scoped rule rendered as `out-of-scope` with exit 0. + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion => out of scope + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + // The point is that an unobserved trigger yields no CLAIM either way: it must + // not be reported as a confident out-of-scope agreement... + assert.equal( + report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow').length, + 0, + 'an unobserved trigger cannot support a version-scope finding' + ); + // ...nor may it invent linter advice from a verdict that never arrived. + assert.equal(report.drifts.length, 0); +}); + +test('an errored control cannot fail open into "widen appliesTo" advice', () => { + // controlAlsoRejected suppresses the version-scope finding when the command + // itself is unsupported. Reading `entry.rejected` raw made that suppression fail + // OPEN on an errored control: the run would then advise lowering minVersion, + // shipping a precise-cause diagnostic for an unknown-command failure. + const dir = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: true } }, + }); + const backend = JSON.parse(fs.readFileSync(path.join(dir, 'backend-report.json'), 'utf8')).map( + (e) => + e.role === 'control' + ? { ruleId: e.ruleId, queryName: e.queryName, role: e.role, outcome: 'error', error: 'timeout' } + : { ...e, outcome: 'observed' } + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + const { report, stdout } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const scoped = report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow'); + assert.equal( + scoped.length, + 0, + 'with the control unobserved there is no evidence the command is supported, so no widening advice' + ); + assert.ok(!/Widen "/.test(stdout)); +}); + test('a bad --leg argument is rejected', () => { const result = spawnSync( process.execPath, diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 5ff5c513a22..2f49bf5b3b8 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -244,24 +244,42 @@ function readBackendObservation(backendEntry, detectorResult) { function classifyOutOfScope({ spec, ruleId, leg, classify }) { const found = []; - // Did this rule's CONTROL query — a valid use of the same command — also get - // rejected on this engine? If so the command itself is unsupported here, and a - // rejected trigger says nothing about the rule's specific condition. Computed - // once, since it is a property of the rule on this engine. - const controlAlsoRejected = Object.entries(spec.queries || {}).some(([name, def]) => { - if ((def.role || 'trigger') !== 'control') return false; - const entry = leg.backend.get(`${ruleId}::${name}`); - return !!(entry && entry.rejected); - }); + // What did this rule's CONTROL queries — valid uses of the same command — do on + // this engine? THREE states, not two, and the difference decides whether a + // rejected trigger means anything: + // rejected the command itself is unsupported here, so the trigger's rejection + // says nothing about the rule's specific condition -> suppress + // accepted the command works, so a rejected trigger really is the rule's + // condition going unreported on this version -> report it + // unknown no control verdict arrived (errored/absent). We cannot tell the two + // apart, so we must not emit confident advice either way. + // Collapsing this to a boolean is what let the suppression fail open: an errored + // control read as "not rejected" and produced the exact "widen appliesTo" advice + // this check exists to prevent. + const controlVerdicts = Object.entries(spec.queries || {}) + .filter(([, def]) => (def.role || 'trigger') === 'control') + .map(([name]) => { + const entry = leg.backend.get(`${ruleId}::${name}`); + const { observed } = readBackendObservation(entry, { actual: 0, severities: [] }); + return observed.backendRejected; + }); + const controlAlsoRejected = controlVerdicts.some((v) => v === true); + // A rule with controls, none of which produced a verdict, cannot be judged here. + const controlUnknown = + controlVerdicts.length > 0 && !controlVerdicts.some((v) => typeof v === 'boolean'); for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); if (!backendEntry) continue; // this leg never ran the query - const observedBackend = backendEntry.observed || {}; const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName ); + // Same reason as above: an errored observation must not read as "the engine + // accepted this". On this path that coercion would turn a genuinely + // mis-scoped rule into a silent `out-of-scope` PASS, because the + // version-scope-too-narrow check requires backendRejected === true. + const { observed: outOfScopeObserved } = readBackendObservation(backendEntry, detectorResult); const drift = classify({ ruleId, version: leg.version, @@ -270,16 +288,13 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { query: queryDef.query.split('{{index}}').join(spec.index), // Out of scope means the rule is expected to stay silent here. expected: { detectorCount: 0 }, - observed: { - detectorCount: detectorResult ? detectorResult.actual : 0, - severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: !!backendEntry.rejected, - backendType: observedBackend.type, - backendReason: observedBackend.reason, - }, + observed: outOfScopeObserved, wiring: spec.wiring, detectorPath: spec.detectorPath, - controlAlsoRejected, + // An unknown control verdict is treated the same as a rejected one: both + // mean "we cannot claim this engine supports the command", and staying quiet + // is the only honest option. + controlAlsoRejected: controlAlsoRejected || controlUnknown, // Deliberately no parser-rule check here: a grammar that lacks the rule is // expected on an engine the command predates. }); @@ -440,6 +455,14 @@ function main() { let ruleDrifts = 0; let compared = 0; + // Triggers are counted separately from controls. A trigger is the rule's + // entire behavioral claim ("this query is flagged"); a control only says the + // rule stays quiet nearby. So a rule that lost every trigger but kept one + // control has proven nothing about itself, even though `compared` is + // non-zero — flat-object-subfield has 3 triggers and 1 control, and would + // otherwise render `agree` off the control alone. + let triggersExpected = 0; + let triggersCompared = 0; const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; @@ -452,6 +475,9 @@ function main() { } const query = queryDef.query.split('{{index}}').join(spec.index); const role = queryDef.role || 'trigger'; + if (role === 'trigger') { + triggersExpected++; + } const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName @@ -470,6 +496,9 @@ function main() { continue; } compared++; + if (role === 'trigger') { + triggersCompared++; + } const drift = classifyDrift({ ruleId, @@ -495,11 +524,13 @@ function main() { ruleDrifts++; } } - // "agree" has to mean "we compared something and it matched". A rule whose - // every case lost its engine verdict (a timed-out leg) or its detector row - // (a runner that died mid-corpus) has proven nothing, and calling that - // agreement is exactly the vacuous pass this check exists to prevent. - if (compared === 0) { + // "agree" has to mean "we compared the rule's claim and it held". A rule + // whose every case lost its engine verdict (a timed-out leg) or its detector + // row (a runner that died mid-corpus) has proven nothing — and so has one + // that lost every TRIGGER while keeping a control, since the triggers are + // where the rule's behavior actually lives. Calling either agreement is the + // vacuous pass this check exists to prevent. + if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { inconclusive.push({ ruleId, file, From 4f6cf244ee5c4da5cc20f81a84da01d75111d881 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:21:59 -0700 Subject: [PATCH 39/78] =?UTF-8?q?test(ci):=20TEMPORARY=20drift=20probe=20?= =?UTF-8?q?=E2=80=94=20do=20not=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deliberately stale the >=3.7.0 pinned rejection reason for invalid-capture-group-name so the multi-version check has a real behavior change to classify. Both the 3.7.0 and pr-build legs reject this trigger with "Invalid capture group name 'user_name'."; this pins the truncated wording. Expected: engine-message-changed / update-contract on two engine legs, and a red Detect job. Reverted immediately after the run. Signed-off-by: Hanyu Wei --- .../ppl-lint/contracts/invalid-capture-group-name.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index 3d39fa2e3dc..d2ca4dca8d2 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -88,7 +88,7 @@ "status": 400, "error": { "type": "IllegalArgumentException", - "reason": "Invalid capture group name 'user_name'." + "reason": "Invalid capture group name" } } } From 91fa87d04ab5aa8dfcb869b73a3ee4a130baef10 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:34:52 -0700 Subject: [PATCH 40/78] =?UTF-8?q?Revert=20"test(ci):=20TEMPORARY=20drift?= =?UTF-8?q?=20probe=20=E2=80=94=20do=20not=20merge"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 052fbb190ae7520daa420bbee93050b47df48a8e. Signed-off-by: Hanyu Wei --- .../ppl-lint/contracts/invalid-capture-group-name.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index d2ca4dca8d2..3d39fa2e3dc 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -88,7 +88,7 @@ "status": 400, "error": { "type": "IllegalArgumentException", - "reason": "Invalid capture group name" + "reason": "Invalid capture group name 'user_name'." } } } From 596398f6a60535657cd8c517361bab1b949a3d9f Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:55:32 -0700 Subject: [PATCH 41/78] feat(ci): surface PPL lint drift as GitHub annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drift report already says what to change, but it only reached the job summary. GitHub renders annotations at the top of the run page, and the only one a failing multi-version run produced was "Process completed with exit code 1" — so the natural next click went to raw job logs instead of the remediation. Emit each finding as a workflow command before the summary. An update-contract finding anchors on the exact expectations[] entry whose version range produced it, so when the contract is part of the PR's diff the drift also attaches inline to the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on the contract's ruleId instead. A line number is emitted only when it is unambiguous: a contract that pins the same range twice, or a range that cannot be found, yields file-only. A wrong line sends the reader to edit the wrong expectation, which is worse than making them find it. Severity carries meaning. Inconclusive findings are warnings, not errors: they mean the leg did not answer, the run is already red from the exit code, and listing them beside real drift invites editing a rule because a leg timed out. Non-enforced drift is a warning for the same reason it does not fail the run. Verified against the real leg artifacts from a multi-version run (three engine versions, three distinct grammar hashes): a clean corpus emits nothing, and a stale pinned reason annotates the correct expectation line on both affected engine legs. 12 new tests; 59 green. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 31 +++ scripts/ppl-lint/__tests__/annotate.test.mjs | 193 ++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 24 +- scripts/ppl-lint/annotate.mjs | 227 +++++++++++++++++++ 4 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 scripts/ppl-lint/__tests__/annotate.test.mjs create mode 100644 scripts/ppl-lint/annotate.mjs diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 76b3d56a0e0..0faebbb57ff 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -266,6 +266,37 @@ engine also accepts is reported as `n/a (out of scope)`, not as drift — that i the version window working. But if the engine *rejects* the trigger there, it is `version-scope-too-narrow`. +### Where a failure shows up in the GitHub UI + +Every finding is emitted twice, because the run page and the diff are two +different places a developer looks: + +1. **Annotations** (top of the run page, and inline on the file in *Files + changed* when the contract is part of the PR's diff). Each carries the drift + class, the rule, the engine version, and the one-line action. An + `update-contract` finding anchors on the exact `expectations[]` entry whose + `version` range produced it — not the top of the file — so the drift appears on + the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on + the contract's `ruleId` instead. +2. **The job summary** — the rule × version table plus the full grouped + remediation report, which stays the authoritative account. + +Without the annotations the only thing above the summary is `Process completed +with exit code 1`, so the natural next click lands in raw job logs rather than the +remediation. Severity is not cosmetic: + +| Finding | Level | Why | +| --- | --- | --- | +| enforced drift, coverage hole | `error` | a shipped default-error rule disagrees with a supported engine | +| non-enforced drift | `warning` | reported, but it does not block | +| `inconclusive` | `warning` | "we could not check" is a leg problem; the run is already red from the exit code, and rendering it as an error invites editing a rule because a leg timed out | +| unvalidated default-error rule | `error` (no file) | the edit goes in `manifest.json`, not a contract | + +A line number is emitted only when it is unambiguous. If a contract pins the same +version range twice, or the range cannot be found, the annotation carries the file +and no line — a wrong line sends the reader to edit the wrong expectation, which +is worse than making them find it. + ### Running the multi-version check locally Each leg needs a reachable cluster. Point the observe step at any running engine: diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs new file mode 100644 index 00000000000..dc55503a6ad --- /dev/null +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -0,0 +1,193 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + buildAnnotations, + contractRepoPath, + findExpectationLine, + findRuleIdLine, + formatAnnotation, +} from '../annotate.mjs'; + +/** A contract shaped like the real ones, with two version-scoped expectations. */ +const CONTRACT = `{ + "schemaVersion": 3, + "ruleId": "invalid-capture-group-name", + "queries": { + "trigger": { "role": "trigger", "query": "source={{index}} | rex ..." } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": {} + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": {} + } + ] +} +`; + +const readStub = (text) => () => text; + +test('anchors on the expectation entry that drifted, not the first one', () => { + assert.equal(findExpectationLine(CONTRACT, '>=3.7.0'), 14); + assert.equal(findExpectationLine(CONTRACT, '>=3.4.0 <3.7.0'), 9); +}); + +test('an ambiguous or absent range yields no line rather than a wrong one', () => { + // A wrong line number sends the reader to edit the wrong expectation, which is + // worse than making them find it: prefer no anchor. + const duplicated = CONTRACT.replace('">=3.4.0 <3.7.0"', '">=3.7.0"'); + assert.equal(findExpectationLine(duplicated, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, '>=9.9.9'), undefined); + assert.equal(findExpectationLine(undefined, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, undefined), undefined); +}); + +test('incidental whitespace does not defeat the anchor', () => { + const spaced = CONTRACT.replace('"version": ">=3.7.0"', '"version": ">=3.7.0"'); + assert.equal(typeof findExpectationLine(spaced, '>=3.7.0'), 'number'); +}); + +test('falls back to the ruleId line for rule-wide findings', () => { + assert.equal(findRuleIdLine(CONTRACT), 3); +}); + +test('a drift with no expectation range still anchors at the rule', () => { + // grammar-rule-missing is a fact about the rule on that engine, so it carries no + // expectationRange — it must still land on the file at a usable line. + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + driftClass: 'grammar-rule-missing', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'the candidate grammar has no parser rule(s) "rexCommand"', + remediation: { action: 'update-detector', detail: 'Re-anchor the detector.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, 3); + assert.equal(annotations[0].level, 'error'); +}); + +test('inconclusive findings are warnings, never errors', () => { + // "We could not check" must not sit in the error list beside real drift, or the + // reader edits a rule because a leg timed out. + const annotations = buildAnnotations( + { + inconclusive: [ + { + ruleId: 'field-validation', + version: '3.6.0', + enforced: true, + file: 'field-validation.spec.json', + reasons: ['unknown-field-existence (no engine verdict)'], + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].level, 'warning'); + assert.match(annotations[0].message, /NOT a lint finding/); + assert.match(annotations[0].message, /Do not edit the rule/); +}); + +test('a non-enforced drift is a warning so it cannot be read as blocking', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'head-without-sort', + version: '3.8.0', + driftClass: 'detector-noisy', + enforced: false, + contractFile: 'head-without-sort.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations[0].level, 'warning'); +}); + +test('an unvalidated rule has no file to point at', () => { + const annotations = buildAnnotations( + { missingContracts: [{ ruleId: 'sort-on-eval-field', reason: 'has no contract file' }] }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].file, undefined); + assert.equal(annotations[0].level, 'error'); + assert.match(annotations[0].message, /manifest\.defaultError/); +}); + +test('paths are repo-relative so GitHub can render them inline', () => { + // An absolute path still annotates the run, but never attaches to the diff. + assert.equal( + contractRepoPath('/w/integ-test/res/contracts', 'a.spec.json', '/w'), + 'integ-test/res/contracts/a.spec.json' + ); + // No workspace (a local run): absolute is the honest answer. + assert.equal(contractRepoPath('/w/c', 'a.spec.json', undefined), '/w/c/a.spec.json'); +}); + +test('workflow-command metacharacters are escaped', () => { + const line = formatAnnotation({ + level: 'error', + file: 'a,b:c.json', + line: 12, + title: 'has: comma, and colon', + message: 'first\nsecond 100% done', + }); + // Commas/colons in properties would otherwise terminate the property list. + assert.match(line, /file=a%2Cb%3Ac\.json/); + assert.match(line, /title=has%3A comma%2C and colon/); + // Newlines must survive as %0A or the annotation is truncated to one line. + assert.match(line, /first%0Asecond 100%25 done/); + assert.ok(line.startsWith('::error ')); +}); + +test('an unreadable contract still produces a file-less annotation', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'r', + version: '3.8.0', + driftClass: 'detector-silent', + enforced: true, + contractFile: 'gone.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/c', workspace: '/w', readFile: () => undefined } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, undefined); + assert.equal(annotations[0].file, 'c/gone.spec.json'); +}); + +test('a clean report emits nothing', () => { + assert.deepEqual(buildAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 2f49bf5b3b8..83bb5bd39e1 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -33,6 +33,7 @@ import fs from 'fs'; import path from 'path'; +import { emitAnnotations } from './annotate.mjs'; import { classifyDrift, classifyGrammarDrift, @@ -520,7 +521,17 @@ function main() { }); if (drift) { - drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + // `expectationRange` is what the annotation anchors to: the version + // string identifies WHICH `expectations[]` entry produced this finding, + // so a `update-contract` annotation can land on that entry's line rather + // than at the top of the file. + drifts.push({ + ...drift, + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); ruleDrifts++; } } @@ -593,6 +604,17 @@ function main() { fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); log(`wrote ${args.out}`); + // Emitted BEFORE the summary on purpose. GitHub renders annotations at the top + // of the run page, which is where a developer looks first; without them the only + // thing above the summary is "Process completed with exit code 1" and the + // natural next click goes to raw logs instead of the remediation. When the + // contract is part of the PR's diff these also attach inline to the exact + // expectation that drifted. + emitAnnotations(report, { + contractsDir: args.contracts, + workspace: process.env.GITHUB_WORKSPACE, + }); + const markdown = renderMarkdown(report, drifts, coverageHoles, legs); // eslint-disable-next-line no-console console.log(markdown); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs new file mode 100644 index 00000000000..06b1a384db7 --- /dev/null +++ b/scripts/ppl-lint/annotate.mjs @@ -0,0 +1,227 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * GitHub Actions annotations for the PPL lint multi-version check. + * + * The drift report and the job summary already say exactly what to change. The + * problem is WHERE a developer looks first: GitHub renders workflow-command + * annotations at the top of the run page and, when a finding names a file in the + * pull request, inline on that file in the Files-changed view. Without them the + * only thing above the summary is "Process completed with exit code 1", so the + * natural next click leads into raw job logs instead of the remediation. + * + * This module turns findings into `::error file=…,line=…::` commands. Two rules + * govern everything here: + * + * 1. An annotation must point at a line the reader can act on, or carry no line + * at all. A confidently wrong line number sends someone to edit the wrong + * expectation, which is worse than making them find it themselves. + * 2. The annotation is a POINTER, not the report. It carries the finding and the + * one-line action; the summary keeps the full reasoning. Annotation text is + * truncated by the UI, so front-load the identity of the problem. + * + * Inconclusive findings are deliberately `::warning`, not `::error`: they mean + * "we could not check", and the run is already red from the exit code. Rendering + * them as errors next to real drift would invite exactly the response the + * classifier works to prevent — editing a rule because a leg timed out. + */ + +import fs from 'fs'; +import path from 'path'; + +/** Escape a workflow-command property value (file/title). */ +function escapeProperty(value) { + return String(value) + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') + .replace(/:/g, '%3A') + .replace(/,/g, '%2C'); +} + +/** Escape a workflow-command message body; newlines must survive as %0A. */ +function escapeData(value) { + return String(value).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +/** + * Line of the `expectations[]` entry whose `version` is `range`, 1-indexed. + * + * Deliberately a text scan rather than a JSON walk: JSON.parse discards line + * information, and every consumer of this number is a human reading the file in a + * browser. Returns undefined when the range is absent or ambiguous (appears more + * than once), because an annotation with no line still lands on the file while a + * wrong line actively misleads. + */ +export function findExpectationLine(contractText, range) { + if (!contractText || !range) return undefined; + const lines = contractText.split('\n'); + const needle = `"version": ${JSON.stringify(range)}`; + const hits = []; + for (let i = 0; i < lines.length; i++) { + // Match on the normalized form so incidental whitespace does not defeat it. + if (lines[i].replace(/\s+/g, ' ').includes(needle)) hits.push(i + 1); + } + return hits.length === 1 ? hits[0] : undefined; +} + +/** + * Line of the top-level `"ruleId"` key, used as the fallback anchor when the + * finding is about the rule as a whole (a renamed grammar rule) rather than one + * pinned expectation. + */ +export function findRuleIdLine(contractText) { + if (!contractText) return undefined; + const lines = contractText.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (/^\s*"ruleId"\s*:/.test(lines[i])) return i + 1; + } + return undefined; +} + +/** + * Repo-relative path of a contract file, for `file=`. + * + * GitHub only renders an annotation inline when the path is relative to the + * repository root AND the file is in the pull request's diff. `contractsDir` is + * an absolute path inside the workspace, so strip the workspace prefix. + */ +export function contractRepoPath(contractsDir, fileName, workspace) { + const absolute = path.join(contractsDir, fileName); + if (workspace && absolute.startsWith(workspace)) { + return path.relative(workspace, absolute); + } + return absolute; +} + +/** + * Build the annotation list for a drift report. Pure: returns descriptors so the + * caller decides where they are written and the tests can assert on them without + * capturing stdout. + */ +export function buildAnnotations(report, { contractsDir, workspace, readFile = readContract } = {}) { + const annotations = []; + const textCache = new Map(); + const contractText = (fileName) => { + if (!fileName) return undefined; + if (!textCache.has(fileName)) { + textCache.set(fileName, readFile(contractsDir, fileName)); + } + return textCache.get(fileName); + }; + + for (const drift of report.drifts || []) { + const file = drift.contractFile; + const text = contractText(file); + // `update-contract` findings are about one pinned expectation, so anchor + // there. Everything else is about the rule, so anchor at its identity line — + // the reader's next stop is the detector named in the message anyway. + const line = + (drift.expectationRange ? findExpectationLine(text, drift.expectationRange) : undefined) ?? + findRuleIdLine(text); + + annotations.push({ + level: drift.enforced ? 'error' : 'warning', + file: file ? contractRepoPath(contractsDir, file, workspace) : undefined, + line, + title: `PPL lint drift: ${drift.driftClass} (${drift.ruleId} @ ${drift.version})`, + // Message order matters: the UI truncates, so lead with what moved, then the + // action, then where. The summary carries the full rationale. + message: [ + drift.evidence, + `FIX (${drift.remediation.action}): ${drift.remediation.detail}`, + drift.query ? `QUERY: ${drift.query}` : undefined, + ] + .filter(Boolean) + .join('\n'), + }); + } + + for (const hole of report.coverageHoles || []) { + const text = contractText(hole.file); + annotations.push({ + level: hole.enforced ? 'error' : 'warning', + file: hole.file ? contractRepoPath(contractsDir, hole.file, workspace) : undefined, + line: findRuleIdLine(text), + title: `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}`, + message: + `No expectation in this contract matches engine ${hole.version}, so nothing pins ` + + `"${hole.ruleId}" there. Add a reviewed expectation whose version range covers ` + + `${hole.version}; do not widen an existing range to absorb it unless the behavior is ` + + `genuinely identical.`, + }); + } + + // Warning, not error: the linter is not what went wrong, and the run is already + // red from the exit code. See the module comment. + for (const entry of report.inconclusive || []) { + const text = contractText(entry.file); + annotations.push({ + level: 'warning', + file: entry.file ? contractRepoPath(contractsDir, entry.file, workspace) : undefined, + line: findRuleIdLine(text), + title: `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version} (leg problem)`, + message: + `No case could be compared for "${entry.ruleId}" on engine ${entry.version}` + + (entry.reasons && entry.reasons.length > 0 ? ` — ${entry.reasons.join('; ')}` : '') + + `. This is NOT a lint finding: the engine or the detector run did not answer, so ` + + `nothing was validated. Check that leg's job logs and re-run. Do not edit the rule or ` + + `the contract on the strength of this.`, + }); + } + + for (const missing of report.missingContracts || []) { + const ruleId = missing.ruleId || missing; + annotations.push({ + level: 'error', + // A rule with no contract has no file to point at; the manifest is where the + // reader's edit goes. + file: undefined, + title: `PPL lint unvalidated rule: ${ruleId}`, + message: + `"${ruleId}" ships enabled at error severity in OSD's rules_catalog.json but ` + + `${missing.reason || 'has no contract in this corpus'}. A default-error rule with no ` + + `contract is invisible to this check. Add a contract file and list it under ` + + `manifest.defaultError, or lower the rule's severity in OSD.`, + }); + } + + return annotations; +} + +function readContract(contractsDir, fileName) { + try { + return fs.readFileSync(path.join(contractsDir, fileName), 'utf8'); + } catch { + // A contract we cannot read still deserves a file-less annotation. + return undefined; + } +} + +/** Render one descriptor as a workflow command line. */ +export function formatAnnotation(annotation) { + const props = []; + if (annotation.file) props.push(`file=${escapeProperty(annotation.file)}`); + if (annotation.line) props.push(`line=${annotation.line}`); + if (annotation.title) props.push(`title=${escapeProperty(annotation.title)}`); + const suffix = props.length > 0 ? ` ${props.join(',')}` : ''; + return `::${annotation.level}${suffix}::${escapeData(annotation.message)}`; +} + +/** + * Emit annotations for a report. No-op unless running under Actions (or forced), + * so a local run is not spammed with workflow-command noise. + */ +export function emitAnnotations(report, options = {}) { + const enabled = options.force || process.env.GITHUB_ACTIONS === 'true'; + if (!enabled) return []; + const annotations = buildAnnotations(report, options); + for (const annotation of annotations) { + // eslint-disable-next-line no-console + console.log(formatAnnotation(annotation)); + } + return annotations; +} From cfe378b9589348a09177a1bebd720de0ee3bd2df Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 20:48:17 -0700 Subject: [PATCH 42/78] feat(ci): validate the compiled-simplified lint surface too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OSD ships lint on two grammar surfaces and a user gets whichever one their session resolves to. This check only ever validated one of them. lintRuntimePPLQuery falls back to the compiled-simplified surface whenever the runtime bundle is unavailable — no dataset selected, an engine below 3.6, or a bundle that has not loaded. That path is not a degraded copy: field_validation runs a text-side pass keyed on grammarSurface === 'compiled-simplified' that the runtime path never executes. It is also the surface with no engine floor, since it needs no grammar export, so it is where old-engine coverage is possible at all (GET /_plugins/_ppl/_grammar landed in 3.6; 2.19 through 3.5 cannot export a bundle). Add PPL_LINT_SURFACE to the runner. It defaults to runtime-bundle, so the required check is unchanged, and the compiled surface is an explicit opt-in rather than a fallback: a missing bundle on the runtime surface stays a hard failure, because quietly linting OSD's own grammar instead of the candidate would validate the wrong thing. runtimeOnly rules are the trap. lint_runner skips them on the compiled surface because the productions they walk are absent there, so their zero diagnostics mean 'deliberately inert', not 'the detector went silent'. Counted as zero, three healthy rules would classify as detector-silent drift. They are now reported not-applicable, rendered 'n/a (surface)', and a rule whose every case is inert is n/a — not agree (it proved nothing) and not inconclusive (nothing went wrong, and there is nothing to re-run). Also key the summary matrix on the leg label instead of the engine version: two legs can share a version while validating different surfaces, and keying on version alone made them collide so one leg's cells rendered in place of the other's. Verified with a real compiled leg produced by the runner against a live OSD checkout, beside the real 3.7/pr-build runtime legs: baseline passes with the three runtimeOnly rules n/a; a regression injected only on the compiled surface is caught as detector-silent while both runtime legs stay green; the two same-version legs no longer collide; a report predating the surface field still reads as runtime-bundle. 16 existing drift scenarios and 59 unit tests unchanged. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 49 +++++++- scripts/ppl-lint/aggregate-versions.mjs | 83 ++++++++++-- scripts/ppl-lint/run-frontend-contract.mjs | 139 +++++++++++++++++++-- 3 files changed, 247 insertions(+), 24 deletions(-) diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 0faebbb57ff..9dcc2c8d454 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -213,9 +213,52 @@ run the **same** contract oracle (`PplLintRuleValidationIT`) with against expectations — on an older engine a mismatch is the signal being collected, not a broken run. -**Engine floor: 3.6.0.** `GET /_plugins/_ppl/_grammar` landed in #5162, which is -an ancestor of 3.6 but not 3.5, so a 3.5 leg could not export a grammar bundle for -the detectors to lint against. +**Engine floor: 3.6.0 — for the runtime-bundle surface.** +`GET /_plugins/_ppl/_grammar` landed in #5162, which is an ancestor of 3.6 but not +3.5, so a 3.5 leg cannot export a grammar bundle for the detectors to lint against. + +### The two grammar surfaces + +OSD ships lint on **two** surfaces, and a user gets whichever one their session +resolves to (`lintRuntimePPLQuery`): + +| Surface | When the product uses it | Engine floor | +| --- | --- | --- | +| `runtime-bundle` | the engine exported a grammar bundle and it has loaded | 3.6.0 | +| `compiled-simplified` | no bundle — no dataset selected, engine below 3.6, or bundle not yet loaded | none | + +The compiled surface is not a degraded copy of the runtime one: it runs detector +logic the runtime path does not (`field_validation`'s text-side pass keys off +`grammarSurface === 'compiled-simplified'`). It is also the surface with no engine +floor, so it is where old-engine coverage is possible at all. + +`PPL_LINT_SURFACE` selects which surface a detector run validates. It defaults to +`runtime-bundle`, so the required check is unchanged, and the compiled surface is +an **explicit opt-in** — never a silent fallback. A missing bundle on the runtime +surface stays a hard failure, because quietly linting OSD's own grammar instead of +the candidate would validate the wrong thing. + +**`runtimeOnly` rules do not run on the compiled surface.** `lint_runner` skips +them (the productions they walk are absent from the compiled grammar), so a +compiled leg reports them `not-applicable` rather than as zero diagnostics. This +distinction is load-bearing: counted as zero, a healthy rule would classify as +`detector-silent` drift and send someone to "fix" it. In the summary table those +cells read `n/a (surface)`, and a rule whose every case is inert is `n/a` — not +`agree` (it proved nothing) and not `inconclusive` (nothing went wrong, and there +is nothing to re-run). + +Two legs may share an engine version while validating different surfaces, so the +matrix is keyed on the **leg label**, not the version. + +```bash +# A compiled-surface leg: no grammar bundle needed, so any engine version works. +PPL_LINT_SURFACE=compiled-simplified \ +PPL_LINT_CONTRACT_DIR= \ +PPL_LINT_TARGET_MANIFEST=/target.json \ +PPL_LINT_SCHEDULE=nightly \ +PPL_LINT_REPORT=/detector-report.json \ +node -r ./src/setup_node_env /scripts/ppl-lint/run-frontend-contract.mjs +``` This workflow is **non-enforcing for now**: it reports and uploads, while the required check stays the single-version `validation-result`. Promoting it needs a diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 83bb5bd39e1..18026bb095c 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -148,6 +148,9 @@ function loadLeg({ version, dir }) { label: version, dir, grammarHash: target.grammarHash || '', + // Which of OSD's two lint surfaces this leg validated. Older detector reports + // predate the field; they were all runtime-bundle runs. + surface: detector.surface || 'runtime-bundle', parserRuleNames: bundle && Array.isArray(bundle.parserRuleNames) ? bundle.parserRuleNames : undefined, detector, backend, @@ -386,6 +389,10 @@ function main() { // its engine verdicts or its detector rows). Tracked separately from drift // because the answer is "re-run / fix the leg", not "edit the linter". const inconclusive = []; + // Cases a leg's grammar surface cannot express (a `runtimeOnly` rule on a + // compiled-simplified leg). Recorded so the report can say WHY a cell is blank, + // but never a failure: the rule is inert there by design. + const notApplicable = []; const matrix = []; // one row per rule × version, for the summary table // A rule that ships enabled at error severity but has no contract file is @@ -420,7 +427,7 @@ function main() { }); if (grammarDrift) { drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); - matrix.push({ ruleId, version: leg.version, status: 'drift', drifts: 1 }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'drift', drifts: 1 }); continue; } } @@ -443,6 +450,7 @@ function main() { matrix.push({ ruleId, version: leg.version, + leg: leg.label, status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', drifts: outOfScopeDrifts.length, }); @@ -450,7 +458,7 @@ function main() { } // In scope on this engine but nothing pins its behavior there. coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); - matrix.push({ ruleId, version: leg.version, status: 'uncovered', drifts: 0 }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'uncovered', drifts: 0 }); continue; } @@ -464,6 +472,10 @@ function main() { // otherwise render `agree` off the control alone. let triggersExpected = 0; let triggersCompared = 0; + // Cases this leg's surface cannot express. Counted separately from `unusable` + // because the two need opposite advice: not-applicable is expected and needs + // no action, unusable means something did not answer and needs a re-run. + let ruleNotApplicable = 0; const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; @@ -483,6 +495,23 @@ function main() { const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName ); + // A case the surface cannot express at all (a `runtimeOnly` rule on a + // compiled-simplified leg) is excluded rather than compared. Its zero + // diagnostics are `lint_runner` deliberately skipping the rule, so + // comparing them against a non-zero expectation would classify a healthy + // rule as detector-silent and send someone to fix it. This is NOT the same + // as `inconclusive`: nothing went wrong, and there is nothing to re-run. + if (detectorResult && detectorResult.notApplicable) { + notApplicable.push({ + ruleId, + version: leg.version, + queryName, + surface: leg.surface, + reason: detectorResult.notApplicable, + }); + ruleNotApplicable++; + continue; + } const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { @@ -541,7 +570,19 @@ function main() { // that lost every TRIGGER while keeping a control, since the triggers are // where the rule's behavior actually lives. Calling either agreement is the // vacuous pass this check exists to prevent. - if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { + // A rule the surface cannot express at all is `n/a`, not `inconclusive`: + // nothing failed and there is nothing to re-run, so it must not fail the run. + // Checked BEFORE the inconclusive test, which would otherwise catch it + // (compared === 0) and demand a re-run that could never change the outcome. + if (compared === 0 && ruleNotApplicable > 0) { + matrix.push({ + ruleId, + version: leg.version, + leg: leg.label, + status: 'not-applicable', + drifts: 0, + }); + } else if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { inconclusive.push({ ruleId, file, @@ -549,7 +590,7 @@ function main() { enforced: isEnforced, reasons: unusable, }); - matrix.push({ ruleId, version: leg.version, status: 'inconclusive', drifts: ruleDrifts }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'inconclusive', drifts: ruleDrifts }); } else { if (unusable.length > 0) { log( @@ -560,6 +601,7 @@ function main() { matrix.push({ ruleId, version: leg.version, + leg: leg.label, status: ruleDrifts === 0 ? 'agree' : 'drift', drifts: ruleDrifts, }); @@ -577,6 +619,7 @@ function main() { label: l.label, engineVersion: l.version, grammarHash: l.grammarHash, + surface: l.surface, })), enforcedRules: [...enforcedRules].sort(), missingContracts, @@ -585,6 +628,7 @@ function main() { drifts, coverageHoles, inconclusive, + notApplicable, result: { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, @@ -662,25 +706,42 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); } lines.push( - `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + - `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` + // Name the surface when a leg is not the default runtime-bundle one, so a + // reader knows a column speaks for OSD's compiled grammar rather than the + // engine's exported one — the two do not run the same set of rules. + `Engine versions: ${legs + .map((l) => + l.surface && l.surface !== 'runtime-bundle' ? `\`${l.version}\` (${l.surface})` : `\`${l.version}\`` + ) + .join(', ')} — ` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); - const versions = legs.map((l) => l.version); + // Columns are keyed on the LEG LABEL, not the engine version: two legs can share + // a version while validating different surfaces (a 3.7 runtime-bundle leg and a + // 3.7 compiled leg), and keying on version alone made them collide so one leg's + // results silently rendered in place of the other's. + const columns = legs.map((l) => ({ + label: l.label, + heading: + l.surface && l.surface !== 'runtime-bundle' + ? `\`${l.version}\`
${l.surface}` + : `\`${l.version}\``, + })); const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); - lines.push(`| Rule | ${versions.map((v) => `\`${v}\``).join(' | ')} |`); - lines.push(`| ---- | ${versions.map(() => '----').join(' | ')} |`); + lines.push(`| Rule | ${columns.map((c) => c.heading).join(' | ')} |`); + lines.push(`| ---- | ${columns.map(() => '----').join(' | ')} |`); const cell = { agree: 'agree', drift: 'DRIFT', uncovered: 'not covered', 'out-of-scope': 'n/a (out of scope)', + 'not-applicable': 'n/a (surface)', inconclusive: '**inconclusive**', }; for (const ruleId of rules) { - const cells = versions.map((version) => { - const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); + const cells = columns.map((column) => { + const row = report.matrix.find((m) => m.ruleId === ruleId && m.leg === column.label); if (!row) return '—'; if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; // An unmapped status must still render as something visible. A blank cell diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index e45b4a48b04..905a44e41ae 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -45,6 +45,33 @@ * — a trigger the detector flags is one the backend rejected; a control the * detector passes is one the backend accepted (design §3.2, §4.3). * 4. Coverage (nightly only): every enabled catalog rule has a contract file. + * + * ## Two grammar surfaces + * + * OSD ships lint on TWO surfaces, and a user gets whichever one their session + * resolves to: + * + * runtime-bundle the candidate grammar the engine exported. Requires + * `GET /_plugins/_ppl/_grammar`, which landed in 3.6. + * compiled-simplified OSD's own checked-in grammar, used whenever the runtime + * bundle is unavailable — no dataset selected, an engine + * below 3.6, or a bundle that has not loaded yet. This is + * `lintRuntimePPLQuery`'s fallback path, and it runs + * detector logic the runtime path does not (see + * field_validation's text-side pass). + * + * `PPL_LINT_SURFACE` selects which one this run validates; it defaults to + * `runtime-bundle`, so the required check is unchanged. The compiled surface is + * an EXPLICIT opt-in, never a silent fallback: the whole point of the required + * check is that a missing bundle is a hard failure rather than a quiet + * downgrade to OSD's own grammar (which would validate the wrong thing). + * + * The compiled surface is what makes pre-3.6 engine legs meaningful. It also + * carries a mandatory caveat: `runtimeOnly` rules (multisearch/union/replace + * arity) are SKIPPED on it by `lint_runner` because the productions they walk do + * not exist in the compiled grammar. A compiled leg therefore reports them as + * `not-applicable` rather than as zero diagnostics, so the aggregator cannot + * mistake a deliberately-inert rule for a detector that regressed. */ import fs from 'fs'; @@ -54,11 +81,39 @@ import { createRequire } from 'module'; // OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved // against the OSD checkout root, not this script's SQL-repo location. const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +// The COMPILED-simplified surface: OSD's own checked-in grammar, used when the +// engine cannot export a runtime bundle. See `PPL_LINT_SURFACE` below. +const ANALYZER_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; // The Monaco-free engine barrel (@osd/monaco/ppl-lint) exposes the catalog; the // detector registry is a deep import used only for the wiring registration check. const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; +// Source-path fallback for the catalog. The `ppl-lint` subpath is a built export +// that only exists on checkouts that ship it; the compiled surface deliberately +// supports older checkouts (that is the coverage it adds), so fall back to the +// source module, which `setup_node_env` transpiles on require anyway. +const CATALOG_SOURCE_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; +/** + * Which grammar surface this run validates. Defaults to `runtime-bundle` so the + * required check's behavior is unchanged; `compiled-simplified` is an explicit + * opt-in for legs whose engine cannot export a bundle. + */ +const SURFACE = (() => { + const requested = process.env.PPL_LINT_SURFACE || 'runtime-bundle'; + if (requested !== 'runtime-bundle' && requested !== 'compiled-simplified') { + // A typo must not silently select the default: that would report compiled + // results under a runtime-bundle label, or vice versa. + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_SURFACE must be "runtime-bundle" or ` + + `"compiled-simplified", got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + function log(message) { // eslint-disable-next-line no-console console.log(`[ppl-lint-detector-contract] ${message}`); @@ -143,10 +198,41 @@ function loadOsd() { } }; - const headless = resolveOsd(HEADLESS_MODULE); - const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + // Prefer the built subpath (what the required check uses); fall back to source + // so a checkout without the built export can still run the compiled surface. + const catalogModule = + resolveOsd(CATALOG_MODULE, { optional: true }) || resolveOsd(CATALOG_SOURCE_MODULE); + const { getBundledCatalog } = catalogModule; const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); + if (typeof getBundledCatalog !== 'function') { + fatal(`getBundledCatalog not found in ${CATALOG_MODULE} or ${CATALOG_SOURCE_MODULE}.`); + } + const getDetector = registry && registry.getDetector; + + // On the compiled surface the headless bundle API is not needed at all, and + // requiring it would make this mode unusable on an OSD checkout that predates + // it — precisely the older-version coverage the mode exists to provide. + if (SURFACE === 'compiled-simplified') { + const { PPLLanguageAnalyzer } = resolveOsd(ANALYZER_MODULE); + if (typeof PPLLanguageAnalyzer !== 'function') { + fatal(`PPLLanguageAnalyzer not found in ${ANALYZER_MODULE}.`); + } + const analyzer = new PPLLanguageAnalyzer(); + return { + surface: SURFACE, + // Same (query, grammar, context) shape as the bundle path so the main loop + // does not branch per surface; `grammar` is unused here. + lintQuery: (query, _grammar, context) => { + const analysis = analyzer.analyzeLint(query, context); + return (analysis && analysis.result) || { diagnostics: [] }; + }, + getBundledCatalog, + getDetector, + osdRoot, + }; + } + const headless = resolveOsd(HEADLESS_MODULE); const { deserializeBundleOrThrow, lintQueryWithBundle } = headless; if (typeof deserializeBundleOrThrow !== 'function' || typeof lintQueryWithBundle !== 'function') { fatal( @@ -155,12 +241,15 @@ function loadOsd() { `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` ); } - if (typeof getBundledCatalog !== 'function') { - fatal(`getBundledCatalog not found in ${CATALOG_MODULE}.`); - } - const getDetector = registry && registry.getDetector; - return { deserializeBundleOrThrow, lintQueryWithBundle, getBundledCatalog, getDetector, osdRoot }; + return { + surface: SURFACE, + deserializeBundleOrThrow, + lintQuery: lintQueryWithBundle, + getBundledCatalog, + getDetector, + osdRoot, + }; } /** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ @@ -403,10 +492,13 @@ function main() { const reportPath = process.env.PPL_LINT_REPORT; const osd = loadOsd(); - const { getBundledCatalog, getDetector, lintQueryWithBundle, osdRoot } = osd; + const { getBundledCatalog, getDetector, lintQuery, osdRoot, surface } = osd; const catalog = getBundledCatalog(); - const grammar = loadCandidateGrammar(osd); + // The compiled surface lints with OSD's own checked-in grammar, so there is no + // candidate bundle to load. On the runtime surface a missing bundle stays a hard + // failure — never a quiet downgrade to the compiled grammar. + const grammar = surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd); const target = loadTarget(); const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; const backendReport = loadBackendReport(); @@ -417,6 +509,11 @@ function main() { osdRoot, schedule, engineVersion, + // Which of OSD's two lint surfaces produced these results. The aggregator + // needs this to interpret them: a compiled leg legitimately has no verdict for + // `runtimeOnly` rules, and mixing the two surfaces under one label would + // report a deliberately-inert rule as a regression. + surface, grammarHash: target.grammarHash || '', differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD @@ -474,7 +571,29 @@ function main() { const expected = expectedQueries[queryName]; const expectedCount = expected.detectorCount; - const result = lintQueryWithBundle(query, grammar, context); + // A `runtimeOnly` rule walks grammar productions that exist only in the + // runtime bundle, so `lint_runner` skips it on the compiled surface. Its + // zero diagnostics here mean "deliberately inert", NOT "the detector went + // silent" — reporting them as a count would make the aggregator classify a + // healthy rule as detector-silent drift and send someone to fix it. Mark the + // case not-applicable and let the aggregator exclude it. + if (surface === 'compiled-simplified' && entry && entry.runtimeOnly) { + log( + ` SKIP ${ruleId}/${queryName} (${role}): runtimeOnly rule is inert on the ` + + `compiled-simplified surface.` + ); + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + + const result = lintQuery(query, grammar, context); const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); const actual = matches.length; const ok = actual === expectedCount; From e0c84212b48fccf98cf093f2f01593f43cf2e176 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 21:07:08 -0700 Subject: [PATCH 43/78] feat(ci): add compiled-surface legs for engines below the grammar floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the multi-version matrix down to 2.19 by validating the surface that actually reaches those users. Engines below 3.6 cannot export a grammar bundle, so the runtime-bundle surface cannot reach them at all. The compiled-simplified surface has no such floor — it is what a user gets whenever no bundle is available, which on a pre-3.6 cluster is always. Those legs still run the real contract queries against the real engine, so the backend half of the differential is genuine. Workflow: a nightly observe-compiled matrix (2.19.0 / 3.0.0 / 3.5.0, overridable via compiled_versions, [] to skip; pull_request skips by default since three more engine images is too slow per PR). Its observe job omits -Dppl.lint.grammar.bundle, so the IT exports nothing, and writes a marker — which is what distinguishes 'this engine has no _grammar endpoint' from 'the bundle export failed', a distinction that must stay a hard error. Contracts now declare the surface(s) they were verified against and are only scored on a matching leg. This turns grammarSurface from a decorative field into a real guard: judged on a surface it never claimed, a contract produces confident nonsense — a runtime-bundle contract on a compiled leg yielded both version-scope-too-narrow and a coverage hole, each about a surface it does not describe. Four cross-surface contracts are marked 'both' after verifying their grammar is byte-identical 2.19 through 3.7 (headCommand, dedupCommand+CONSECUTIVE, joinType's full alternative list, evalCommand). field-validation and unsupported-window-function-in-eventstats were also marked 'both': their previous 'compiled-simplified' described where the rule CAN run, not a restriction, and honoring it as exclusive would have dropped them from every runtime leg including the required check's. Live-verified on the real 3.7 runtime leg that both still fire exactly as pinned. field-validation gains a <3.4.0 expectation, since its empty appliesTo ships it to users on every engine. Detector behavior is live-verified identical at 2.19.0, 3.0.0, 3.5.0 and 3.7.0. Its backend oracle deliberately omits error.type/reason — that wording has not been observed live on those engines, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. assertRejection now tolerates an omitted error block so 'it rejects' can be asserted without claiming to know how. Found and fixed while wiring this up: division-by-zero pinned modulo-by-zero-flagged at detectorCount 1, but the detector deliberately handles only '/' (division_by_zero.ts: 'Modulo-by-zero was not verified live'). The contract asserted behavior the rule never had; nightly-only with no compiled leg meant it had never run. It is now a control documenting that boundary. Verified against a 4-leg mixed-surface matrix built from the real leg artifacts: PASS with 2.19 correctly reporting 5 rules n/a (surface), eventstats n/a (out of scope), and 4 rules genuinely validated. 5 new surface scenarios, 16 existing drift scenarios and 59 unit tests all green. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 176 +++++++++++++++++- .../remote/PplLintRuleValidationIT.java | 17 +- .../dedup-consecutive-unsupported.spec.json | 2 +- .../contracts/disabled-join-type.spec.json | 2 +- .../contracts/division-by-zero.spec.json | 12 +- .../contracts/field-validation.spec.json | 40 +++- .../contracts/head-without-sort.spec.json | 2 +- ...ed-window-function-in-eventstats.spec.json | 2 +- scripts/ppl-lint/README.md | 15 ++ scripts/ppl-lint/aggregate-versions.mjs | 27 +++ scripts/ppl-lint/run-frontend-contract.mjs | 35 ++++ 11 files changed, 308 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index e9ee6849111..f27079ba1f2 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -72,14 +72,32 @@ on: description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' required: false type: string + compiled_versions: + description: 'JSON array of engine versions to validate on the compiled-simplified surface, e.g. ["2.19.0"]. Use "[]" to skip them.' + required: false + type: string permissions: contents: read env: - # Released engine versions to validate, newest last. Each must be >= 3.6.0 (the - # _grammar endpoint floor) and must have a published distribution image. + # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must + # be >= 3.6.0 (the _grammar endpoint floor) and must have a published + # distribution image. ENGINE_VERSIONS: '["3.6.0","3.7.0"]' + # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. + # + # These engines cannot export a grammar bundle (GET /_plugins/_ppl/_grammar + # landed in 3.6), so the runtime surface cannot reach them at all. But the + # compiled surface has no such floor: it lints with OSD's own checked-in grammar, + # which is exactly what a user gets when no bundle is available — including every + # user on an engine below 3.6. Those legs still run the real contract queries + # against the real engine, so the backend half of the differential is genuine. + # + # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are + # reported not-applicable. Nightly only — see the `compiled_versions` input to + # run one ad hoc. + COMPILED_ENGINE_VERSIONS: '["2.19.0","3.0.0","3.5.0"]' jobs: # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a @@ -96,6 +114,7 @@ jobs: runs-on: ubuntu-latest outputs: released: ${{ steps.plan.outputs.released }} + compiled: ${{ steps.plan.outputs.compiled }} osd_repo: ${{ steps.plan.outputs.osd_repo }} osd_ref: ${{ steps.plan.outputs.osd_ref }} steps: @@ -104,6 +123,9 @@ jobs: env: REQUESTED_VERSIONS: ${{ inputs.engine_versions }} DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} + REQUESTED_COMPILED: ${{ inputs.compiled_versions }} + DEFAULT_COMPILED: ${{ env.COMPILED_ENGINE_VERSIONS }} + EVENT_NAME: ${{ github.event_name }} REQUESTED_REPO: ${{ inputs.osd_repo }} REQUESTED_REF: ${{ inputs.osd_ref }} VAR_REPO: ${{ vars.OSD_REPO }} @@ -121,6 +143,28 @@ jobs: assert isinstance(item,str), 'engine_versions entries must be strings' " echo "released=$released" >> "$GITHUB_OUTPUT" + + # Compiled-surface legs add three more engine images, so they run on the + # nightly schedule (and on an explicit dispatch), not on every PR that + # touches the corpus. An explicit input always wins, including "[]". + if [ -n "${REQUESTED_COMPILED:-}" ]; then + compiled="$REQUESTED_COMPILED" + elif [ "$EVENT_NAME" = "pull_request" ]; then + compiled='[]' + else + compiled="$DEFAULT_COMPILED" + fi + # An EMPTY list is legitimate here (unlike engine_versions): it means "skip + # the compiled surface this run". Still reject a non-list. + echo "$compiled" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list), 'compiled_versions must be a JSON array' + for item in v: + assert isinstance(item,str), 'compiled_versions entries must be strings' + " + echo "compiled=$compiled" >> "$GITHUB_OUTPUT" + echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" # Same precedence as the sibling workflow: dispatch input, then repo # variable, then the canonical upstream default. echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" @@ -223,6 +267,103 @@ jobs: name: ppl-lint-leg-${{ matrix.version }}-logs path: integ-test/build/reports/** + # Legs for engines BELOW the _grammar endpoint floor (3.6). These cannot export a + # grammar bundle, so they are observed for backend behavior only and their + # detector pass runs on the compiled-simplified surface — which is what a real + # user on such an engine gets, since no bundle can ever load there. + # + # Identical to observe-released except that `-Dppl.lint.grammar.bundle` is + # omitted: the IT skips the export when that property is unset, so no bundle + # fetch is attempted against an engine that has no such endpoint. + observe-compiled: + name: Observe engine ${{ matrix.version }} (compiled surface) + needs: plan + # An empty compiled list means "skip this surface" (the pull_request default). + if: ${{ needs.plan.outputs.compiled != '[]' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.compiled) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + # No -Dppl.lint.grammar.bundle: this engine predates the _grammar endpoint, + # and the IT correctly exports nothing when the property is unset. + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + # Mark the leg so the detect job knows to lint it on the compiled surface. + # A leg with no bundle would otherwise look like a failed export. + echo 'compiled-simplified' > leg/surface + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled-logs + path: integ-test/build/reports/** + # The PR's own engine build, so the newest point in the matrix is the code under # review rather than the last release. Same oracle as the released legs; the only # difference is a Gradle-managed cluster instead of a published image, which is @@ -294,6 +435,7 @@ jobs: needs: - plan - observe-released + - observe-compiled - observe-pr-build if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest @@ -369,13 +511,25 @@ jobs: exit 1 fi for leg in "${legs[@]}"; do - # Skip the log-only artifacts an observation failure may have uploaded. - [ -f "$leg/ppl-grammar-bundle.json" ] || { echo "skipping $leg (no grammar bundle)"; continue; } version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - echo "=== detectors vs engine $version ===" + # A leg is compiled-surface when its observe job said so. That marker is + # what distinguishes "this engine has no _grammar endpoint" from "the + # bundle export failed", which must stay a hard error. + if [ -f "$leg/surface" ] && [ "$(cat "$leg/surface")" = 'compiled-simplified' ]; then + surface_env=(PPL_LINT_SURFACE=compiled-simplified) + echo "=== detectors vs engine $version (compiled-simplified surface) ===" + elif [ -f "$leg/ppl-grammar-bundle.json" ]; then + surface_env=(PPL_LINT_SURFACE=runtime-bundle + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json") + echo "=== detectors vs engine $version (runtime-bundle surface) ===" + else + # Skip the log-only artifacts an observation failure may have uploaded. + echo "skipping $leg (no grammar bundle and no compiled-surface marker)" + continue + fi + env "${surface_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ PPL_LINT_REPORT="$leg/detector-report.json" \ node -r ./src/setup_node_env \ @@ -399,6 +553,7 @@ jobs: id: aggregate env: RELEASED: ${{ needs.plan.outputs.released }} + COMPILED: ${{ needs.plan.outputs.compiled }} run: | set -euo pipefail shopt -s nullglob @@ -419,7 +574,14 @@ jobs: # a matrix that silently lost one — the exact vacuous pass this workflow # exists to prevent. A dead leg is a failure, not a smaller matrix. missing=() - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do + # Compiled legs are labelled "-compiled" to match their artifact + # name, so they occupy their own column even when a runtime leg validated + # the same engine version. + compiled_wanted=$(echo "$COMPILED" | python3 -c " + import json,sys + print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) + ") + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build; do found=no for have in "${present[@]}"; do [ "$have" = "$want" ] && found=yes && break diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 7ae505e9b35..0a0b686cc44 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -419,12 +419,21 @@ private void assertRejection( expectedBody.getInt("status"), obs.body.getInt("status")); + // A contract may omit `error` entirely to assert only THAT the engine rejects, + // without pinning wording that has not been observed live on that version. That + // is weaker than a full oracle but honest; inventing a type/reason would either + // fail spuriously or get "fixed" by pinning whatever CI first happened to see. + if (!expectedBody.has("error")) { + return; + } JSONObject expectedError = expectedBody.getJSONObject("error"); JSONObject actualError = obs.body.getJSONObject("error"); - assertEquals( - "case \"" + queryName + "\": unexpected error.type for query: " + query, - expectedError.getString("type"), - actualError.getString("type")); + if (expectedError.has("type")) { + assertEquals( + "case \"" + queryName + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + } if (expectedError.has("reason")) { assertEquals( "case \"" + queryName + "\": unexpected error.reason for query: " + query, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index ca414f6f103..90a614307b7 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "dedup-consecutive-unsupported", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "dedup-consecutive-unsupported", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index bd0769934ea..25cd3232e88 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "disabled-join-type", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "disabled-join-type", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index d9071cc4978..5e1b33884c8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,8 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "grammarSurface": "compiled-simplified", + "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL — it documents that boundary rather than asserting a gap.", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "division-by-zero", @@ -29,8 +30,8 @@ "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" }, - "modulo-by-zero-flagged": { - "role": "trigger", + "modulo-by-zero-not-flagged": { + "role": "control", "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" } }, @@ -47,9 +48,8 @@ "detectorCount": 0, "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } }, - "modulo-by-zero-flagged": { - "detectorCount": 1, - "severity": "warning", + "modulo-by-zero-not-flagged": { + "detectorCount": 0, "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 45f85e3b769..984fcf851c1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "field-validation", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "field-validation", @@ -56,6 +56,44 @@ } }, "expectations": [ + { + "version": "<3.4.0", + "note": "This rule has an empty appliesTo, so it ships to users on EVERY engine, including pre-3.4. Detector behavior is live-verified identical from 2.19 up (1/1/0 on the compiled surface at 2.19.0, 3.0.0, 3.5.0, 3.7.0). The backend oracle deliberately omits error.type/reason: this engine's wording for an unknown field has not been observed live, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. A compiled-surface leg records the real wording; pin it then.", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, { "version": ">=3.4.0 <3.7.0", "queries": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index e86a19c1002..695f1b6b550 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "head-without-sort", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "head-without-sort", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index d6821273175..4f436a383cb 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "unsupported-window-function-in-eventstats", "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "pr", "wiring": { "detector": "unsupported-window-function-in-eventstats", diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 9dcc2c8d454..f20633768d4 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -250,6 +250,21 @@ is nothing to re-run). Two legs may share an engine version while validating different surfaces, so the matrix is keyed on the **leg label**, not the version. +Each contract declares the surface(s) it was verified against, and a contract is +only scored on a matching leg — `"both"` opts into either. Judged on a surface it +never claimed, every verdict is meaningless: a runtime-bundle contract on a +compiled leg yields both `version-scope-too-narrow` ("the engine rejects but the +rule is scoped away") and a coverage hole, each about a surface the contract does +not describe. Contracts declaring `"both"` are what a pre-3.6 leg can actually +validate; the rest report `n/a (surface)`. + +**Compiled-surface legs run nightly** (`COMPILED_ENGINE_VERSIONS`, default +`2.19.0` / `3.0.0` / `3.5.0`) — three more engine images is too slow for every PR. +Dispatch with `compiled_versions` to run one ad hoc, or `[]` to skip. Their observe +job omits `-Dppl.lint.grammar.bundle` (the IT then exports nothing) and writes a +`surface` marker file, which is what tells the detect job to lint them on the +compiled surface rather than treating a missing bundle as a failed export. + ```bash # A compiled-surface leg: no grammar bundle needed, so any engine version works. PPL_LINT_SURFACE=compiled-simplified \ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 18026bb095c..a0ab0f69613 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -411,6 +411,33 @@ function main() { const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; for (const leg of legs) { + // A contract only speaks for the surface(s) it declares. Judge it on any + // other leg and every verdict is meaningless: a runtime-bundle contract on a + // compiled leg yields "the engine rejects but the rule is scoped away" + // (version-scope-too-narrow) and "no expectation covers this engine" + // (coverage hole) — both about a surface the contract never claimed to + // describe. Checked FIRST, before scope, grammar and coverage, because all + // three of those produce confident findings from an irrelevant comparison. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + const legSurface = leg.surface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== legSurface) { + notApplicable.push({ + ruleId, + version: leg.version, + leg: leg.label, + surface: legSurface, + reason: `contract declares grammarSurface "${contractSurface}"`, + }); + matrix.push({ + ruleId, + version: leg.version, + leg: leg.label, + status: 'not-applicable', + drifts: 0, + }); + continue; + } + const inScope = versionInAppliesTo(appliesTo, leg.version); // A parser rule that vanished from the grammar is one fact about this diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 905a44e41ae..6f7ecd6de50 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -505,6 +505,9 @@ function main() { const contracts = loadContracts(); const failures = []; + // Contracts this surface did not score, recorded so the report says a rule was + // skipped for surface rather than leaving its absence unexplained. + const skippedForSurface = []; const report = { osdRoot, schedule, @@ -514,6 +517,9 @@ function main() { // `runtimeOnly` rules, and mixing the two surfaces under one label would // report a deliberately-inert rule as a regression. surface, + // Contracts whose declared `grammarSurface` excludes this run, so a reader can + // see WHY a rule has no scored cases here. + skippedForSurface, grammarHash: target.grammarHash || '', differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD @@ -547,6 +553,35 @@ function main() { continue; } + // A contract declares the surface its expectations were verified against. + // Until now that field was decorative; honoring it keeps a runtime-bundle + // contract from being scored on a compiled leg, where its rule may legitimately + // behave differently. `both` opts into being checked on either surface. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + log( + `SKIP ${ruleId} (grammarSurface=${contractSurface}, running ${surface}) — ` + + `${path.basename(file)}` + ); + skippedForSurface.push({ ruleId, contractSurface }); + // Emit an explicit not-applicable row per query rather than dropping the rule. + // Dropping it leaves the aggregator with no rows at all, which it correctly + // reads as `inconclusive` — "we could not check" — and fails on. But nothing + // went wrong here and there is nothing to re-run: this contract simply does + // not describe this surface. + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + report.results.push({ + ruleId, + queryName, + role: queryDef.role || 'trigger', + query: (queryDef.query || '').split('{{index}}').join(index), + surface, + notApplicable: `contract declares grammarSurface "${contractSurface}"`, + }); + } + continue; + } + const entry = checkWiring(spec, catalog, getDetector, failures); if (!entry) { continue; From 79bb36b63db7f6123d81fea072fc8cea309d25d2 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 21:36:00 -0700 Subject: [PATCH 44/78] fix(ci): stop the index wipe racing plugin initialization The 2.19 observation leg failed with a 60s SocketTimeoutException before any contract query ran. Root cause, from the engine's own logs: 04:16:04 MetadataCreateIndexService [.plugins-ml-config] creating index 04:16:05 MLSyncUpCron ML configuration initialized ~04:16:38 test client DELETE /.plugins-ml-config -> blocks 04:17:38 test SocketTimeoutException: 60000 MILLISECONDS The IT's first act is cleanUpIndices -> wipeAllOpenSearchIndices, which lists every index including hidden ones and DELETEs anything not matching .opensearch / .opendistro / .ql. The index ".plugins-ml-config" matches none of those, so it gets deleted -- and on an engine still initializing it, that request never completes. Why only 2.19: the cluster-health API reports GREEN before bundled plugins finish creating their system indices. The 3.7 container settled ~11s sooner, so its wipe landed 154s after ML init finished and succeeded. Same code, same args, different timing -- the 3.7 leg passed by luck, not by design. Two fixes, because either alone leaves a window: - Never delete indices under the ".plugins-" prefix. They belong to bundled plugins, wiping them is never the point of a test, and doing it during initialization hangs the suite. - Wait for the index set to stop changing before starting the IT (three identical consecutive listings), so a slow engine cannot race the wipe at all. Warns rather than fails if it does not settle: a slow-but-working engine should still be observed, and a genuinely unreachable one fails loudly in the next step. Also corrects an earlier claim of mine: the Gradle-provisioned cluster visible in that leg's log (build/testclusters/integTestRemote-0) started 7 seconds AFTER the failure was reported, so it was a consequence, not the cause. The released legs do reach their own engines -- their target.json files report three distinct versions and three distinct grammar hashes. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 34 +++++++++++++++++++ .../sql/legacy/OpenSearchSQLRestTestCase.java | 12 ++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f27079ba1f2..4d3e9797bc3 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -323,6 +323,40 @@ jobs: esac curl -sf http://localhost:9200/_cat/plugins | grep -i sql + # `_cluster/health` goes GREEN before the bundled plugins finish creating + # their system indices, and the IT's first act is to wipe every non-system + # index. On 2.19 that DELETE landed while ML Commons was still initializing + # `.plugins-ml-config` and blocked until the client's 60s socket timeout, + # failing the leg before a single contract query ran. Wait for the plugin + # indices to stop appearing, so the wipe cannot race initialization. + - name: Wait for bundled plugin system indices to settle + run: | + set -euo pipefail + previous="" + stable=0 + for i in $(seq 1 30); do + current=$(curl -sf "http://localhost:9200/_cat/indices?h=index&expand_wildcards=all" \ + | sort | tr '\n' ',' || true) + if [ -n "$current" ] && [ "$current" = "$previous" ]; then + stable=$((stable + 1)) + # Three consecutive identical listings: no plugin is still creating + # indices. One match is not enough — initialization has gaps between + # an index being created and the next one starting. + if [ "$stable" -ge 3 ]; then + echo "index set stable after ${i} poll(s): $current" + exit 0 + fi + else + stable=0 + fi + previous="$current" + sleep 2 + done + # Not fatal: a slow-but-working engine should still be observed. The IT + # tolerates a wipe failure per index, and a genuinely unreachable cluster + # fails loudly in the next step anyway. + echo "::warning::plugin index set did not stabilize; continuing" + - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 91584fb45cf..267adea43c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -213,9 +213,19 @@ protected static void wipeAllOpenSearchIndices(RestClient client) throws IOExcep String indexName = jsonObject.getString("index"); try { // System index, mostly named .opensearch-xxx or .opendistro-xxx, are not allowed to - // delete + // delete. + // + // `.plugins-` covers the system indices of bundled plugins (ML Commons' + // `.plugins-ml-config`, and friends). Deleting those is never the point of a + // test wipe, and it is actively harmful: on an engine whose plugins are + // still initializing, the DELETE blocks until the client's socket timeout + // and fails the suite before any test runs. That is what broke the 2.19 + // observation leg of the PPL lint multi-version matrix — `_cluster/health` + // reports GREEN before ML Commons finishes creating its config index, so + // the wipe raced initialization. if (!indexName.startsWith(".opensearch") && !indexName.startsWith(".opendistro") + && !indexName.startsWith(".plugins-") && !indexName.startsWith(".ql")) { client.performRequest(new Request("DELETE", "/" + indexName)); } From 1dbeea4078dd45a94c10bed2209b346a27727d61 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 08:37:54 -0700 Subject: [PATCH 45/78] test(ci): probe the framework's startup requests on a compiled leg The 2.19 leg still times out after the settle fix, and a bare SocketTimeoutException does not say WHICH request hung. The stack points at OpenSearchRestTestCase.initClient line 216, which is GET _nodes/plugins -- a node-level API, unlike the _cat/plugins call the health step already makes successfully. Time each of the framework's startup requests explicitly so the next run identifies the culprit instead of inviting another guess. Diagnostic only; warns rather than fails. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 4d3e9797bc3..d0d18eca2fb 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -357,6 +357,24 @@ jobs: # fails loudly in the next step anyway. echo "::warning::plugin index set did not stabilize; continuing" + # Probe the EXACT requests the test framework makes before any test runs. + # `OpenSearchRestTestCase.initClient` issues `GET _nodes/plugins`, and + # `wipeAllOpenSearchIndices` issues `GET _cat/indices?expand_wildcards=all`. + # A leg that dies with a bare socket timeout gives no clue which of those + # hung, so time them here where the output is readable. + - name: Probe the framework's own startup requests + run: | + set -uo pipefail + for path in "_nodes/plugins" "_cat/indices?format=json&expand_wildcards=all" "_cluster/health"; do + start=$(date +%s) + if curl -sS --max-time 30 -o /tmp/probe.out -w '%{http_code}' \ + "http://localhost:9200/${path}" > /tmp/probe.code 2>/tmp/probe.err; then + echo "OK $(($(date +%s) - start))s HTTP $(cat /tmp/probe.code) ${path} ($(wc -c < /tmp/probe.out) bytes)" + else + echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" + fi + done + - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: From 5780fda75453282a220a46431a4059eae82a45b7 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 08:51:35 -0700 Subject: [PATCH 46/78] test(ci): probe HTTP/2 negotiation on a pre-3.x engine The first probe showed all three of the framework's startup requests returning HTTP 200 in 0s over curl, while the test JVM still times out on _nodes/plugins. So the endpoint works and the address is right; the remaining difference is how the client connects. RestClientBuilder.createHttpClient builds an HttpAsyncClient with no version policy, which in HttpClient 5.x negotiates h2-with-upgrade. curl defaults to HTTP/1.1, which is why it succeeds. Probe an explicit h2 upgrade to confirm or refute that a 2.19 node completes it before changing any client code. Diagnostic only. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-multiversion-validation.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index d0d18eca2fb..dd89d02cc12 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -374,6 +374,19 @@ jobs: echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" fi done + # The requests above all succeed over HTTP/1.1 (curl's default) yet the + # test client still times out on the same endpoint. The remaining + # difference is protocol negotiation: RestClientBuilder builds an + # HttpAsyncClient with no version policy, which in HttpClient 5.x means + # h2-with-upgrade. Probe an explicit h2 upgrade to see whether this engine + # completes it. + start=$(date +%s) + if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ + "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then + echo "h2 probe: negotiated HTTP/$(cat /tmp/h2.out) in $(($(date +%s) - start))s" + else + echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" + fi - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 From a6ccce54b20bd07c901020a171019056a3cdaa64 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:25:39 -0700 Subject: [PATCH 47/78] fix(test): pin the REST test client to HTTP/1.1 Every request from the integ-test REST client to a 2.x engine fails with SocketTimeoutException after the full 60s response timeout, thrown from AbstractSingleCoreIOReactor.execute during client construction -- before a single test runs. The 2.19 leg of the PPL lint multi-version matrix could not observe anything because of it. Root cause: RestClientBuilder.createHttpClient builds its async client with HttpAsyncClientBuilder.create() and never sets a version policy, which in HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against one that does not, the async I/O reactor stalls rather than falling back to 1.1. Isolated by probing from inside the runner, which ruled out three other explanations in turn: all three of the framework's startup requests (_nodes/plugins, _cat/indices?expand_wildcards=all, _cluster/health) returned HTTP 200 in 0s over curl, so neither the engine, the endpoint, nor the address was at fault. An explicit --http2 probe then split the two legs cleanly: engine 3.5.0 negotiated HTTP/2 leg PASSED engine 2.19.0 negotiated HTTP/1.1 leg timed out at exactly 60s curl falls back cleanly where this client does not. These tests never need h2, so asking for 1.1 up front removes the negotiation entirely and works across every supported engine line. Applied inside each existing config callback rather than as its own setHttpClientConfigCallback call: that setter replaces rather than accumulates, so a separate call would have silently dropped the credentials provider on a secured cluster and the TLS strategy on an https one -- only on the paths where those matter. Signed-off-by: Hanyu Wei --- .../sql/legacy/OpenSearchSQLRestTestCase.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 267adea43c3..0abe1f4ab7c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; @@ -28,6 +29,7 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.http2.HttpVersionPolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.LogManager; @@ -255,12 +257,39 @@ protected static void configureClient(RestClientBuilder builder, Settings settin credentialsProvider.setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password.toCharArray())); - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return forceHttp11( + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); }); + } else { + builder.setHttpClientConfigCallback(OpenSearchSQLRestTestCase::forceHttp11); } OpenSearchRestTestCase.configureClient(builder, settings); } + /** + * Pin a client to HTTP/1.1. + * + *

{@code RestClientBuilder} builds its async client with no version policy, which in + * HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against + * one that does not, the async I/O reactor stalls instead of falling back, so every request fails + * with {@code SocketTimeoutException} after the full response timeout — thrown from {@code + * AbstractSingleCoreIOReactor.execute} before a single test runs. + * + *

Live-verified on the PPL lint multi-version matrix: an {@code --http2} probe against engine + * 3.5.0 negotiated HTTP/2 and that leg PASSED, while the same probe against 2.19.0 reported + * HTTP/1.1 and the leg timed out at exactly 60s. curl falls back cleanly; this client does not. + * These tests never need h2, so asking for 1.1 up front removes the negotiation and works across + * every supported engine line. + * + *

Applied INSIDE each config callback rather than as its own {@code + * setHttpClientConfigCallback} call, because that setter replaces rather than accumulates: a + * separate call would silently drop the credentials or TLS configuration set here, and only on + * the paths where it matters. + */ + private static HttpAsyncClientBuilder forceHttp11(HttpAsyncClientBuilder httpClientBuilder) { + return httpClientBuilder.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1); + } + protected static void configureHttpsClient( RestClientBuilder builder, Settings settings, HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); @@ -292,12 +321,13 @@ protected static void configureHttpsClient( .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .build(); - return httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy(tlsStrategy) - .build()); + return forceHttp11( + httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build())); } catch (Exception e) { throw new RuntimeException(e); } From 7ea51cfb6e109bd853b1bcb548776400bdb25e95 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:29:00 -0700 Subject: [PATCH 48/78] fix(ci): tell a partial engine fix from a full one before advising version scoping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classifyDrift` runs per query, so a single trigger the engine started accepting looked identical whether the rule's other triggers still failed or not. Its remediation then advised on the whole rule: "set appliesTo.maxVersion just below this version". That advice is wrong whenever the fix was partial — scoping the rule away drops the diagnostics that are still correct, turning a partial engine fix into a shipped false negative on the shapes that remain broken. The default lean was toward version-scoping, so the tool nudged toward the regression. Adds `classifyRelaxationScope`, which decides per RULE per version: every observed trigger relaxed -> engine-relaxed, version-scope-rule some relaxed, others rejected -> engine-partially-relaxed, update-detector The aggregator now buffers per-query findings, collects each trigger's engine verdict, and emits one rule-level verdict that supersedes the per-query ones — so the report shows a single decision rather than one "scope this away" paragraph per trigger. Three details that keep it from producing confident nonsense: - A trigger with no usable verdict counts as neither relaxed nor holding. Counting it as holding would let a timed-out leg read as a partial fix and send someone to narrow a healthy detector; the advice names those triggers and says to re-run. - Only triggers whose contract pinned a rejection can relax. An advisory rule's queries are all valid PPL, so counting them would fabricate a full-fix verdict for a rule the engine never rejected. - The evidence states the tally ("2 of 3 observed trigger(s) relaxed"), and a single-trigger rule gets an explicit warning that a full-fix verdict rests on one observation — the inference the corpus size cannot yet support. Signed-off-by: Hanyu Wei --- .../__tests__/aggregate-versions.test.mjs | 127 ++++++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 94 ++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 70 ++++++++- scripts/ppl-lint/drift.mjs | 141 ++++++++++++++++++ 4 files changed, 430 insertions(+), 2 deletions(-) diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 65606dbb926..8100829eb9d 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -215,6 +215,133 @@ test('a version where only one engine relaxed is red, and names just that versio assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); }); +// --- partial vs full relaxation, end to end --------------------------------- +// +// Driven through the real script because the bug this guards is in the AGGREGATION: +// `classifyDrift` is per-query and cannot see the other triggers, so the rollup has +// to happen here or the advice is wrong whenever a rule has more than one trigger. + +/** A two-trigger contract: the shape that makes partial-vs-full decidable. */ +function writeTwoTriggerContracts() { + const dir = makeTmp('ppl-lint-contracts-multi-'); + const spec = { + ...SPEC, + queries: { + triggerA: { role: 'trigger', query: 'union [ source={{index}} ]' }, + triggerB: { role: 'trigger', query: 'union [ source={{index}} ] extra' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + triggerA: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + triggerB: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], + }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +test('ALL triggers relaxing is a full fix: one rule-level version-scope finding', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + // Exactly ONE finding, not one per trigger: the decision is per rule. + assert.equal(report.drifts.length, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + assert.deepEqual(drift.scope.relaxed.sort(), ['triggerA', 'triggerB']); + assert.deepEqual(drift.scope.holding, []); +}); + +test('SOME triggers relaxing is a partial fix: narrow the detector, do NOT scope', () => { + // The regression this pins: acting on the per-query view would advise + // maxVersion < 3.7, dropping the diagnostic for triggerB which the engine STILL + // rejects — converting a partial engine fix into a shipped false negative. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report, stdout } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + const partial = report.drifts.find((d) => d.driftClass === 'engine-partially-relaxed'); + assert.ok(partial, 'a partial relaxation must be classified as such'); + assert.equal(partial.remediation.action, 'update-detector'); + assert.deepEqual(partial.scope.relaxed, ['triggerA']); + assert.deepEqual(partial.scope.holding, ['triggerB']); + // No finding may survive that tells the engineer to version-scope this rule. + assert.equal( + report.drifts.filter((d) => d.remediation.action === 'version-scope-rule').length, + 0, + 'a partial fix must never advise version-scoping' + ); + assert.match(stdout, /Do NOT scope/); +}); + +test('an unobserved trigger does not fake a partial fix', () => { + // If the unobserved trigger were counted as "still rejects", this would classify + // as partial and send someone to narrow a detector on the strength of a leg that + // never answered. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'triggerB', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-partially-relaxed').length, + 0, + 'an unobserved trigger must not be counted as holding' + ); + const relaxed = report.drifts.find((d) => d.driftClass === 'engine-relaxed'); + assert.ok(relaxed); + assert.deepEqual(relaxed.scope.unobserved, ['triggerB']); + assert.match(relaxed.remediation.detail, /produced no verdict/); +}); + test('a rule out of scope on an older engine that accepts is not drift', () => { const legs = healthyLegs(); // 3.6 predates the rule's minVersion and accepts the query: intended silence. diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 4c8d4e6f4dc..251ecb3c9bf 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -23,6 +23,7 @@ import { DRIFT_CLASSES, REMEDIATIONS, classifyDrift, + classifyRelaxationScope, formatDriftReport, parseVersion, suggestParserRules, @@ -436,6 +437,99 @@ test('rename suggestions prefer containment then near spellings', () => { assert.deepEqual(suggestParserRules('rexCommand', ['whereClause', 'sortCommand'], 3), []); }); +// --- partial vs full relaxation ---------------------------------------------- +// +// The distinction these tests protect: a rule whose triggers ALL relaxed should be +// scoped away from the version; a rule where only SOME relaxed must NOT be, because +// scoping it would drop the diagnostics that are still correct. Getting this +// backwards converts a partial engine fix into a shipped false negative, so each +// branch is pinned including the advice text that names the wrong action. + +const scopeBase = { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + detectorFlagged: true, +}; + +test('no relaxed trigger yields no rule-level finding', () => { + assert.equal( + classifyRelaxationScope({ ...scopeBase, relaxedTriggers: [], holdingTriggers: ['a', 'b'] }), + null + ); +}); + +test('every trigger relaxed is a FULL fix and advises version scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen', 'leading-digit'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.evidence, /FULL fix, 2 of 2 observed trigger\(s\) relaxed/); +}); + +test('some triggers still rejected is a PARTIAL fix and advises the detector, NOT scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen'], + holdingTriggers: ['leading-digit', 'all-digits'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /PARTIAL fix, 1 of 3 observed trigger\(s\) relaxed/); + // The advice must say the wrong action out loud. An engineer reading only the + // action verb could still reach for maxVersion, which is the regression. + assert.match(drift.remediation.detail, /Do NOT scope .* away from 3\.8\.0/); + assert.match(drift.remediation.detail, /false NEGATIVE/); +}); + +test('a single-trigger rule warns that a FULL verdict rests on one observation', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['only-one'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.match(drift.remediation.detail, /only 1 trigger/); + assert.match(drift.remediation.detail, /confirm with more shapes/); +}); + +test('unobserved triggers are excluded from the tally and named in the advice', () => { + // The trap: counting an unobserved trigger as "holding" turns a dead leg into a + // partial fix and sends someone to narrow a healthy detector. + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a'], + holdingTriggers: [], + unobservedTriggers: ['b'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED, 'must not read as partial'); + assert.match(drift.evidence, /1 of 1 observed trigger\(s\) relaxed/); + assert.match(drift.evidence, /1 trigger\(s\) produced no verdict \(b\) and were NOT counted/); + assert.match(drift.remediation.detail, /re-run it before acting/); +}); + +test('a silent detector on a fully relaxed rule needs no linter change', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a', 'b'], + holdingTriggers: [], + detectorFlagged: false, + }); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +test('the per-query relaxed finding is marked supersedable', () => { + // The aggregator drops these in favour of the rule-level verdict; without the + // marker it would report both, and the per-query one gives the wrong action. + const drift = classifyDrift( + agreeingTrigger({ observed: { detectorCount: 1, severities: ['error'], backendRejected: false } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.supersededBy, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); +}); + // --- report ------------------------------------------------------------------ test('the report groups by action, most urgent first', () => { diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index a0ab0f69613..d7d6b25731f 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -37,6 +37,8 @@ import { emitAnnotations } from './annotate.mjs'; import { classifyDrift, classifyGrammarDrift, + classifyRelaxationScope, + DRIFT_CLASSES, formatDriftReport, versionInAppliesTo, } from './drift.mjs'; @@ -504,6 +506,19 @@ function main() { // no action, unusable means something did not answer and needs a re-run. let ruleNotApplicable = 0; const unusable = []; + // Per-trigger engine verdicts for this rule on this leg, so a relaxation can + // be judged across the WHOLE rule rather than one query at a time. A single + // relaxed trigger cannot distinguish a full engine fix (scope the rule away) + // from a partial one (narrow the detector), and those actions are opposites — + // acting on the per-query view ships a false negative in the partial case. + // `unobserved` is kept apart from `holding` on purpose: a trigger that never + // answered must not be counted as "still rejects", or a timed-out leg would + // read as a partial fix and send someone to narrow a healthy detector. + const relaxedTriggers = []; + const holdingTriggers = []; + const unobservedTriggers = []; + let relaxedDetectorFlagged = false; + const perQueryDrifts = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; if (!queryDef) { @@ -550,11 +565,28 @@ function main() { unusable.push( `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` ); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } continue; } compared++; if (role === 'trigger') { triggersCompared++; + // Bucket this trigger by what the ENGINE did, but only where the contract + // pinned a rejection — a trigger pinned as accepted (an advisory rule like + // head-without-sort, whose queries are all valid PPL) never "relaxes", and + // counting it as relaxed would fabricate a full-fix verdict for a rule the + // engine was never rejecting in the first place. + const pinnedRejection = (expected.backend && expected.backend.kind) === 'rejection'; + if (pinnedRejection) { + if (observed.backendRejected === false) { + relaxedTriggers.push(queryName); + if ((observed.detectorCount || 0) > 0) relaxedDetectorFlagged = true; + } else if (observed.backendRejected === true) { + holdingTriggers.push(queryName); + } + } } const drift = classifyDrift({ @@ -581,16 +613,50 @@ function main() { // string identifies WHICH `expectations[]` entry produced this finding, // so a `update-contract` annotation can land on that entry's line rather // than at the top of the file. - drifts.push({ + // Buffered rather than pushed: a relaxation finding is only final once + // every trigger has been seen, because the rule-level rollup below + // replaces the per-query ones with a single full-vs-partial verdict. + perQueryDrifts.push({ ...drift, enforced: isEnforced, contractFile: file, expectationRange: expectation.version, expectationEngine: expectation.engine, }); - ruleDrifts++; } } + + // Every trigger has now been observed, so a relaxation can be judged for the + // rule as a whole. This supersedes the per-query `engine-relaxed` findings — + // they each said "scope this rule away from this version", which is the wrong + // action whenever another trigger still rejects. + const relaxationScope = classifyRelaxationScope({ + ruleId, + version: leg.version, + relaxedTriggers, + holdingTriggers, + unobservedTriggers, + detectorFlagged: relaxedDetectorFlagged, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + }); + const kept = relaxationScope + ? perQueryDrifts.filter((d) => d.supersededBy !== DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED) + : perQueryDrifts; + for (const drift of kept) { + drifts.push(drift); + ruleDrifts++; + } + if (relaxationScope) { + drifts.push({ + ...relaxationScope, + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); + ruleDrifts++; + } // "agree" has to mean "we compared the rule's claim and it held". A rule // whose every case lost its engine verdict (a timed-out leg) or its detector // row (a runner that died mid-corpus) has proven nothing — and so has one diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 43c9979bb6d..3fe9c6fa5dd 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -32,6 +32,7 @@ export const DRIFT_CLASSES = { GRAMMAR_RULE_MISSING: 'grammar-rule-missing', ENGINE_RELAXED: 'engine-relaxed', + ENGINE_PARTIALLY_RELAXED: 'engine-partially-relaxed', ENGINE_TIGHTENED: 'engine-tightened', ENGINE_MESSAGE_CHANGED: 'engine-message-changed', DETECTOR_SILENT: 'detector-silent', @@ -241,6 +242,139 @@ export function classifyGrammarDrift({ }; } +/** + * Decide, for ONE rule on ONE engine version, whether an observed relaxation is + * total or partial — the difference between "scope the rule away from this + * version" and "narrow the detector". + * + * `classifyDrift` sees a single query, so it cannot tell these apart: one trigger + * that the engine now accepts looks identical whether the rule's other triggers + * still fail or not. Acting on that one query is actively harmful in the partial + * case, because scoping the rule out of the version drops the diagnostics that + * are STILL correct — turning a partial engine fix into a false negative on the + * shapes that remain broken. That is why this runs over the whole rule. + * + * Inputs are the per-trigger verdicts the caller already gathered: + * relaxed engine ACCEPTS a trigger the contract pinned as rejected + * holding engine still REJECTS the trigger + * Triggers with no usable verdict are passed as neither, and are reported as the + * reason a verdict is being withheld rather than silently treated as `holding` + * (which would read a dead leg as a partial fix and narrow a healthy detector). + * + * Returns null when nothing relaxed — the caller's per-query drifts stand on + * their own. Otherwise returns ONE rule-level drift that supersedes the + * per-query `engine-relaxed` findings, so the report shows a single decision + * instead of one "scope this away" paragraph per trigger. + * + * @param {object} input + * @param {string[]} input.relaxedTriggers trigger names the engine now accepts + * @param {string[]} input.holdingTriggers trigger names the engine still rejects + * @param {string[]} [input.unobservedTriggers] triggers with no comparable verdict + * @param {boolean} [input.detectorFlagged] did the detector fire on any relaxed trigger + */ +export function classifyRelaxationScope({ + ruleId, + version, + relaxedTriggers = [], + holdingTriggers = [], + unobservedTriggers = [], + detectorFlagged = false, + wiring, + detectorPath, +}) { + if (relaxedTriggers.length === 0) { + return null; + } + + const where = `${ruleId} @ ${version}`; + const base = { + ruleId, + version, + driftVersion: version, + role: 'trigger', + scope: { + relaxed: [...relaxedTriggers], + holding: [...holdingTriggers], + unobserved: [...unobservedTriggers], + }, + }; + // How thin is the basis for a "fully relaxed" claim? A rule with ONE pinned + // trigger that relaxes proves only that one shape changed; calling that "the + // behavior is gone" is a much bigger inference than the data supports. The + // count goes in the evidence either way so the reader can judge it, rather + // than the tool quietly presenting 1-of-1 as though it were 5-of-5. + const observed = relaxedTriggers.length + holdingTriggers.length; + const basis = `${relaxedTriggers.length} of ${observed} observed trigger(s) relaxed`; + + // --- Partial: some triggers relaxed, others still rejected ------------------ + // The engine fixed part of the condition. Scoping the rule out of this version + // would ship a false negative on everything in `holding`, so the action is to + // narrow the detector to the shapes that still fail. + if (holdingTriggers.length > 0) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS ${relaxedTriggers.length} of this rule's triggers ` + + `(${relaxedTriggers.join(', ')}) but still REJECTS ${holdingTriggers.length} ` + + `(${holdingTriggers.join(', ')}) — a PARTIAL fix, ${basis}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Do NOT scope "${ruleId}" away from ${version}: the engine still rejects ` + + `${holdingTriggers.join(', ')}, so a maxVersion below ${version} would drop diagnostics that ` + + `are still correct and ship a false NEGATIVE there. Narrow the detector so it stops matching ` + + `the now-valid shape(s) (${relaxedTriggers.join(', ')}) while still flagging the rest, then ` + + `re-pin the ${version} expectation for the relaxed trigger(s) to detectorCount 0.`, + }, + }; + } + + // --- Full: every observed trigger relaxed ----------------------------------- + // Nothing the rule claims is still true on this engine. Version-scoping is now + // the right action — with the caveat that "every observed trigger" is only as + // strong as the trigger count, which the evidence states. + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS every observed trigger for this rule ` + + `(${relaxedTriggers.join(', ')}) — a FULL fix, ${basis}` + + (unobservedTriggers.length > 0 + ? `; ${unobservedTriggers.length} trigger(s) produced no verdict (${unobservedTriggers.join(', ')}) ` + + `and were NOT counted` + : '') + + '.', + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Every trigger this contract pins is now valid on ${version}, so "${ruleId}" is a FALSE ` + + `POSITIVE there. Set appliesTo.maxVersion just below ${version} to keep protecting users on ` + + `older engines; if no supported engine rejects any trigger any more, set "enabled": false and ` + + `drop the detector. Then re-pin the ${version} expectations to detectorCount 0.` + + (observed < 2 + ? ` NOTE: this rule pins only ${observed} trigger, so "fully relaxed" rests on a single ` + + `observation — confirm with more shapes of the same condition before scoping the rule away.` + : '') + + (unobservedTriggers.length > 0 + ? ` NOTE: ${unobservedTriggers.length} trigger(s) produced no verdict on this leg; re-run it ` + + `before acting, since one of them may still reject.` + : ''), + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectations to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; +} + /** * Classify one query's outcome on one engine version. * @@ -379,6 +513,13 @@ export function classifyDrift(input) { if (role === 'trigger' && expectRejection && backendRejected === false) { return { ...base, + // A single relaxed trigger cannot tell a full fix from a partial one, and the + // two need OPPOSITE actions (scope the rule away vs narrow the detector). The + // caller aggregates every trigger through `classifyRelaxationScope` and drops + // the findings carrying this marker in favour of that one rule-level verdict. + // Kept as a finding rather than returning null so a caller that does not + // aggregate still reports the relaxation instead of silently passing. + supersededBy: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, driftClass: DRIFT_CLASSES.ENGINE_RELAXED, evidence: `${where}: engine ${version} now ACCEPTS a query the contract pinned as rejected ` + From 044c832ab8d11fa0b2600e04f73b69c346e67725 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:29:18 -0700 Subject: [PATCH 49/78] feat(ci): harvest a discovery corpus so trigger coverage can support a scope decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The partial-vs-full relaxation verdict added in the previous commit needs several triggers per rule to be meaningful. The enforced corpus has 27 queries across 11 rules — mostly one trigger each — so for most rules "every trigger relaxed" is a single observation, and the report has to say so rather than present it as proof. OSD's own lint tests already contain the variety: whoever wrote each detector wrote several queries that should fire and several that should not, grouped by rule. `invalid-capture-group-name` has three distinct reasons a name is invalid there versus one in the contract. Harvesting them yields 109 queries across 12 rules — 4x the enforced corpus — at no authoring cost. harvest-queries.mjs extract PPL literals from OSD's lint tests; attribute each to the rule whose describe(...) block encloses it, unescape JS string escapes, remap the index onto the fixture probe-discovery-backend.mjs POST each query to /_plugins/_ppl and record the verdict (no Gradle, no test cluster — there is nothing to assert) label-discovery.mjs derive each role from real detector output and report detector/engine disagreements Roles are derived; expectations never are. An auto-derived expectation could only confirm current behavior, locking in whatever the detector does today including its bugs. `--specs-out` writes the corpus as ordinary spec files so the EXISTING detector runner produces real counts unmodified — the enforced check's behavior is untouched. Promotion into the enforced corpus stays a human writing a spec entry. The new `discovery` job is `continue-on-error` and the labeler always exits zero: a finding here is a lead, not a proven defect, and failing unrelated PRs on an auto-generated guess would destroy the check's credibility. Verified end to end against a live 3.8 cluster. That run found a defect in the labeler itself: `head-without-sort` and `rex-scan-cost` are `info` rules flagging non-determinism and scan cost, which the engine executes happily and will never reject — so "accepted + flagged" is the rule working as designed, not a false positive. Severity is the discriminator, read from what the detector actually emitted. With that fixed the run reports 0 findings on 109 queries, 75 rejections suppressed as uninformative (unknown field, missing index, unsupported command, syntax error) and 2 advisory triggers excluded. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 210 +++++++ scripts/ppl-lint/README.md | 119 +++- .../__tests__/harvest-queries.test.mjs | 240 ++++++++ .../__tests__/label-discovery.test.mjs | 242 ++++++++ .../probe-discovery-backend.test.mjs | 66 ++ scripts/ppl-lint/harvest-queries.mjs | 576 ++++++++++++++++++ scripts/ppl-lint/label-discovery.mjs | 467 ++++++++++++++ scripts/ppl-lint/probe-discovery-backend.mjs | 196 ++++++ 8 files changed, 2111 insertions(+), 5 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/harvest-queries.test.mjs create mode 100644 scripts/ppl-lint/__tests__/label-discovery.test.mjs create mode 100644 scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs create mode 100644 scripts/ppl-lint/harvest-queries.mjs create mode 100644 scripts/ppl-lint/label-discovery.mjs create mode 100644 scripts/ppl-lint/probe-discovery-backend.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index dd89d02cc12..2438e665fbe 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -115,6 +115,7 @@ jobs: outputs: released: ${{ steps.plan.outputs.released }} compiled: ${{ steps.plan.outputs.compiled }} + discovery_engine: ${{ steps.plan.outputs.discovery_engine }} osd_repo: ${{ steps.plan.outputs.osd_repo }} osd_ref: ${{ steps.plan.outputs.osd_ref }} steps: @@ -165,6 +166,24 @@ jobs: " echo "compiled=$compiled" >> "$GITHUB_OUTPUT" echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" + + # Discovery runs against ONE engine — the newest released version in the + # matrix. It is a lead-generator, not a version-drift check, so paying for + # a full matrix would multiply cost without adding signal: a false positive + # found on the newest engine is the one users hit soonest, and per-version + # differences are already the enforced corpus's job. + discovery_engine=$(echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + # Newest by semver, not list order, so a reordered matrix cannot silently + # point discovery at an old engine. + def key(s): + parts=[int(p) for p in s.split('-')[0].split('.') if p.isdigit()] + return parts + [0]*(3-len(parts)) + print(sorted(v,key=key)[-1]) + ") + echo "discovery_engine=$discovery_engine" >> "$GITHUB_OUTPUT" + echo "Discovery engine: \`$discovery_engine\`" >> "$GITHUB_STEP_SUMMARY" # Same precedence as the sibling workflow: dispatch input, then repo # variable, then the canonical upstream default. echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" @@ -674,3 +693,194 @@ jobs: legs/**/detector-report.json legs/**/detector.log legs/**/target.json + + # Discovery: harvest queries from OSD's own lint tests, run both halves over them, + # and report detector/engine disagreements as LEADS. + # + # Why this is separate from `detect`, and why it can never fail the build: + # + # The enforced corpus is hand-pinned — every expectation is a reviewed claim, which + # is what lets a mismatch red the build. That corpus is also small (about one + # trigger per rule), and `classifyRelaxationScope` needs SEVERAL triggers per rule + # to tell a FULL engine fix (version-scope the rule away) from a PARTIAL one + # (narrow the detector). Those need opposite actions, so with one trigger the + # advice can be confidently wrong. + # + # This job supplies that trigger variety from queries OSD's own detector authors + # already wrote. It pins NOTHING: roles are derived from real detector output and + # the engine supplies the other half, so no expectation is ever auto-generated. + # An auto-derived expectation could only confirm current behavior — locking in + # whatever the detector does today, bugs included. + # + # `continue-on-error` AND a zero exit from the labeler: a finding here is a lead to + # investigate, not a proven defect, and blocking unrelated PRs on an auto-generated + # guess would poison the whole check's credibility. + discovery: + name: Discovery corpus (harvested, not enforced) + # Only `plan`, for the OSD target and the engine version. Deliberately NOT the + # observe legs: discovery runs its own engine and harvests its own queries, so + # depending on them would idle this job behind ~30 minutes of matrix work it + # never reads, and a failed leg would block a report that does not need it. + needs: plan + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 40 + services: + opensearch: + image: opensearchproject/opensearch:${{ needs.plan.outputs.discovery_engine }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # The rule id list comes from the OSD catalog being validated, not a hardcoded + # copy: attribution keys off `describe('')` titles, so a stale list + # would silently stop harvesting queries for any newly added rule. + - name: Harvest the discovery corpus from OSD's lint tests + run: | + set -euo pipefail + node -e " + const c = require('./.ci/OpenSearch-Dashboards/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map((r) => r.id))); + " > /tmp/catalog-rules.json + node scripts/ppl-lint/harvest-queries.mjs \ + --osd .ci/OpenSearch-Dashboards \ + --catalog-rules @/tmp/catalog-rules.json \ + --index opensearch-sql_test_index_account \ + --out "$GITHUB_WORKSPACE/discovery-corpus.json" \ + --specs-out "$GITHUB_WORKSPACE/discovery-specs" + + # Seed the one index every harvested query was rewritten onto. Without it the + # engine rejects everything with IndexNotFoundException — which the labeler + # would correctly suppress as uninformative, yielding a run that reports + # nothing at all. + - name: Seed the fixture index + run: | + set -euo pipefail + for i in $(seq 1 40); do + curl -sf http://localhost:9200 > /dev/null && break + echo "waiting for engine (${i}/40)..." + sleep 5 + done + curl -sf -X PUT "http://localhost:9200/opensearch-sql_test_index_account" \ + -H 'content-type: application/json' -d '{ + "mappings": { "properties": { + "account_number": { "type": "long" }, + "balance": { "type": "long" }, + "age": { "type": "integer" }, + "status": { "type": "keyword" }, + "firstname": { "type": "text" }, + "lastname": { "type": "text" }, + "msg": { "type": "text" }, + "body": { "type": "text" }, + "raw": { "type": "object", "enabled": false } + } } + }' + curl -sf -X POST "http://localhost:9200/opensearch-sql_test_index_account/_doc?refresh=true" \ + -H 'content-type: application/json' \ + -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' + + - name: Run the detectors over the discovery corpus + working-directory: .ci/OpenSearch-Dashboards + run: | + set -uo pipefail + # A non-zero exit is EXPECTED and ignored: the generated specs carry + # placeholder expectations, so the runner reports a "failure" for every + # query whose real diagnostic count differs. Only the report is read. + PPL_LINT_SURFACE=compiled-simplified \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$GITHUB_WORKSPACE/discovery-detector.log" 2>&1 || true + if [ ! -f "$GITHUB_WORKSPACE/discovery-detector-report.json" ]; then + echo "::warning::the detector runner produced no discovery report; skipping." + tail -50 "$GITHUB_WORKSPACE/discovery-detector.log" || true + fi + + - name: Probe the engine with the discovery corpus + run: | + set -euo pipefail + node scripts/ppl-lint/probe-discovery-backend.mjs \ + --corpus discovery-corpus.json \ + --endpoint http://localhost:9200 \ + --out discovery-backend-report.json + + - name: Label and report + run: | + set -euo pipefail + if [ ! -f discovery-detector-report.json ]; then + echo "::warning::no detector report; nothing to label." + exit 0 + fi + node scripts/ppl-lint/label-discovery.mjs \ + --corpus discovery-corpus.json \ + --detector discovery-detector-report.json \ + --backend discovery-backend-report.json \ + --version "${{ needs.plan.outputs.discovery_engine }}" \ + --out discovery-findings.json \ + --summary "$GITHUB_STEP_SUMMARY" + + - name: Upload discovery artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-discovery + path: | + discovery-corpus.json + discovery-findings.json + discovery-detector-report.json + discovery-backend-report.json + discovery-detector.log diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f20633768d4..a7cbd31714b 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -292,11 +292,36 @@ Every finding names a drift class, the evidence, and one remediation action: Drift classes: `grammar-rule-missing` (a parser rule the detector walks was renamed or removed — the finding names the closest current rule names), -`engine-relaxed` / `engine-tightened` (the engine's verdict flipped), -`engine-message-changed` (same verdict, reworded error), `detector-silent` / -`detector-noisy` (false negative / false positive), `version-scope-too-narrow` -(the engine rejects but the rule is scoped away from that version, so users see no -diagnostic), and `severity-mismatch`. +`engine-relaxed` / `engine-partially-relaxed` / `engine-tightened` (the engine's +verdict flipped), `engine-message-changed` (same verdict, reworded error), +`detector-silent` / `detector-noisy` (false negative / false positive), +`version-scope-too-narrow` (the engine rejects but the rule is scoped away from +that version, so users see no diagnostic), and `severity-mismatch`. + +#### Full vs partial relaxation: scope the rule, or narrow the detector? + +When an engine starts accepting a query a rule flags, the fix depends on a question +a single query cannot answer: is the behavior **fully** gone on that version, or +only **partially**? + +- **Every trigger relaxed** → `engine-relaxed`, action `version-scope-rule`. Nothing + the rule claims is still true on that engine, so bound it with `maxVersion`. +- **Some triggers relaxed, others still rejected** → `engine-partially-relaxed`, + action `update-detector`. The engine fixed *part* of the condition. Scoping the + rule away here would drop the diagnostics that are still correct, converting a + partial engine fix into a shipped **false negative**. Narrow the detector so it + stops matching the now-valid shapes while still flagging the rest. + +This is decided per rule, not per query: the aggregator collects every trigger's +engine verdict for a rule on a leg, then emits **one** rule-level finding that +supersedes the per-query ones. A trigger with no verdict is counted as neither — +treating it as "still rejects" would let a timed-out leg masquerade as a partial fix +and send someone to narrow a healthy detector. + +The evidence always states the tally (`2 of 3 observed trigger(s) relaxed`), and a +rule with only one pinned trigger gets an explicit warning that a "fully relaxed" +verdict rests on a single observation. That is the gap the discovery corpus below +closes. Four guards keep the check from passing vacuously. Each exists because "we could not check" must never render as "it is fine": @@ -385,6 +410,90 @@ anywhere: node --test "scripts/ppl-lint/__tests__/*.test.mjs" ``` +## Discovery corpus (harvested, never enforced) + +The enforced corpus is hand-pinned, which is what lets a mismatch red the build — +and also why it is small (about one trigger per rule). One trigger is not enough to +tell a full engine fix from a partial one, so the `discovery` job builds a second, +much larger corpus that pins nothing. + +``` +harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report + + discovery-specs/ └──▶ probe-discovery-backend.mjs ─▶ backend report + │ + label-discovery.mjs ──▶ findings + trigger coverage +``` + +1. **Harvest.** `harvest-queries.mjs` extracts PPL literals from OSD's own lint test + suite and attributes each to the rule whose `describe(...)` block encloses it + (matched as a prefix, so `describe('rex-scan-cost (compiled surface)')` counts). + A query with no rule-owning ancestor is recorded unattributed and dropped rather + than guessed at. Indices are rewritten onto the fixture index; JS string escapes + are unescaped so the query matches what the test actually linted. Against OSD + `main` today this yields **~109 queries across 12 rules** versus 27 across 11 in + the enforced corpus. +2. **Observe both halves.** `--specs-out` writes the corpus as ordinary spec files so + the **existing** detector runner produces real diagnostic counts with no changes to + it; a non-zero exit is expected there and ignored, because the generated + expectations are placeholders. `probe-discovery-backend.mjs` sends each query to + `POST /_plugins/_ppl` directly — no Gradle, no test cluster, since there is + nothing to assert. +3. **Label and report.** `label-discovery.mjs` derives each role from real detector + output (fired → trigger, silent → control) and reports disagreements. + +Roles are derived; **expectations never are**. An auto-derived expectation could only +confirm current behavior, locking in whatever the detector does today including its +bugs. Promotion into the enforced corpus stays a human writing a spec entry. + +### What it reports, and how much to trust it + +| Detector | Engine | Reported as | +| --- | --- | --- | +| fires (error/warning) | accepts | `possible-false-positive` — nearly conclusive | +| fires (**info** only) | accepts | nothing — advisory rules are never contradicted by acceptance | +| silent | rejects | `possible-false-negative` — weak, verify first | +| either | no verdict | nothing; the query is labelled but claims nothing | + +The asymmetry is deliberate. A query the engine *ran* successfully but the linter +called broken is unambiguous. A rejection may be for a reason unrelated to the rule, +so rejections matching an unknown field, a missing index, an unsupported command, or +a syntax error are **suppressed** rather than reported — without that filter every +harvested query naming an invented field becomes a finding and buries the real ones. +Suppression never applies to the false-positive side. + +Advisory (`info`) rules are the other exclusion, and it was found by running this +against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` flag +non-determinism and scan cost, which the engine executes happily and will never +reject. For those, "accepted + flagged" is the rule working as designed. Severity is +the discriminator because it already encodes the claim — only an error/warning rule +asserts the engine will refuse the query, and only such a claim can be contradicted +by acceptance. The judgement uses the severities the detector actually emitted, so a +mixed-severity diagnostic is still reported. + +The report also prints per-rule trigger counts and whether each rule has enough +(≥2) to support a scope decision. That table is the direct input to the full-vs-partial +question above: a rule showing **1 trigger** cannot distinguish the two, and a rule +showing **0** was not observed at all. + +This job is `continue-on-error: true` and the labeler always exits zero. A finding +here is a lead, not a proven defect; failing unrelated PRs on an auto-generated +guess would destroy the check's credibility. It runs against one engine (the newest +in the matrix) because it generates leads rather than checking version drift. + +```bash +# Locally, against a running cluster and an OSD checkout: +node -e "const c=require('/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map(r=>r.id)))" > /tmp/rules.json +node scripts/ppl-lint/harvest-queries.mjs --osd --catalog-rules @/tmp/rules.json \ + --index opensearch-sql_test_index_account --out /tmp/corpus.json --specs-out /tmp/specs +( cd && PPL_LINT_SURFACE=compiled-simplified PPL_LINT_CONTRACT_DIR=/tmp/specs \ + PPL_LINT_SCHEDULE=nightly PPL_LINT_REPORT=/tmp/detector.json \ + node -r ./src/setup_node_env "$PWD/../sql/scripts/ppl-lint/run-frontend-contract.mjs" || true ) +node scripts/ppl-lint/probe-discovery-backend.mjs --corpus /tmp/corpus.json --out /tmp/backend.json +node scripts/ppl-lint/label-discovery.mjs --corpus /tmp/corpus.json \ + --detector /tmp/detector.json --backend /tmp/backend.json --out /tmp/findings.json +``` + ## Interpreting a failure | Failure | Meaning | diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs new file mode 100644 index 00000000000..fc9bf7d8276 --- /dev/null +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -0,0 +1,240 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery-corpus harvester. + * + * node --test scripts/ppl-lint/__tests__/harvest-queries.test.mjs + * + * The harvester's job is to attribute a query to the rule that owns it and to hand + * the labeler something the cluster can actually run. Both have a wrong-answer mode + * that is worse than dropping the query: a misattributed query produces a + * "disagreement" for a rule that never claimed anything about it, and an + * unremapped index produces a rejection that reads as engine behavior. Those two + * failure modes are what these tests pin. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + harvestFile, + referencedIdentifiers, + remapIndex, + ruleFromDescribeTitle, + toRunnerSpecs, +} from '../harvest-queries.mjs'; + +const RULES = [ + 'invalid-capture-group-name', + 'field-validation', + 'rex-scan-cost', + 'head-without-sort', + 'division-by-zero', +]; + +// --- attribution ------------------------------------------------------------- + +test('an exact describe title names its rule', () => { + assert.equal(ruleFromDescribeTitle('rex-scan-cost', RULES), 'rex-scan-cost'); +}); + +test('a suffixed title still names its rule', () => { + // OSD's real titles: `describe('rex-scan-cost (compiled surface)')`. Requiring an + // exact match dropped ~75% of harvestable queries. + assert.equal(ruleFromDescribeTitle('rex-scan-cost (compiled surface)', RULES), 'rex-scan-cost'); + assert.equal( + ruleFromDescribeTitle('field-validation alternate-source suppression', RULES), + 'field-validation' + ); +}); + +test('a title that merely mentions a rule mid-sentence does NOT claim it', () => { + // Prefix-only matching is the guard: this describe sits under some OTHER rule and + // must not steal attribution, or its queries get judged against the wrong rule. + assert.equal(ruleFromDescribeTitle('does not fire on rex-scan-cost candidates', RULES), null); +}); + +test('a longer rule id wins over a prefix of itself', () => { + const rules = ['field-validation', 'field-validation-shape']; + assert.equal(ruleFromDescribeTitle('field-validation-shape cases', rules), 'field-validation-shape'); +}); + +test('a rule id must end at a word boundary', () => { + assert.equal(ruleFromDescribeTitle('head-without-sorting quirks', RULES), null); +}); + +test('the innermost rule-owning describe wins', () => { + const source = ` + describe('PPL silent-failure lint rules (compiled surface)', () => { + describe('division-by-zero', () => { + it('flags it', () => { + expect(ids('source=logs | eval x = a / 0')).toContain('division-by-zero'); + }); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, 'division-by-zero'); +}); + +test('a query with no rule-owning ancestor is recorded unattributed, not guessed', () => { + const source = ` + describe('some unrelated suite', () => { + it('x', () => { lint('source=logs | head 10'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, null); +}); + +// --- extraction -------------------------------------------------------------- + +test('JS string escapes are unescaped to the runtime query', () => { + // A test source containing '(?\\\\d+)' is the 4 chars `\\d+` at runtime, which + // is what the detector and the engine both see. Leaving the JS layer escaped + // sends a different query than the test actually linted. + const source = String.raw` + describe('invalid-capture-group-name', () => { + it('x', () => { lint('source=logs | rex field=m "(?\\d+)"'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].query, 'source=logs | rex field=m "(?\\d+)"'); +}); + +test('template literals with interpolation are dropped', () => { + // `${...}` is filled at runtime; sending the placeholder to the engine tests + // nothing and would be reported as a syntax error. + const source = 'describe(\'field-validation\', () => { lint(`source=${idx} | fields a`); });'; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 0); +}); + +test('only PPL-opening strings are treated as queries', () => { + const source = ` + describe('field-validation', () => { + it('x', () => { + expect(msg).toBe('this is not a query at all'); + lint('source=logs | fields a'); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.deepEqual( + out.map((o) => o.query), + ['source=logs | fields a'] + ); +}); + +test('the harvest records where each query came from', () => { + const source = `describe('division-by-zero', () => {\n lint('source=logs | eval x = a / 0');\n});`; + const out = harvestFile(source, { file: 'pkg/x.test.ts', knownRules: RULES }); + assert.match(out[0].source, /^pkg\/x\.test\.ts:2$/); +}); + +// --- index remapping --------------------------------------------------------- + +test('source= is remapped onto the fixture index', () => { + assert.equal( + remapIndex('source=logs | fields a', 'acct'), + 'source=acct | fields a' + ); +}); + +test('a backticked source is remapped', () => { + assert.equal(remapIndex('source=`my-logs` | fields a', 'acct'), 'source=acct | fields a'); +}); + +test('the bare `search ` form is remapped', () => { + assert.equal( + remapIndex('search accounts | eval x = balance / 0', 'acct'), + 'search acct | eval x = balance / 0' + ); +}); + +test('remapping is a no-op without a target index', () => { + assert.equal(remapIndex('source=logs | fields a', ''), 'source=logs | fields a'); +}); + +test('the original query is kept alongside the remapped one', () => { + // Needed to explain a finding: a reader has to be able to see what the OSD test + // actually asserted before trusting a disagreement derived from the rewrite. + const source = `describe('field-validation', () => { lint('source=logs | fields a'); });`; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES, index: 'acct' }); + assert.equal(out[0].query, 'source=acct | fields a'); + assert.equal(out[0].originalQuery, 'source=logs | fields a'); +}); + +test('identifiers are collected for the labeler to intersect against the fixture', () => { + const ids = referencedIdentifiers('source=acct | eval x = durationNano / 0'); + assert.ok(ids.includes('durationNano')); + assert.ok(ids.includes('acct')); +}); + +// --- runner specs ------------------------------------------------------------ + +const CORPUS = { + index: 'acct', + queries: [ + { ruleId: 'division-by-zero', name: 'discovery-0', query: 'source=acct | eval x = a / 0' }, + { ruleId: 'division-by-zero', name: 'discovery-1', query: 'source=acct | eval x = a / 2' }, + { ruleId: 'head-without-sort', name: 'discovery-2', query: 'source=acct | head 5' }, + { ruleId: null, name: 'discovery-3', query: 'source=acct | fields a' }, + ], +}; + +test('one spec is emitted per rule, and unattributed queries are excluded', () => { + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual( + specs.map((s) => s.spec.ruleId), + ['division-by-zero', 'head-without-sort'] + ); + assert.equal(Object.keys(specs[0].spec.queries).length, 2); +}); + +test('generated specs carry no wiring block', () => { + // The runner deep-equals `wiring` against the OSD catalog when present. A + // generated approximation would fail the run for a reason unrelated to discovery. + const specs = toRunnerSpecs(CORPUS); + assert.equal(specs[0].spec.wiring, undefined); +}); + +test('every generated query is declared a trigger', () => { + // Roles are derived later from real detector output. Declaring some as controls + // would make the runner apply control-specific cross-checks whose failures are + // pure noise on a corpus with no pinned verdicts. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + for (const q of Object.values(spec.spec.queries)) { + assert.equal(q.role, 'trigger'); + } + } +}); + +test('exactly one expectation matches any engine version', () => { + // Two matching entries make the runner report the version as uncovered; an + // open-ended empty range is what guarantees a single match on every leg. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + assert.equal(spec.spec.expectations.length, 1); + assert.equal(spec.spec.expectations[0].version, ''); + } +}); + +test('generated specs are scored on either grammar surface', () => { + const specs = toRunnerSpecs(CORPUS); + assert.ok(specs.every((s) => s.spec.grammarSurface === 'both')); +}); + +test('query names are preserved so the two halves can be joined', () => { + // The detector runner and the engine probe key on these names. If they disagreed, + // every row would lose its counterpart and the corpus would read as unobserved. + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual(Object.keys(specs[0].spec.queries), ['discovery-0', 'discovery-1']); +}); diff --git a/scripts/ppl-lint/__tests__/label-discovery.test.mjs b/scripts/ppl-lint/__tests__/label-discovery.test.mjs new file mode 100644 index 00000000000..59ca138470e --- /dev/null +++ b/scripts/ppl-lint/__tests__/label-discovery.test.mjs @@ -0,0 +1,242 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for discovery-corpus labelling. + * + * node --test scripts/ppl-lint/__tests__/label-discovery.test.mjs + * + * The labeler turns two observations into a role and, sometimes, a finding. Every + * way it can produce a CONFIDENT finding from a non-observation is a way to send an + * engineer after a bug that does not exist, so the three-state read and the + * uninformative-rejection filter are pinned case by case. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + FINDINGS, + ROLES, + labelQuery, + renderMarkdown, + uninformativeRejection, +} from '../label-discovery.mjs'; + +const base = { ruleId: 'invalid-capture-group-name', query: 'source=acct | rex field=m "(?x)"' }; + +// --- role assignment --------------------------------------------------------- + +test('a query the detector fires on is a trigger', () => { + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: true }); + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a query the detector ignores is a control', () => { + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: false }); + assert.equal(row.role, ROLES.CONTROL); +}); + +// --- the two findings -------------------------------------------------------- + +test('detector fires + engine accepts is a possible false positive', () => { + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['error', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); + assert.match(row.finding.evidence, /ACCEPTED this query/); +}); + +test('an ADVISORY rule the engine accepts is not a false positive', () => { + // Found against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` are + // `info` rules that flag non-determinism and cost. The engine runs those queries + // happily and will never reject them, so "accepted + flagged" is the rule working + // as designed. Without this, every advisory trigger becomes a finding and buries + // the real ones. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 1, + severities: ['info'], + backendRejected: false, + }); + assert.equal(row.finding, null); + assert.equal(row.advisory, true); + // Still a trigger: it is exactly the kind of trigger the relaxation rollup counts. + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a mixed-severity diagnostic is not treated as advisory', () => { + // Only an ALL-info diagnostic is advisory. One error-severity marker means the + // rule is asserting the engine will refuse the query, which acceptance contradicts. + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['info', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); + +test('an advisory rule the engine REJECTS is still evidence', () => { + // The advisory carve-out applies only to the accepted direction. A silent detector + // on a query the engine refused is unaffected by severity. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'head requires a positive integer', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); +}); + +test('detector silent + engine rejects is a possible false negative', () => { + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: 'capture group name is invalid', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); + // Must tell the reader to verify the rejection is this rule's condition; the + // engine rejecting is weaker evidence than the engine accepting. + assert.match(row.finding.evidence, /VERIFY the rejection is this rule's condition/); +}); + +test('agreement in either direction is not a finding', () => { + assert.equal(labelQuery({ ...base, detectorCount: 1, backendRejected: true }).finding, null); + assert.equal(labelQuery({ ...base, detectorCount: 0, backendRejected: false }).finding, null); +}); + +// --- the three-state read ---------------------------------------------------- + +test('no engine verdict produces no finding', () => { + // The trap this closes: coercing an absent verdict to `false` reads a timed-out + // leg as "the engine accepted this" and manufactures a false-positive finding + // against a healthy rule. + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: undefined }); + assert.equal(row.finding, null); + assert.equal(row.unobserved, true); +}); + +test('a silent detector with no engine verdict is unknown, not a control', () => { + // Calling it a control would claim the rule correctly stayed quiet, which nothing + // observed. It also inflates control counts that the coverage table reports. + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: undefined }); + assert.equal(row.role, ROLES.UNKNOWN); +}); + +// --- uninformative rejections ------------------------------------------------ + +test('an unknown-field rejection is suppressed, not reported', () => { + // Harvested queries name fields their original OSD test invented. Without this + // filter every such query becomes a false-negative finding and buries the real + // ones. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: "can't resolve Symbol(namespace=FIELD_NAME, name=durationNano)", + }); + assert.equal(row.finding, null); + assert.equal(row.suppressed, 'unknown field'); +}); + +test('a syntax error is suppressed', () => { + // After index remapping some harvested queries are genuinely malformed; a query + // the grammar cannot parse says nothing about a semantic rule. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'mismatched input ', + }); + assert.equal(row.suppressed, 'syntax error'); +}); + +test('an unsupported-command rejection is suppressed', () => { + // Same reasoning as the enforced corpus's control-also-rejected guard: the + // command not existing is not evidence about a rule's condition. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendReason: 'union is not supported in this version', + }); + assert.equal(row.suppressed, 'unsupported command'); +}); + +test('an on-topic rejection survives the filter', () => { + assert.equal( + uninformativeRejection('IllegalArgumentException', 'Union command requires at least two datasets'), + null + ); +}); + +test('a rule with no observed trigger is distinguished from one with a single trigger', () => { + // These are different problems: zero triggers means this corpus proves nothing + // about the rule at all (every harvested query was a control, or the detector is + // gated off on this surface), whereas one trigger means the rule is observable but + // a "fully relaxed" verdict would rest on a single case. Rendering both as + // "1 trigger only" hid the first, which is the more serious gap. + const markdown = renderMarkdown({ + stats: { queries: 9, triggers: 4, controls: 5, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [ + { ruleId: 'none-observed', triggers: 0, controls: 4, sufficientForScopeDecision: false }, + { ruleId: 'single', triggers: 1, controls: 2, sufficientForScopeDecision: false }, + { ruleId: 'plenty', triggers: 3, controls: 2, sufficientForScopeDecision: true }, + ], + }); + assert.match(markdown, /`none-observed` \| 0 \| 4 \| \*\*none — no trigger observed\*\*/); + assert.match(markdown, /`single` \| 1 \| 2 \| \*\*no — 1 trigger only\*\*/); + assert.match(markdown, /`plenty` \| 3 \| 2 \| yes/); +}); + +test('a run with no engine half says so instead of implying agreement', () => { + // "0 finding(s)" beside 109 queries reads as "everything agrees". With no engine + // verdicts nothing was compared at all, and the report has to distinguish those. + const withoutEngine = renderMarkdown({ + differential: false, + stats: { queries: 109, triggers: 19, controls: 0, unknown: 90, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.match(withoutEngine, /No engine verdicts were supplied/); + + const withEngine = renderMarkdown({ + differential: true, + stats: { queries: 10, triggers: 4, controls: 6, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.doesNotMatch(withEngine, /No engine verdicts were supplied/); +}); + +test('suppression never applies to the false-POSITIVE side', () => { + // The filter exists to protect the weak (false-negative) direction. A query the + // engine RAN successfully is conclusive regardless of what any error text says, + // so an accepted query must still report even with a suppressible-looking reason. + const row = labelQuery({ + ...base, + detectorCount: 1, + // Explicit rather than relying on the default: with an empty severities list the + // advisory check cannot fire, so the test would pass for the wrong reason and + // stop covering the suppression filter at all. + severities: ['error'], + backendRejected: false, + backendReason: "can't resolve Symbol(name=whatever)", + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); diff --git a/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs new file mode 100644 index 00000000000..5368f8afabb --- /dev/null +++ b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery engine probe's response mapping. + * + * node --test scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs + * + * The probe has one job that can go wrong quietly: turning an HTTP response into a + * verdict. Reading a non-answer as acceptance is what converts a network blip into + * "the engine now accepts this query", so the accept/reject/no-verdict split is + * pinned here. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { readResponse } from '../probe-discovery-backend.mjs'; + +test('a 200 is acceptance', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('a 400 is rejection and keeps the engine type and reason', () => { + // The labeler's uninformative-rejection filter keys on these two strings; losing + // them would let an unknown-field rejection be reported as a missed diagnostic. + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ + error: { type: 'SemanticCheckException', reason: "can't resolve Symbol(name=foo)" }, + }), + }); + assert.equal(v.rejected, true); + assert.equal(v.observed.type, 'SemanticCheckException'); + assert.match(v.observed.reason, /can't resolve/); +}); + +test('a 500 is also rejection', () => { + assert.equal(readResponse({ status: 500, bodyText: '{}' }).rejected, true); +}); + +test('an unparseable body still yields a verdict from the status', () => { + // The engine answered; the body being junk does not change whether it ran the + // query. Discarding the verdict here would lose real signal. + const v = readResponse({ status: 200, bodyText: 'oops' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('an enormous reason is truncated', () => { + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ error: { type: 'X', reason: 'y'.repeat(5000) } }), + }); + assert.equal(v.observed.reason.length, 500); +}); + +test('an accepted response carries no error fields', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.observed.type, undefined); + assert.equal(v.observed.reason, undefined); +}); diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs new file mode 100644 index 00000000000..b641f4612b1 --- /dev/null +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -0,0 +1,576 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Harvest PPL queries out of OSD's own lint test suite into a discovery corpus. + * + * ## Why this exists + * + * The enforced contract corpus is hand-written and therefore small — around one + * trigger and one control per rule. That is right for an enforced contract (every + * expectation is a reviewed claim) but it is too thin to answer the question that + * decides a remediation: when an engine starts accepting a query a rule flags, is + * the behavior FULLY gone on that version, or only PARTIALLY? + * + * `classifyRelaxationScope` in drift.mjs needs several triggers per rule to tell + * those apart, because they need opposite actions — version-scope the rule away + * (full) versus narrow the detector (partial). With one pinned trigger, a partial + * engine fix is indistinguishable from a total one, and the advice that follows + * ships a false negative. + * + * OSD's lint tests already contain that variety: whoever wrote each detector wrote + * several queries that should fire and several that should not, grouped by rule. + * `invalid-capture-group-name` has three distinct reasons a name is invalid + * (hyphen, leading digit, all digits) in OSD's tests versus one in the contract. + * Harvesting them costs nothing and is exactly the input the rollup needs. + * + * ## What this does NOT do + * + * It does not produce expectations, and the discovery corpus never fails a build. + * A harvested query carries no pinned verdict — `label-discovery.mjs` derives its + * role by running the real detectors, and the engine supplies the other half. That + * is deliberate: auto-deriving an expectation from current behavior can only ever + * confirm current behavior, locking in whatever the detector does today, bugs and + * all. Promotion into the enforced corpus stays a human writing a spec entry. + * + * ## Attribution + * + * A query is attributed to a rule by the innermost enclosing `describe('')` + * whose title is a known catalog rule id (OSD's tests are organized that way, see + * `__tests__/silent_failure_rules.test.ts`). A query with no such ancestor is + * recorded with `ruleId: null` and skipped by the labeler unless `--keep-unowned` + * is passed — guessing an owner from a filename would attribute queries to the + * wrong rule, which is worse than dropping them. + * + * Usage: + * node scripts/ppl-lint/harvest-queries.mjs \ + * --osd \ + * --catalog-rules \ + * --index opensearch-sql_test_index_account \ + * --out discovery-corpus.json + */ + +import fs from 'fs'; +import path from 'path'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-harvest] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-harvest] FATAL: ${message}`); + process.exit(2); +} + +/** + * Directories under an OSD checkout that hold PPL lint tests. Kept explicit + * rather than globbing the whole repo: a wide sweep would pull in queries from + * autocomplete/highlighting suites that were never written as lint trigger or + * control cases, and a query harvested from the wrong intent produces a + * "disagreement" that is really just a query nobody claimed anything about. + */ +const LINT_TEST_DIRS = [ + 'packages/osd-monaco/src/ppl/lint/__tests__', + 'packages/osd-monaco/src/ppl/lint/hover/__tests__', + 'packages/osd-monaco/src/ppl/lint/explain/__tests__', +]; + +/** + * Benchmarks and repro captures are excluded. Bench files hold deliberately + * pathological queries built to be slow rather than to be right or wrong, and + * they would dominate the corpus with near-duplicates. + */ +const EXCLUDED_FILE_PATTERNS = [/\.bench\.test\.ts$/, /\.verify\.test\.ts$/]; + +function parseArgs(argv) { + const args = { + osd: '', + out: 'discovery-corpus.json', + index: '', + catalogRules: [], + keepUnowned: false, + maxPerRule: 40, + specsOut: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--osd') args.osd = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--index') args.index = next(); + else if (arg === '--catalog-rules') args.catalogRules = readRuleList(next()); + else if (arg === '--keep-unowned') args.keepUnowned = true; + else if (arg === '--max-per-rule') args.maxPerRule = Number(next()); + else if (arg === '--specs-out') args.specsOut = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.osd) fatal('--osd is required'); + return args; +} + +/** Rule ids either inline (`a,b,c`) or from a file (`@path`), one per line or JSON. */ +function readRuleList(value) { + if (!value.startsWith('@')) { + return value + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + } + const file = value.slice(1); + if (!fs.existsSync(file)) fatal(`--catalog-rules file not found: ${file}`); + const raw = fs.readFileSync(file, 'utf8'); + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + // Accept both a bare id list and OSD's rules_catalog.json shape. + return parsed.map((e) => (typeof e === 'string' ? e : e && e.id)).filter(Boolean); + } + } catch { + // not JSON; fall through to line-delimited + } + return raw + .split('\n') + .map((s) => s.trim()) + .filter((s) => s && !s.startsWith('#')); +} + +/** Every lint test file under the OSD checkout, excluding benches. */ +function findTestFiles(osdRoot) { + const files = []; + for (const dir of LINT_TEST_DIRS) { + const abs = path.join(osdRoot, dir); + if (!fs.existsSync(abs)) continue; + for (const name of fs.readdirSync(abs)) { + if (!name.endsWith('.test.ts') && !name.endsWith('.test.tsx')) continue; + if (EXCLUDED_FILE_PATTERNS.some((re) => re.test(name))) continue; + files.push(path.join(abs, name)); + } + } + return files.sort(); +} + +/** + * Track which `describe(...)` blocks enclose a given offset, so a query can be + * attributed to the rule whose block it sits in. + * + * Brace counting is enough here and a real TS parser is not worth the dependency: + * these are test files whose describes are conventional `describe('x', () => {` + * calls. The failure mode of miscounting is a query attributed to an outer block + * (or to none), which the `ruleId: null` path already handles safely — never a + * query attributed to a rule that does not own it, because titles must match a + * known catalog id. + */ +function buildDescribeScopes(source) { + const scopes = []; + const describeRe = /\bdescribe(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g; + let match; + while ((match = describeRe.exec(source)) !== null) { + const title = match[2]; + // Find the block's opening brace after the describe call, then its matching + // close, ignoring braces inside strings and comments. + const braceStart = source.indexOf('{', match.index + match[0].length); + if (braceStart === -1) continue; + const end = matchBrace(source, braceStart); + scopes.push({ title, start: braceStart, end: end === -1 ? source.length : end }); + } + return scopes; +} + +/** Index of the `}` matching the `{` at `open`, or -1. Skips strings/comments. */ +function matchBrace(source, open) { + let depth = 0; + for (let i = open; i < source.length; i++) { + const ch = source[i]; + if (ch === '/' && source[i + 1] === '/') { + const nl = source.indexOf('\n', i); + i = nl === -1 ? source.length : nl; + continue; + } + if (ch === '/' && source[i + 1] === '*') { + const close = source.indexOf('*/', i + 2); + i = close === -1 ? source.length : close + 1; + continue; + } + if (ch === "'" || ch === '"' || ch === '`') { + i = skipString(source, i); + continue; + } + if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + +/** Index of the closing quote of the string starting at `start`. */ +function skipString(source, start) { + const quote = source[start]; + for (let i = start + 1; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === quote) return i; + } + return source.length; +} + +/** + * PPL query literals. Anchored on the commands that can open a PPL statement, so + * arbitrary strings in a test file are not mistaken for queries. `search` and + * `source=`/`index=` are the real openers; `describe` is deliberately absent + * because it collides with the test function of the same name. + */ +const QUERY_RE = /(['"`])((?:source\s*=|index\s*=|search\s+)(?:\\.|(?!\1).)*)\1/g; + +/** + * A harvested literal is a JS string literal, so its escapes are JS-level. The + * detectors want the RUNTIME string: `'(?\\\\d+)'` in a test source is + * the four characters `\\d+` on the wire... which is itself a regex escape the + * engine sees. Unescaping the JS layer (and only that layer) is what makes a + * harvested query identical to what the test actually linted. + */ +function unescapeJsString(raw) { + return raw.replace(/\\(u\{[0-9a-fA-F]+\}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|.)/g, (all, esc) => { + switch (esc[0]) { + case 'n': + return '\n'; + case 't': + return '\t'; + case 'r': + return '\r'; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'v': + return '\v'; + case '0': + return '\0'; + case 'x': + return String.fromCharCode(parseInt(esc.slice(1), 16)); + case 'u': + return esc[1] === '{' + ? String.fromCodePoint(parseInt(esc.slice(2, -1), 16)) + : String.fromCharCode(parseInt(esc.slice(1), 16)); + default: + // Covers \\ \' \" \` and any other single-character escape. + return esc; + } + }); +} + +/** + * Rewrite the test's index onto the backend fixture's index. + * + * OSD's unit tests lint against invented sources (`source=logs`, `search + * accounts`) that do not exist in the SQL integ-test cluster. A harvested query + * must name a real index or the engine rejects it for a reason that has nothing to + * do with the rule — which would read as "the engine rejects this" and be counted + * as a trigger holding. + * + * Only the leading source/index/search clause is rewritten. Subsearch sources + * inside the query body are rewritten too, since those are equally invented. + * Returns null when no rewrite was possible, so the caller can drop the query + * rather than send an unresolvable index to the cluster. + */ +export function remapIndex(query, targetIndex) { + if (!targetIndex) return query; + let out = query + .replace(/\bsource\s*=\s*`[^`]+`/g, `source=${targetIndex}`) + .replace(/\bsource\s*=\s*[A-Za-z_][\w.*-]*/g, `source=${targetIndex}`) + .replace(/\bindex\s*=\s*`[^`]+`/g, `index=${targetIndex}`) + .replace(/\bindex\s*=\s*[A-Za-z_][\w.*-]*/g, `index=${targetIndex}`); + // `search ` — the bare-index form. Only the opener, and only when the + // token is not already a keyword-led clause. + out = out.replace(/^(\s*search\s+)(?!source\s*=|index\s*=)([A-Za-z_][\w.*-]*)/, `$1${targetIndex}`); + return out; +} + +/** + * Does this query reference fields the backend fixture will not have? + * + * A harvested query naming `durationNano` against the `account` index is rejected + * for an unknown field, not for the rule's condition. Counting that as "the engine + * still rejects" would fake a partial fix and send someone to narrow a healthy + * detector — the same class of vacuous result the enforced contract's + * control-also-rejected guard exists to prevent. + * + * This cannot be decided statically, so it is not decided here: the query is + * harvested with the field names it mentions recorded, and the labeler drops the + * ones the fixture cannot satisfy. Extracting identifiers is best-effort and + * deliberately over-broad (it will include command keywords), because the labeler + * intersects against the fixture's real field list rather than trusting this. + */ +export function referencedIdentifiers(query) { + const ids = new Set(); + for (const match of query.matchAll(/\b([A-Za-z_][\w.]*)\b/g)) { + ids.add(match[1]); + } + return [...ids]; +} + +/** + * The rule a `describe(...)` title names, or null. + * + * OSD's titles are not bare ids — a rule-scoped suite reads + * `describe('rex-scan-cost (compiled surface)')` or + * `describe('field-validation alternate-source suppression')`. Requiring an exact + * match dropped ~75% of harvestable queries, all of them from files dedicated to a + * single rule, so the title is matched as a PREFIX at a word boundary. + * + * Prefix-only is the point: matching a rule id anywhere in the title would let + * `describe('does not fire on rex-scan-cost candidates')`, nested under a + * different rule, steal the attribution. A title that merely mentions another rule + * mid-sentence is not that rule's suite. Longest match wins, so + * `field-validation-shape` is preferred over `field-validation` when both exist. + */ +export function ruleFromDescribeTitle(title, knownRules) { + const text = String(title || ''); + let best = null; + for (const ruleId of knownRules) { + if (!text.startsWith(ruleId)) continue; + // Must end at a word boundary: `head-without-sorting` is not `head-without-sort`. + const after = text.charAt(ruleId.length); + if (after && /[\w-]/.test(after)) continue; + if (!best || ruleId.length > best.length) best = ruleId; + } + return best; +} + +/** Harvest one file into `{ ruleId, query, source }` records. */ +export function harvestFile(source, { file, knownRules, index }) { + const scopes = buildDescribeScopes(source); + const known = new Set(knownRules || []); + const out = []; + for (const match of source.matchAll(QUERY_RE)) { + const raw = match[2]; + const query = unescapeJsString(raw); + // Template literals with interpolation are not real queries — the `${...}` is + // a placeholder the test fills at runtime, and sending it to the engine tests + // nothing. Dropped rather than guessed at. + if (/\$\{/.test(query)) continue; + // A query must have at least one pipe or be a bare source read; anything + // shorter is usually a fragment asserted against, not a lintable statement. + if (query.trim().length < 8) continue; + + // Innermost enclosing describe whose title is a known rule id. + const at = match.index; + const enclosing = scopes + .filter((s) => at > s.start && at < s.end) + .sort((a, b) => b.start - a.start); + // Innermost first: a query inside `describe('flat-object-subfield')` nested in + // `describe('silent-failure rules')` belongs to the specific rule, not the file. + let owner = null; + for (const scope of enclosing) { + owner = ruleFromDescribeTitle(scope.title, known); + if (owner) break; + } + const line = source.slice(0, at).split('\n').length; + + out.push({ + ruleId: owner, + query: index ? remapIndex(query, index) : query, + originalQuery: query, + identifiers: referencedIdentifiers(query), + source: `${file}:${line}`, + }); + } + return out; +} + +/** + * Emit the harvested corpus as spec files the EXISTING detector runner can consume. + * + * `run-frontend-contract.mjs` is expectation-driven: it walks `expectations[]`, + * scores each query against a pinned `detectorCount`, and records the real `actual` + * count in its report either way. Discovery needs only that `actual`, so rather + * than teach the runner a second mode — which would risk changing how the ENFORCED + * check behaves — the corpus is written out as ordinary specs whose expectations are + * deliberately arbitrary. + * + * Two consequences, both intended: + * - The runner will report failures for every query whose real count differs from + * the placeholder. Those are meaningless here and the caller discards the exit + * code; only `detector-report.json` is read. This is why discovery must never + * be wired to a required check. + * - `grammarSurface: 'both'` so a rule is scored on whichever surface the leg + * ran, and `schedule: 'nightly'` to match how the aggregate legs invoke it. + * + * One spec per rule, because the runner keys wiring checks off `spec.ruleId`. + */ +export function toRunnerSpecs(corpus) { + const byRule = new Map(); + for (const [i, entry] of (corpus.queries || []).entries()) { + if (!entry.ruleId) continue; + if (!byRule.has(entry.ruleId)) byRule.set(entry.ruleId, []); + byRule.get(entry.ruleId).push({ ...entry, name: entry.name || `discovery-${i}` }); + } + + const specs = []; + for (const [ruleId, entries] of [...byRule].sort()) { + const queries = {}; + const expected = {}; + for (const entry of entries) { + // Every query is declared a trigger: the runner needs SOME role, and the real + // role is derived later from the detector's actual output. Calling them all + // triggers keeps the runner from applying its control-specific cross-checks, + // whose failures would be pure noise on a corpus with no pinned verdicts. + queries[entry.name] = { role: 'trigger', query: entry.query }; + expected[entry.name] = { detectorCount: 0 }; + } + specs.push({ + fileName: `${ruleId}.discovery.spec.json`, + spec: { + schemaVersion: 3, + ruleId, + grammarSurface: 'both', + schedule: 'nightly', + // No `wiring` block: the runner deep-equals it against the catalog when + // present, and a mismatch there would fail the run for a reason that has + // nothing to do with discovery. + index: corpus.index || undefined, + queries, + // A single open expectation so exactly one entry matches every engine + // version; the pinned counts are placeholders (see the note above). + expectations: [{ version: '', queries: expected }], + }, + }); + } + return specs; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const files = findTestFiles(args.osd); + if (files.length === 0) { + fatal(`no lint test files found under ${args.osd}; is --osd an OSD checkout root?`); + } + + const records = []; + for (const file of files) { + const source = fs.readFileSync(file, 'utf8'); + const rel = path.relative(args.osd, file); + records.push( + ...harvestFile(source, { file: rel, knownRules: args.catalogRules, index: args.index }) + ); + } + + // Dedupe on (ruleId, query): the same query legitimately appears in several + // tests, and running it repeatedly against the cluster buys nothing. + const seen = new Set(); + const unique = []; + for (const record of records) { + const key = `${record.ruleId}::${record.query}`; + if (seen.has(key)) continue; + seen.add(key); + unique.push(record); + } + + const owned = unique.filter((r) => r.ruleId); + const unowned = unique.filter((r) => !r.ruleId); + + // Cap per rule so one heavily-tested rule cannot dominate a leg's runtime. The + // drop is LOGGED rather than silent: a truncated corpus that reads as complete + // is how a coverage gap hides. + const byRule = new Map(); + for (const record of owned) { + if (!byRule.has(record.ruleId)) byRule.set(record.ruleId, []); + byRule.get(record.ruleId).push(record); + } + const kept = []; + for (const [ruleId, group] of [...byRule].sort()) { + if (group.length > args.maxPerRule) { + log( + `NOTE: ${ruleId} harvested ${group.length} queries; keeping the first ${args.maxPerRule} ` + + `(--max-per-rule). ${group.length - args.maxPerRule} dropped.` + ); + } + kept.push(...group.slice(0, args.maxPerRule)); + } + + // Names are assigned ONCE, here, and every downstream artifact keys off them. If + // the detector runner and the engine probe derived names independently they could + // disagree, and every row would silently lose its counterpart — the whole corpus + // would read as unobserved rather than as a bug. + kept.forEach((entry, i) => { + entry.name = `discovery-${i}`; + }); + + const corpus = { + schemaVersion: 1, + kind: 'discovery', + // Stated in the artifact itself so no downstream consumer can mistake this for + // the enforced corpus and start failing builds on it. + enforced: false, + note: + 'Auto-harvested from OSD lint tests. Roles are assigned by label-discovery.mjs from real ' + + 'detector output; there are no pinned expectations and this corpus must never fail a build.', + index: args.index || null, + sourceFiles: files.map((f) => path.relative(args.osd, f)), + queries: kept, + unowned: args.keepUnowned ? unowned : [], + stats: { + files: files.length, + harvested: records.length, + unique: unique.length, + owned: kept.length, + unowned: unowned.length, + rules: byRule.size, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(corpus, null, 2)); + log( + `wrote ${args.out}: ${kept.length} owned query(s) across ${byRule.size} rule(s) from ` + + `${files.length} file(s); ${unowned.length} unattributed` + + (args.keepUnowned ? ' (kept)' : ' (dropped)') + ); + + // Optional: the same corpus as spec files, so the existing detector runner can + // produce real diagnostic counts for it without being modified. + if (args.specsOut) { + fs.mkdirSync(args.specsOut, { recursive: true }); + const specs = toRunnerSpecs(corpus); + for (const { fileName, spec } of specs) { + fs.writeFileSync(path.join(args.specsOut, fileName), JSON.stringify(spec, null, 2)); + } + fs.writeFileSync( + path.join(args.specsOut, 'manifest.json'), + JSON.stringify( + { + schemaVersion: 3, + description: + 'AUTO-GENERATED discovery corpus. No reviewed expectations; the pinned counts are ' + + 'placeholders. Never list these under defaultError and never wire them to a required check.', + contracts: specs.map((s) => s.fileName), + // Empty on purpose: `defaultError` is the ENFORCED set, and nothing here is + // enforced. A non-empty value would make the aggregator fail the build on + // auto-generated expectations. + defaultError: [], + }, + null, + 2 + ) + ); + log(`wrote ${specs.length} runner spec(s) to ${args.specsOut}`); + } + for (const [ruleId, group] of [...byRule].sort()) { + log(` ${ruleId}: ${Math.min(group.length, args.maxPerRule)}`); + } +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && path.resolve(process.argv[1]).endsWith('harvest-queries.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/label-discovery.mjs b/scripts/ppl-lint/label-discovery.mjs new file mode 100644 index 00000000000..3ee6c318860 --- /dev/null +++ b/scripts/ppl-lint/label-discovery.mjs @@ -0,0 +1,467 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assign roles to harvested queries and report detector/engine disagreements. + * + * ## Why roles are derived, not authored + * + * A trigger is a query the rule's detector fires on; a control is one it stays + * silent on. That is mechanically checkable, so it is checked rather than declared: + * the enforced corpus's hand-set `role` field carries a reviewer's intent, but for + * a harvested corpus of 100+ queries hand-labelling is both the bottleneck and a + * source of error. This script reads the detector report the OSD runner already + * produces and labels from it. + * + * Deriving roles is safe. Deriving EXPECTATIONS would not be: an expectation + * auto-set from current behavior can only ever confirm current behavior, locking in + * whatever the detector does today including its bugs. So this script pins nothing + * and never fails a build. It emits findings. + * + * ## The finding it exists for + * + * With both halves observed, the interesting cell needs no expected output at all: + * + * detector engine meaning + * silent rejects possible FALSE NEGATIVE + * fires accepts possible FALSE POSITIVE + * fires rejects agreement + * silent accepts agreement + * + * "Possible", not "confirmed", and the asymmetry is deliberate. A false positive + * is nearly conclusive: the engine ran the query fine and the linter called it + * broken. A false negative is much weaker — the engine may have rejected the query + * for a reason that has nothing to do with this rule (an unknown field, an index + * that does not exist, a command the version predates), in which case the linter + * was right to stay quiet. Both are reported, ranked, and neither is ever asserted. + * + * ## What this feeds + * + * `classifyRelaxationScope` needs several triggers per rule to tell a FULL engine + * fix (version-scope the rule away) from a PARTIAL one (narrow the detector). This + * corpus is where that trigger variety comes from. For that use it needs only + * "does any trigger still get rejected" — a single counterexample settles the + * question, which is why no pinned verdict is required. + * + * Usage: + * node scripts/ppl-lint/label-discovery.mjs \ + * --corpus discovery-corpus.json \ + * --detector discovery-detector-report.json \ + * --backend discovery-backend-report.json \ + * --fixture-fields account-fields.json \ + * --out discovery-findings.json [--summary $GITHUB_STEP_SUMMARY] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-discovery] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-discovery] FATAL: ${message}`); + process.exit(2); +} + +/** Roles a harvested query can be assigned. */ +export const ROLES = { + TRIGGER: 'trigger', + CONTROL: 'control', + UNKNOWN: 'unknown', +}; + +/** Finding kinds, ranked by how conclusive they are. */ +export const FINDINGS = { + FALSE_POSITIVE: 'possible-false-positive', + FALSE_NEGATIVE: 'possible-false-negative', +}; + +function parseArgs(argv) { + const args = { + corpus: '', + detector: '', + backend: '', + fixtureFields: '', + out: 'discovery-findings.json', + summary: '', + version: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--detector') args.detector = next(); + else if (arg === '--backend') args.backend = next(); + else if (arg === '--fixture-fields') args.fixtureFields = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--summary') args.summary = next(); + else if (arg === '--version') args.version = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + if (!args.detector) fatal('--detector is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +/** + * Reasons an engine rejection tells us nothing about the rule under test. + * + * This is the guard that keeps the false-negative side honest. A harvested query + * mentions whatever fields its original OSD unit test invented, and those rarely + * exist in the SQL integ-test index. The engine then rejects for an unknown field + * — and reading that as "the engine rejects this, so the silent detector is a false + * negative" would generate a finding per harvested query and bury the real ones. + * + * Also excluded: a command the engine version does not have. Same logic as the + * enforced corpus's control-also-rejected guard — "unsupported command" is not + * evidence about a rule's specific condition. + */ +export const UNINFORMATIVE_REJECTION_PATTERNS = [ + { re: /can't resolve|cannot resolve|unknown field|no such field|field \[[^\]]+\] not found/i, why: 'unknown field' }, + { re: /IndexNotFoundException|no such index|index \[[^\]]+\] (does not exist|not found)/i, why: 'missing index' }, + { re: /unsupported (command|operation)|is not supported|not yet supported/i, why: 'unsupported command' }, + { re: /SyntaxCheckException|ParseException|mismatched input|extraneous input/i, why: 'syntax error' }, +]; + +/** + * Is this rejection informative about the rule, or an artifact of the harvested + * query not fitting the fixture? Returns the reason it is uninformative, or null. + * + * A syntax error counts as uninformative on purpose. A harvested query that the + * grammar cannot even parse says nothing about a semantic rule — and after index + * remapping some harvested queries genuinely are malformed (a join whose right-hand + * index was a bare identifier the remap could not reach). Treating those as + * evidence would be the vacuous-finding equivalent of a timed-out leg. + */ +export function uninformativeRejection(backendType, backendReason) { + const text = `${backendType || ''} ${backendReason || ''}`; + for (const { re, why } of UNINFORMATIVE_REJECTION_PATTERNS) { + if (re.test(text)) return why; + } + return null; +} + +/** + * Label one query and decide whether it is a finding. + * + * `detectorCount` and `backendRejected` come from the two observation halves. + * `backendRejected === undefined` means no verdict arrived, which is a third state: + * the query is labelled but produces no finding, because a leg that did not answer + * must never generate linter advice. + */ +export function labelQuery({ + ruleId, + query, + detectorCount, + backendRejected, + backendType, + backendReason, + severities = [], +}) { + const fired = (detectorCount || 0) > 0; + // An advisory diagnostic is one the engine will never contradict: `info` severity + // marks cost or non-determinism, not an error the engine would refuse. Read from + // the severities the detector actually EMITTED rather than from the catalog, so a + // rule that emits mixed severities is judged on what this query produced. + const advisory = fired && severities.length > 0 && severities.every((s) => s === 'info'); + const role = fired ? ROLES.TRIGGER : ROLES.CONTROL; + const base = { + ruleId, + query, + role, + detectorCount: detectorCount || 0, + backendRejected, + severities, + }; + + // No engine verdict: label the role (which only needs the detector) but claim + // nothing about correctness. + if (typeof backendRejected !== 'boolean') { + return { ...base, role: fired ? ROLES.TRIGGER : ROLES.UNKNOWN, finding: null, unobserved: true }; + } + + // Detector fires, engine accepts → the linter called a working query broken. + // Nearly conclusive: nothing about the fixture can make a query the engine RAN + // into a rule violation. + // + // EXCEPT for advisory rules. `head-without-sort` (info) and `rex-scan-cost` (info) + // flag non-determinism and cost — things the engine executes happily and will + // never reject. For those, "engine accepts + detector fires" is the rule working + // exactly as designed, not a false positive. Without this the report is dominated + // by every advisory rule's every trigger, and the real findings are unreadable. + // + // Severity is the discriminator because it already encodes the distinction: an + // error/warning rule asserts the engine will refuse or mishandle the query, and + // only such a claim can be contradicted by the engine accepting it. + if (fired && backendRejected === false) { + if (advisory) { + return { ...base, finding: null, advisory: true }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_POSITIVE, + evidence: + `the engine ACCEPTED this query but "${ruleId}" emitted ${detectorCount} diagnostic(s). ` + + `A user running this query sees an error marker on a query that works.`, + }, + }; + } + + // Detector silent, engine rejects → possible missed diagnostic, but only if the + // rejection is about something this rule could have caught. + if (!fired && backendRejected === true) { + const uninformative = uninformativeRejection(backendType, backendReason); + if (uninformative) { + return { ...base, finding: null, suppressed: uninformative }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_NEGATIVE, + evidence: + `the engine REJECTED this query (${backendType || 'error'}: ${backendReason || 'no reason'}) ` + + `and "${ruleId}" stayed silent. VERIFY the rejection is this rule's condition before acting — ` + + `a rejection for an unrelated reason is not a missed diagnostic.`, + }, + }; + } + + return { ...base, finding: null }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = readJson(args.corpus); + const detectorReport = readJson(args.detector); + const backendReport = readJson(args.backend, { optional: true }); + + if (corpus.enforced) { + // A corpus that claims to be enforced does not belong on this path: this script + // pins nothing and exits zero, so running it over the enforced corpus would + // look like validation while asserting nothing. + fatal('--corpus is marked enforced; this script only labels the discovery corpus.'); + } + + const detectorByKey = new Map(); + for (const row of detectorReport.results || []) { + detectorByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + const backendByKey = new Map(); + for (const row of Array.isArray(backendReport) ? backendReport : []) { + backendByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + + const labelled = []; + for (const [i, entry] of (corpus.queries || []).entries()) { + const queryName = entry.name || `discovery-${i}`; + const key = `${entry.ruleId}::${queryName}`; + const detectorRow = detectorByKey.get(key); + const backendRow = backendByKey.get(key); + // Same three-state read as the enforced aggregator: `outcome: "error"` carries + // no verdict, and coercing that absence to "accepted" would manufacture a + // false-positive finding out of a network blip. + const hasVerdict = + !!backendRow && + backendRow.outcome !== 'error' && + (typeof backendRow.rejected === 'boolean' || !!backendRow.observed); + + labelled.push({ + ...labelQuery({ + ruleId: entry.ruleId, + query: entry.query, + detectorCount: detectorRow ? detectorRow.actual : undefined, + severities: (detectorRow && detectorRow.severities) || [], + backendRejected: hasVerdict ? !!backendRow.rejected : undefined, + backendType: backendRow && backendRow.observed ? backendRow.observed.type : undefined, + backendReason: backendRow && backendRow.observed ? backendRow.observed.reason : undefined, + }), + queryName, + source: entry.source, + noDetectorRow: !detectorRow, + }); + } + + const findings = labelled.filter((l) => l.finding); + const byRule = new Map(); + for (const row of labelled) { + if (!byRule.has(row.ruleId)) { + byRule.set(row.ruleId, { triggers: [], controls: [], unknown: [], suppressed: 0 }); + } + const bucket = byRule.get(row.ruleId); + if (row.suppressed) bucket.suppressed++; + if (row.role === ROLES.TRIGGER) bucket.triggers.push(row.queryName); + else if (row.role === ROLES.CONTROL) bucket.controls.push(row.queryName); + else bucket.unknown.push(row.queryName); + } + + const report = { + schemaVersion: 1, + kind: 'discovery-findings', + enforced: false, + engineVersion: args.version || detectorReport.engineVersion || null, + surface: detectorReport.surface || null, + // Whether an engine half was supplied at all. Without it the run yields trigger + // counts but structurally cannot yield findings, and the report has to say which + // of those two it is. + differential: backendByKey.size > 0, + stats: { + queries: labelled.length, + triggers: labelled.filter((l) => l.role === ROLES.TRIGGER).length, + controls: labelled.filter((l) => l.role === ROLES.CONTROL).length, + unknown: labelled.filter((l) => l.role === ROLES.UNKNOWN).length, + suppressed: labelled.filter((l) => l.suppressed).length, + // Advisory triggers the engine accepted. Counted so the number is visible: it + // is the single largest category the filter removes, and a silent removal + // would make the corpus look smaller than it is. + advisory: labelled.filter((l) => l.advisory).length, + findings: findings.length, + falsePositives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_POSITIVE).length, + falseNegatives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_NEGATIVE).length, + }, + // Per-rule trigger counts are the payload the relaxation rollup consumes: a + // rule with several triggers can distinguish a partial engine fix from a full + // one, and a rule with one cannot. + triggerCoverage: [...byRule] + .map(([ruleId, b]) => ({ + ruleId, + triggers: b.triggers.length, + controls: b.controls.length, + unknown: b.unknown.length, + suppressed: b.suppressed, + // Below two triggers, "every trigger relaxed" is a single observation and + // cannot support a version-scoping decision. Flagged so the gap is visible + // rather than implied by a number nobody reads. + sufficientForScopeDecision: b.triggers.length >= 2, + })) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + findings: findings.map((f) => ({ + ruleId: f.ruleId, + kind: f.finding.kind, + evidence: f.finding.evidence, + query: f.query, + source: f.source, + })), + labelled, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + const markdown = renderMarkdown(report); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + // ALWAYS exit zero. This corpus has no reviewed expectations, so a finding here + // is a lead to investigate, not a proven defect — failing the build on it would + // block unrelated PRs on the strength of an auto-generated guess. + log( + `discovery: ${report.stats.findings} finding(s) from ${report.stats.queries} query(s) ` + + `(${report.stats.falsePositives} possible false positive(s), ` + + `${report.stats.falseNegatives} possible false negative(s)); ` + + `${report.stats.suppressed} rejection(s) suppressed as uninformative, ` + + `${report.stats.advisory} advisory trigger(s) excluded. Not enforced.` + ); +} + +export function renderMarkdown(report) { + const lines = []; + lines.push('## PPL lint discovery corpus (not enforced)'); + lines.push(''); + lines.push( + `Engine \`${report.engineVersion || 'unknown'}\`${report.surface ? ` (${report.surface})` : ''} — ` + + `${report.stats.queries} harvested query(s): ${report.stats.triggers} trigger, ` + + `${report.stats.controls} control, ${report.stats.unknown} unknown. ` + + `**${report.stats.findings} finding(s)** — these are LEADS, not failures.` + ); + lines.push(''); + // Without an engine half there is nothing to disagree WITH, so the run can only + // count triggers. Saying so beats printing "0 finding(s)" next to a large corpus, + // which reads as "everything agrees" when in fact nothing was compared. + if (!report.differential) { + lines.push( + '> No engine verdicts were supplied, so no agreement was checked and no finding can be ' + + 'produced. Trigger counts below are still valid — they come from the detector alone.' + ); + lines.push(''); + } + + if (report.stats.findings > 0) { + lines.push('### Findings'); + lines.push(''); + // False positives first: a query the engine ran successfully but the linter + // marked broken is nearly conclusive, while a false negative may be a rejection + // for an unrelated reason. + const order = [FINDINGS.FALSE_POSITIVE, FINDINGS.FALSE_NEGATIVE]; + for (const kind of order) { + const group = report.findings.filter((f) => f.kind === kind); + if (group.length === 0) continue; + lines.push(`#### ${kind} (${group.length})`); + for (const f of group) { + lines.push(`- \`${f.ruleId}\`: ${f.evidence}`); + lines.push(` QUERY: \`${f.query}\``); + if (f.source) lines.push(` HARVESTED FROM: ${f.source}`); + } + lines.push(''); + } + } + + lines.push('### Trigger coverage'); + lines.push(''); + lines.push('Whether each rule has enough triggers to tell a PARTIAL engine fix from a FULL one.'); + lines.push(''); + lines.push('| Rule | Triggers | Controls | Enough for a scope decision? |'); + lines.push('| ---- | -------- | -------- | ---------------------------- |'); + for (const row of report.triggerCoverage) { + // Zero triggers and one trigger are different problems and must not read the + // same. No trigger at all means this corpus proves nothing about the rule — + // usually that the harvested queries are all controls, or that the detector + // never fired because it is gated off on this surface. One trigger means the + // rule is observable but a "fully relaxed" verdict would rest on a single case. + let verdict; + if (row.triggers === 0) { + verdict = '**none — no trigger observed**'; + } else if (row.triggers === 1) { + verdict = '**no — 1 trigger only**'; + } else { + verdict = 'yes'; + } + lines.push(`| \`${row.ruleId}\` | ${row.triggers} | ${row.controls} | ${verdict} |`); + } + lines.push(''); + return lines.join('\n'); +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && process.argv[1].endsWith('label-discovery.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/probe-discovery-backend.mjs b/scripts/ppl-lint/probe-discovery-backend.mjs new file mode 100644 index 00000000000..8ebb0d25d75 --- /dev/null +++ b/scripts/ppl-lint/probe-discovery-backend.mjs @@ -0,0 +1,196 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Run the discovery corpus against a live engine and record each verdict. + * + * This is the engine half of the discovery pipeline, and it deliberately does NOT + * go through `PplLintRuleValidationIT`. The IT is an assertion harness built around + * reviewed contract files: it selects a pinned expectation per query and compares + * against it. Discovery queries have no pinned expectation by design, so there is + * nothing for the IT to assert and no reason to pay for a Gradle test-cluster run. + * All that is needed is `POST /_plugins/_ppl` per query and the verdict recorded — + * which is what this does, in the same report shape the aggregator and the labeler + * already read. + * + * Emits `[{ ruleId, queryName, rejected, outcome, observed: { httpStatus, type, + * reason } }]`, matching `backend-report.json` so `label-discovery.mjs` can read + * either source without a special case. + * + * The `outcome` field carries the distinction everything downstream depends on: + * + * observed the engine answered; `rejected` is a real verdict + * error no answer arrived (timeout, connection refused, unparseable body) + * + * Never collapse `error` into `rejected: false`. That coercion is what turns a + * network blip into "the engine now ACCEPTS this query" and generates a + * false-positive finding against a healthy rule. + * + * Usage: + * node scripts/ppl-lint/probe-discovery-backend.mjs \ + * --corpus discovery-corpus.json \ + * --endpoint http://localhost:9200 \ + * --out discovery-backend-report.json [--timeout-ms 15000] [--concurrency 4] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-probe] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-probe] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { + corpus: '', + endpoint: 'http://localhost:9200', + out: 'discovery-backend-report.json', + timeoutMs: 15000, + concurrency: 4, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--endpoint') args.endpoint = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--timeout-ms') args.timeoutMs = Number(next()); + else if (arg === '--concurrency') args.concurrency = Math.max(1, Number(next())); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + return args; +} + +/** + * Read one PPL response into a verdict. + * + * A 2xx is acceptance. A 4xx/5xx is rejection, and the engine's `error.type` / + * `error.reason` are extracted because the labeler's uninformative-rejection filter + * keys on them — a rejection for an unknown field must not be read as evidence + * about a lint rule. + * + * Exported so the mapping is unit-testable without a cluster. + */ +export function readResponse({ status, bodyText }) { + let body; + try { + body = bodyText ? JSON.parse(bodyText) : undefined; + } catch { + body = undefined; + } + const error = (body && body.error) || {}; + const rejected = status >= 400; + return { + outcome: 'observed', + rejected, + observed: { + httpStatus: status, + rejected, + ...(rejected + ? { + type: error.type || undefined, + // Truncated: engine reasons can embed a whole stack trace, and the full + // text bloats the report without adding signal for the filter. + reason: typeof error.reason === 'string' ? error.reason.slice(0, 500) : undefined, + } + : {}), + }, + }; +} + +/** One query against the engine. Never throws: a failure becomes `outcome: error`. */ +async function probeOne({ endpoint, query, timeoutMs }) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(`${endpoint.replace(/\/$/, '')}/_plugins/_ppl`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ query }), + signal: controller.signal, + }); + const bodyText = await response.text(); + return readResponse({ status: response.status, bodyText }); + } catch (error) { + // No verdict. Recorded as such rather than guessed at — see the header note. + return { + outcome: 'error', + observed: undefined, + error: String((error && error.message) || error), + }; + } finally { + clearTimeout(timer); + } +} + +/** Run `tasks` with at most `limit` in flight, preserving input order. */ +async function mapLimit(items, limit, fn) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = JSON.parse(fs.readFileSync(args.corpus, 'utf8')); + const queries = corpus.queries || []; + if (queries.length === 0) fatal(`corpus ${args.corpus} has no queries`); + + log(`probing ${queries.length} query(s) against ${args.endpoint} (concurrency ${args.concurrency})`); + + const report = await mapLimit(queries, args.concurrency, async (entry, i) => { + const verdict = await probeOne({ + endpoint: args.endpoint, + query: entry.query, + timeoutMs: args.timeoutMs, + }); + return { + ruleId: entry.ruleId, + // Must match the name `label-discovery.mjs` derives, or every row misses its + // detector counterpart and the whole corpus reads as unobserved. + queryName: entry.name || `discovery-${i}`, + role: 'discovery', + query: entry.query, + ...verdict, + }; + }); + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + const errors = report.filter((r) => r.outcome === 'error').length; + const rejected = report.filter((r) => r.rejected === true).length; + log( + `wrote ${args.out}: ${report.length} probed, ${rejected} rejected, ` + + `${report.length - rejected - errors} accepted, ${errors} unobserved` + ); + if (errors > 0) { + // A warning, not a failure: discovery is best-effort and the labeler already + // withholds findings for unobserved queries. Saying nothing would let a leg + // that mostly failed look like a leg that mostly agreed. + log(`WARN: ${errors} query(s) produced no verdict; those yield no findings.`); + } +} + +if (process.argv[1] && process.argv[1].endsWith('probe-discovery-backend.mjs')) { + main().catch((error) => fatal(String((error && error.stack) || error))); +} From 84f551ca1d67cc7bb395c3ebaf7980552aa3b09d Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:35:23 -0700 Subject: [PATCH 50/78] feat(ci): harvest each test file's lint context so context-gated rules produce triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harvesting queries without the context they were written against left 6 of 12 rules at ZERO triggers. All six are `needsContext: true` — they self-suppress without a `typeMap`, so the detector never ran. In the report that is indistinguishable from a rule that fired on nothing, and it is the state the trigger-coverage table exists to surface: `rex-scan-cost` contributed 26 queries and no triggers at all. The context now comes from the same test file as the queries — its `typeMap` and `disabledObjectFields` — because the OSD author wrote it to make exactly those queries fire. A hand-written substitute would be a guess about which field types each query depends on, and a wrong guess silently suppresses the detector again. A rule tested under two different contexts gets two spec files; merging them would hand a query field types its own test never used, so the verdict would describe a scenario nobody wrote. Two further fixes found by running the pipeline rather than by reading it: - `visibleIndices` is now supplied unconditionally, not only when a typeMap exists. `wildcard-source-zero-match` reads only that list and self-suppresses when it is empty, and its test file declares no typeMap — so keying it off the mapping left the rule permanently inert. - A wildcard source is no longer remapped. Rewriting `source=`nope-*`` onto the fixture index destroyed the only thing that rule detects, turning its one harvested query into a control. Verified against a live 3.8 cluster: triggers 19 -> 41, rules with enough triggers to support a scope decision 5 -> 9 of 12, still 0 findings. The three rules left at one trigger are at the ceiling of what OSD's tests contain — their remaining queries are genuine controls (`stats avg(balance)` on a numeric is valid; `row_number` is the one window function eventstats supports), so raising those needs queries nobody has written yet. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 23 ++++ .../__tests__/harvest-queries.test.mjs | 101 ++++++++++++++++ scripts/ppl-lint/harvest-queries.mjs | 111 +++++++++++++++++- 3 files changed, 231 insertions(+), 4 deletions(-) diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index a7cbd31714b..f4eb2bb89e2 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -432,6 +432,20 @@ harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-front are unescaped so the query matches what the test actually linted. Against OSD `main` today this yields **~109 queries across 12 rules** versus 27 across 11 in the enforced corpus. + + Each file's **lint context** is harvested alongside its queries. Seven of the + nineteen rules are `needsContext: true` and self-suppress without a `typeMap`, so + harvesting queries alone produced 26 `rex-scan-cost` queries and zero triggers — + the detector never ran, which in the report is indistinguishable from a rule that + fired on nothing. The context is taken from the test file (its `typeMap`, + `disabledObjectFields`) because its author wrote it to make exactly those queries + fire; a hand-written substitute would be a guess about which field types each + query depends on, and a wrong guess silently suppresses the detector again. A rule + tested under two different contexts gets two spec files rather than a merged one. + + A **wildcard** source is deliberately not remapped: `wildcard-source-zero-match` + exists to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + to a concrete index destroys the only thing it detects. 2. **Observe both halves.** `--specs-out` writes the corpus as ordinary spec files so the **existing** detector runner produces real diagnostic counts with no changes to it; a non-zero exit is expected there and ignored, because the generated @@ -475,6 +489,15 @@ The report also prints per-rule trigger counts and whether each rule has enough question above: a rule showing **1 trigger** cannot distinguish the two, and a rule showing **0** was not observed at all. +Against OSD `main` on the compiled surface this currently yields **41 triggers with +9 of 12 rules at ≥2**. The three that remain at one trigger are at the ceiling of +what OSD's tests contain — `agg-on-text`, `wildcard-source-zero-match` and +`unsupported-window-function-in-eventstats` each have exactly one trigger written +there, and their other queries are genuine controls (`stats avg(balance)` on a +numeric field is valid; `row_number` is the one window function eventstats +supports). Raising those needs queries nobody has written yet — the point where +generation, rather than harvesting, is what adds coverage. + This job is `continue-on-error: true` and the labeler always exits zero. A finding here is a lead, not a proven defect; failing unrelated PRs on an auto-generated guess would destroy the check's credibility. It runs against one engine (the newest diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs index fc9bf7d8276..42539f4529e 100644 --- a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -20,6 +20,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { + harvestContext, harvestFile, referencedIdentifiers, remapIndex, @@ -162,6 +163,106 @@ test('remapping is a no-op without a target index', () => { assert.equal(remapIndex('source=logs | fields a', ''), 'source=logs | fields a'); }); +test('a WILDCARD source is left alone', () => { + // Found by running the pipeline: rewriting `source=`nope-*`` to a concrete index + // destroyed the only thing `wildcard-source-zero-match` detects, so its single + // harvested query became a control and the rule reported zero triggers. + assert.equal(remapIndex('source=`nope-*`', 'acct'), 'source=`nope-*`'); + assert.equal(remapIndex('source=logs-* | fields a', 'acct'), 'source=logs-* | fields a'); + assert.equal(remapIndex('index=a* | head 1', 'acct'), 'index=a* | head 1'); +}); + +test('a non-wildcard source is still remapped when a wildcard appears elsewhere', () => { + // The guard must key on a wildcard in the SOURCE, not anywhere in the query — a + // regex or a field list containing `*` is unrelated to index resolution. + assert.equal( + remapIndex('source=logs | rex field=m "(?.*)"', 'acct'), + 'source=acct | rex field=m "(?.*)"' + ); +}); + +// --- context harvesting ------------------------------------------------------ + +test('a typeMap declaration is harvested', () => { + // Seven of nineteen rules are needsContext and self-suppress without this. The + // context is taken from the test file because its author wrote it to make exactly + // these queries fire; a hand-written substitute would be a guess. + const source = ` + const typeMap = new Map([ + ['age', 'long'], + ['firstname', 'text'], + ['attributes', 'flat_object'], + ]); + `; + assert.deepEqual(harvestContext(source).typeMap, { + age: 'long', + firstname: 'text', + attributes: 'flat_object', + }); +}); + +test('disabledObjectFields is harvested', () => { + const source = "const ctx = { typeMap, disabledObjectFields: new Set(['raw', 'blob']) };"; + assert.deepEqual(harvestContext(source).disabledObjectFields, ['raw', 'blob']); +}); + +test('a file with no context declaration yields an empty context', () => { + const out = harvestContext("describe('x', () => { lint('source=a | head 1'); });"); + assert.deepEqual(out.typeMap, {}); + assert.deepEqual(out.disabledObjectFields, []); +}); + +test('generated specs carry the harvested context as frontendContext', () => { + const corpus = { + index: 'acct', + queries: [ + { + ruleId: 'flat-object-subfield', + name: 'd0', + query: 'source=acct | where attributes.x = 1', + context: { typeMap: { attributes: 'flat_object' }, disabledObjectFields: ['raw'] }, + }, + ], + }; + const spec = toRunnerSpecs(corpus)[0].spec; + assert.deepEqual(spec.frontendContext.deriveFromMapping, { attributes: 'flat_object' }); + assert.deepEqual(spec.frontendContext.disabledObjectFields, ['raw']); + // Two rules ship `enabled: false` and only run when the host overrides them. + assert.equal(spec.frontendContext.forceEnable, true); +}); + +test('visibleIndices is supplied even with no typeMap', () => { + // `wildcard-source-zero-match` reads ONLY visibleIndices and self-suppresses on an + // empty list; its test file declares no typeMap, so keying this off the mapping + // left the rule permanently inert. + const spec = toRunnerSpecs({ + index: 'acct', + queries: [ + { ruleId: 'wildcard-source-zero-match', name: 'd0', query: 'source=`nope-*`', context: {} }, + ], + })[0].spec; + assert.deepEqual(spec.frontendContext.visibleIndices, ['{{index}}']); +}); + +test('one rule tested under two different contexts yields two specs', () => { + // Merging them would hand a query field types its own test never used, so the + // verdict would describe a scenario nobody wrote. + const corpus = { + index: 'acct', + queries: [ + { ruleId: 'rex-scan-cost', name: 'd0', query: 'source=acct | rex field=a ""', context: { typeMap: { a: 'text' } } }, + { ruleId: 'rex-scan-cost', name: 'd1', query: 'source=acct | rex field=b ""', context: { typeMap: { b: 'keyword' } } }, + ], + }; + const specs = toRunnerSpecs(corpus); + assert.equal(specs.length, 2); + // Suffixed only when a rule actually has more than one context. + assert.deepEqual(specs.map((s) => s.fileName).sort(), [ + 'rex-scan-cost.1.discovery.spec.json', + 'rex-scan-cost.2.discovery.spec.json', + ]); +}); + test('the original query is kept alongside the remapped one', () => { // Needed to explain a finding: a reader has to be able to see what the OSD test // actually asserted before trusting a disagreement derived from the rewrite. diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs index b641f4612b1..82023a8b187 100644 --- a/scripts/ppl-lint/harvest-queries.mjs +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -286,6 +286,15 @@ function unescapeJsString(raw) { */ export function remapIndex(query, targetIndex) { if (!targetIndex) return query; + // A WILDCARD source is left alone. `wildcard-source-zero-match` exists precisely + // to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + // to a concrete index destroys the only thing the rule detects — the query became + // a control and the rule reported zero triggers. More generally, a wildcard is + // part of the query's meaning rather than an incidental index name, and the + // engine resolves a non-matching pattern on its own without erroring. + if (/\bsource\s*=\s*`?[^\s`|,]*\*/.test(query) || /\bindex\s*=\s*`?[^\s`|,]*\*/.test(query)) { + return query; + } let out = query .replace(/\bsource\s*=\s*`[^`]+`/g, `source=${targetIndex}`) .replace(/\bsource\s*=\s*[A-Za-z_][\w.*-]*/g, `source=${targetIndex}`) @@ -348,8 +357,54 @@ export function ruleFromDescribeTitle(title, knownRules) { return best; } +/** + * Harvest the lint CONTEXT a test file declares, not just its queries. + * + * Seven of nineteen rules are `needsContext: true` — they self-suppress without a + * `typeMap`, and `enabled-false-object` additionally needs `disabledObjectFields`. + * Harvesting their queries without their context produced 26 `rex-scan-cost` + * queries and zero triggers: the detector never ran, which is indistinguishable in + * the report from a rule that fired on nothing. + * + * The context is taken from the file rather than invented because the OSD test + * author wrote it to make exactly these queries fire. A hand-written substitute + * would be a guess about which field types each query depends on, and a wrong guess + * silently suppresses the detector again. + * + * Parses the conventional shapes those files use: + * const typeMap = new Map([ ['age', 'long'], ... ]); + * disabledObjectFields: new Set(['raw']), + * + * Regex rather than a TS parser for the same reason as `buildDescribeScopes`: these + * are conventional declarations, and the failure mode is an empty context, which + * leaves the rule visibly at zero triggers rather than producing a wrong verdict. + */ +export function harvestContext(source) { + const typeMap = {}; + // Every `['name', 'type']` pair inside a `new Map...([ ... ])` initializer. Scoped + // to Map literals so unrelated tuple arrays in the file are not picked up. + for (const mapMatch of source.matchAll(/new Map\s*(?:<[^>]*>)?\s*\(\s*\[([\s\S]*?)\]\s*\)/g)) { + for (const pair of mapMatch[1].matchAll(/\[\s*'([^']+)'\s*,\s*'([^']+)'\s*\]/g)) { + typeMap[pair[1]] = pair[2]; + } + } + + const disabledObjectFields = []; + for (const match of source.matchAll(/disabledObjectFields:\s*new Set\s*\(\s*\[([^\]]*)\]/g)) { + for (const item of match[1].matchAll(/'([^']+)'/g)) { + disabledObjectFields.push(item[1]); + } + } + + return { + typeMap, + disabledObjectFields: [...new Set(disabledObjectFields)], + }; +} + /** Harvest one file into `{ ruleId, query, source }` records. */ export function harvestFile(source, { file, knownRules, index }) { + const context = harvestContext(source); const scopes = buildDescribeScopes(source); const known = new Set(knownRules || []); const out = []; @@ -384,6 +439,10 @@ export function harvestFile(source, { file, knownRules, index }) { originalQuery: query, identifiers: referencedIdentifiers(query), source: `${file}:${line}`, + // Carried per query, not per rule: two files can test the same rule with + // different field types, and merging them would give a query a typeMap its + // own test never used. + context, }); } return out; @@ -410,15 +469,34 @@ export function harvestFile(source, { file, knownRules, index }) { * One spec per rule, because the runner keys wiring checks off `spec.ruleId`. */ export function toRunnerSpecs(corpus) { + // Grouped by rule AND by harvested context. A `needsContext` rule self-suppresses + // without a typeMap, so a query has to be scored under the context its own test + // declared — merging two files' contexts into one spec would hand a query field + // types its test never used, and the resulting verdict would describe a scenario + // nobody wrote. const byRule = new Map(); for (const [i, entry] of (corpus.queries || []).entries()) { if (!entry.ruleId) continue; - if (!byRule.has(entry.ruleId)) byRule.set(entry.ruleId, []); - byRule.get(entry.ruleId).push({ ...entry, name: entry.name || `discovery-${i}` }); + const contextKey = JSON.stringify(entry.context || {}); + const key = `${entry.ruleId}${contextKey}`; + if (!byRule.has(key)) { + byRule.set(key, { ruleId: entry.ruleId, context: entry.context, entries: [] }); + } + byRule.get(key).entries.push({ ...entry, name: entry.name || `discovery-${i}` }); } + // A rule with more than one distinct context needs more than one spec file, so + // names are suffixed only when that happens — keeping the common case readable. + const groupCount = new Map(); + for (const { ruleId } of byRule.values()) { + groupCount.set(ruleId, (groupCount.get(ruleId) || 0) + 1); + } + const seenPerRule = new Map(); + const specs = []; - for (const [ruleId, entries] of [...byRule].sort()) { + for (const [, group] of [...byRule].sort((a, b) => a[0].localeCompare(b[0]))) { + const { ruleId, context } = group; + const entries = group.entries; const queries = {}; const expected = {}; for (const entry of entries) { @@ -429,8 +507,32 @@ export function toRunnerSpecs(corpus) { queries[entry.name] = { role: 'trigger', query: entry.query }; expected[entry.name] = { detectorCount: 0 }; } + const ordinal = (seenPerRule.get(ruleId) || 0) + 1; + seenPerRule.set(ruleId, ordinal); + const suffix = groupCount.get(ruleId) > 1 ? `.${ordinal}` : ''; + + const typeMap = (context && context.typeMap) || {}; + const disabledObjectFields = (context && context.disabledObjectFields) || []; + const frontendContext = { isCalcite: true }; + if (Object.keys(typeMap).length > 0) { + // `deriveFromMapping` is what the runner turns into `fields` + `typeMap`, the + // context every `needsContext` rule requires before it will emit anything. + frontendContext.deriveFromMapping = typeMap; + } + // Always supplied, independent of the typeMap. `wildcard-source-zero-match` + // reads ONLY `visibleIndices` and self-suppresses on an empty list (otherwise + // every wildcard would false-fire "matched 0 of 0") — and its test file declares + // no typeMap, so keying this off the mapping left the rule permanently inert. + frontendContext.visibleIndices = ['{{index}}']; + if (disabledObjectFields.length > 0) { + frontendContext.disabledObjectFields = disabledObjectFields; + } + // Two rules ship `enabled: false` and only run when the host overrides them. + // Without this they are inert and every harvested query reads as a control. + frontendContext.forceEnable = true; + specs.push({ - fileName: `${ruleId}.discovery.spec.json`, + fileName: `${ruleId}${suffix}.discovery.spec.json`, spec: { schemaVersion: 3, ruleId, @@ -440,6 +542,7 @@ export function toRunnerSpecs(corpus) { // present, and a mismatch there would fail the run for a reason that has // nothing to do with discovery. index: corpus.index || undefined, + frontendContext, queries, // A single open expectation so exactly one entry matches every engine // version; the pinned counts are placeholders (see the note above). From d5900c8fc1632ded50a1c09306d2810536736304 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:39:40 -0700 Subject: [PATCH 51/78] fix(test): write the target manifest even without a grammar bundle The 3.5.0 compiled-surface leg observed all 27 contract cases against the real engine and reported success, but uploaded no target.json -- and the multi-version aggregator treats a leg without one as fatal, so those observations were unusable. Cause: exportGrammarArtifacts returns early when -Dppl.lint.grammar.bundle is unset, and the target manifest was written inside that same block. A compiled-surface leg deliberately omits the bundle flag (its engine predates the grammar endpoint), so it silently produced a leg with no engine version recorded. Write the manifest on both paths. Every consumer keys on engineVersion, which has nothing to do with whether a bundle exists; grammarHash and grammarBundle are simply empty when there is none. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 0a0b686cc44..e9d1ad6a947 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -606,6 +606,13 @@ JSONObject toJson() { private void exportGrammarArtifacts(List failures) { String bundlePath = System.getProperty("ppl.lint.grammar.bundle"); if (bundlePath == null || bundlePath.isEmpty()) { + // No bundle requested. That is a compiled-surface leg (an engine predating + // GET /_plugins/_ppl/_grammar) or a local run. The target manifest still has + // to be written: it carries the engine version every consumer keys on, and + // the multi-version aggregator treats a leg without one as fatal. Writing it + // only alongside the bundle silently produced legs the aggregator could not + // read. + writeTargetManifest("", failures); return; } try { @@ -615,16 +622,7 @@ private void exportGrammarArtifacts(List failures) { JSONObject bundle = new JSONObject(bundleBody); String grammarHash = bundle.optString("grammarHash", ""); - - String targetPath = System.getProperty("ppl.lint.target"); - if (targetPath != null && !targetPath.isEmpty()) { - JSONObject target = - new JSONObject() - .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) - .put("grammarHash", grammarHash) - .put("grammarBundle", Paths.get(bundlePath).getFileName().toString()); - Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); - } + writeTargetManifest(grammarHash, Paths.get(bundlePath).getFileName().toString(), failures); log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); } catch (Exception e) { failures.add( @@ -632,6 +630,33 @@ private void exportGrammarArtifacts(List failures) { } } + /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ + private void writeTargetManifest(String grammarHash, List failures) { + writeTargetManifest(grammarHash, "", failures); + } + + /** + * Write {@code ppl.lint.target}: the engine version, the grammar hash when there is one, and the + * bundle filename when one was exported. Every consumer keys on {@code engineVersion}, so this is + * written whether or not a bundle exists. + */ + private void writeTargetManifest(String grammarHash, String bundleName, List failures) { + String targetPath = System.getProperty("ppl.lint.target"); + if (targetPath == null || targetPath.isEmpty()) { + return; + } + try { + JSONObject target = + new JSONObject() + .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) + .put("grammarHash", grammarHash) + .put("grammarBundle", bundleName); + Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + failures.add("[grammar-export] failed to write " + targetPath + ": " + e.getMessage()); + } + } + // --- cluster settings ------------------------------------------------------ /** True when the contract's fixture leaves Calcite enabled (the default). */ From cf551e97fc61a57837daa96fa5e187d17b72229c Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:40:41 -0700 Subject: [PATCH 52/78] fix(ci): run discovery on the runtime-bundle surface, not only the compiled one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The discovery job hardcoded `PPL_LINT_SURFACE=compiled-simplified`. That was an unnecessary restriction: `lint_runner` SKIPS the four `runtimeOnly` rules on the compiled grammar because the productions they walk do not exist there, so a compiled-only run cannot observe them at all — and three of the four ship at error severity, where a false positive is most expensive. The job now exports the engine's grammar via GET /_plugins/_ppl/_grammar and lints on the runtime surface, falling back to the compiled surface with a warning if the export fails. Best-effort rather than fatal: a lead-generator that produces nothing because one endpoint was unavailable is worse than one with narrower coverage, and the surface is recorded in the report so a reader can tell which ran. Also fixes an unbound-variable crash in that step. Expanding an empty array as "${extra[@]}" under `set -u` is an error in bash before 4.4, so the compiled-surface fallback would have died — the one path that only runs when something else already went wrong. Verified both branches. Worth recording since it bounds what harvesting can achieve: the four runtimeOnly rules are at zero harvested queries and no surface changes that. OSD's lint tests contain no trigger for union-min-datasets, multisearch-min-subsearch or replace-wildcard-asymmetry; the only place they appear is a negative assertion that they no-op on the compiled surface. Harvesting cannot invent what was never written. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 46 ++++++++++++++++++- scripts/ppl-lint/README.md | 15 +++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 2438e665fbe..1a8aefbb535 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -830,14 +830,58 @@ jobs: -H 'content-type: application/json' \ -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' + # Export this engine's grammar bundle so the detector pass can run on the + # RUNTIME surface. That surface matters more than the compiled one here: the + # four `runtimeOnly` rules (union/multisearch/replace arity) are SKIPPED by + # lint_runner on the compiled grammar because the productions they walk do not + # exist there — so a compiled-only discovery run cannot observe them at all, + # and three of the four ship at error severity. + - name: Export the engine grammar bundle + id: bundle + run: | + set -uo pipefail + if curl -sf --max-time 60 "http://localhost:9200/_plugins/_ppl/_grammar" \ + -o "$GITHUB_WORKSPACE/discovery-bundle.json"; then + hash=$(python3 -c " + import json + print(json.load(open('$GITHUB_WORKSPACE/discovery-bundle.json')).get('grammarHash','')) + ") + python3 -c " + import json + json.dump({'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '$hash'}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " + echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" + else + # Not fatal. Discovery is best-effort, and the compiled surface still + # covers 12 of the rules — a lead-generator that produces nothing because + # one endpoint was unavailable is worse than one with narrower coverage. + # The surface is recorded in the report, so a reader can see which ran. + echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." + echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" + fi + - name: Run the detectors over the discovery corpus working-directory: .ci/OpenSearch-Dashboards + env: + SURFACE: ${{ steps.bundle.outputs.surface }} run: | set -uo pipefail + # Seeded with a harmless assignment rather than left empty: under `set -u`, + # expanding an empty array as "${a[@]}" is an unbound-variable error in bash + # before 4.4, which would crash the compiled-surface fallback — the very + # path that only runs when something else already went wrong. + extra=(PPL_LINT_DISCOVERY=1) + if [ "$SURFACE" = 'runtime-bundle' ]; then + extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") + fi # A non-zero exit is EXPECTED and ignored: the generated specs carry # placeholder expectations, so the runner reports a "failure" for every # query whose real diagnostic count differs. Only the report is read. - PPL_LINT_SURFACE=compiled-simplified \ + env "${extra[@]}" \ + PPL_LINT_SURFACE="$SURFACE" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f4eb2bb89e2..e1e6a1439d6 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -489,7 +489,7 @@ The report also prints per-rule trigger counts and whether each rule has enough question above: a rule showing **1 trigger** cannot distinguish the two, and a rule showing **0** was not observed at all. -Against OSD `main` on the compiled surface this currently yields **41 triggers with +Measured on the compiled surface against OSD `main`, this yields **41 triggers with 9 of 12 rules at ≥2**. The three that remain at one trigger are at the ceiling of what OSD's tests contain — `agg-on-text`, `wildcard-source-zero-match` and `unsupported-window-function-in-eventstats` each have exactly one trigger written @@ -498,6 +498,19 @@ numeric field is valid; `row_number` is the one window function eventstats supports). Raising those needs queries nobody has written yet — the point where generation, rather than harvesting, is what adds coverage. +The job prefers the **runtime-bundle** surface, exporting the engine's grammar via +`GET /_plugins/_ppl/_grammar` and falling back to the compiled surface (with a +warning) if that fails. The runtime surface matters because `lint_runner` SKIPS the +four `runtimeOnly` rules on the compiled grammar — the productions they walk do not +exist there — and three of those ship at error severity. + +Those four are nonetheless still at **zero** harvested queries, and no surface fixes +that: OSD's lint tests contain no trigger for `union-min-datasets`, +`multisearch-min-subsearch` or `replace-wildcard-asymmetry` at all. The only place +they appear is a negative assertion that they no-op on the compiled surface +(`analyzer_lint.test.ts`, "runtime-only rules no-op"). Harvesting cannot invent what +was never written, so these are generation's job, not the harvester's. + This job is `continue-on-error: true` and the labeler always exits zero. A finding here is a lead, not a proven defect; failing unrelated PRs on an auto-generated guess would destroy the check's credibility. It runs against one engine (the newest From 8437efa70ef2455bc5086deacb9174574adcc506 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:53:25 -0700 Subject: [PATCH 53/78] test(ppl-lint): add a second trigger to every enforced contract that had one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The partial-vs-full relaxation verdict reads the ENFORCED corpus, and 8 of its 11 contracts pinned exactly one trigger. With one trigger, "every trigger relaxed" is a single observation, so the verdict cannot distinguish a full engine fix (version-scope the rule away) from a partial one (narrow the detector) — and those need opposite actions. The classifier warns about it, but the fix is more triggers. Each new trigger exercises a DIFFERENT shape of the same condition, so a partial engine fix is visible as a disagreement between them rather than as a uniform flip: union-min-datasets single dataset that carries a pipeline (| fields) multisearch-min-subsearch single subsearch that carries a pipeline (| where) replace-wildcard-asymmetry reversed asymmetry (2 wildcards -> 1, not 1 -> 2) invalid-capture-group-name hyphen, not just underscore unsupported-window-function dense_rank, not just rank dedup-consecutive-unsupported multi-field dedup with consecutive=true division-by-zero decimal 0.0, not just integer 0 head-without-sort head after a where stage, not a bare source Every expectation was verified on a live 3.8 engine rather than inferred, including the exact error type and reason string: union/fields 400 IllegalArgumentException Union command requires ... Provided: 1 multisearch/where 400 SyntaxCheckException Invalid Query replace 2->1 400 IllegalArgumentException pattern has 2 wildcard(s), replacement has 1 rex hyphen 400 IllegalArgumentException Invalid capture group name 'user-name'. eventstats dense 400 CalciteUnsupportedException Unexpected window function: dense_rank dedup multi-field 200 (advisory; succeeds via the Calcite-to-v2 fallback) head after where 200 (advisory) balance / 0.0 200 with the ratio column all-null Detector counts and severities were confirmed by running the real detector runner over the corpus: all four compiled-surface triggers score 1 at the contracted severity, and eventstats-dense-rank scores 1/error once a version is supplied. The four runtime-bundle-only contracts are reported not-applicable on the compiled surface, as before. Failure count is unchanged from baseline (8, all pre-existing "enabled catalog rule has no contract file" coverage warnings). Worth noting for review: the pre-3.8 expectations for eventstats-dense-rank reuse the existing rank() pins (500 / UnsupportedOperationException) by analogy — both are CalciteUnsupportedException on 3.8, and only 3.8 was available to verify. The 3.6 and 3.7 legs will confirm or correct them. Signed-off-by: Hanyu Wei --- .../dedup-consecutive-unsupported.spec.json | 45 ++++++++- .../contracts/division-by-zero.spec.json | 50 ++++++++-- .../contracts/head-without-sort.spec.json | 40 +++++++- .../invalid-capture-group-name.spec.json | 34 +++++++ .../multisearch-min-subsearch.spec.json | 55 +++++++++-- .../replace-wildcard-asymmetry.spec.json | 34 +++++++ .../contracts/union-min-datasets.spec.json | 57 ++++++++++-- ...ed-window-function-in-eventstats.spec.json | 91 +++++++++++++++++-- 8 files changed, 370 insertions(+), 36 deletions(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index 90a614307b7..fb9d44de7c3 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -10,11 +10,19 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.3.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.3.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": true } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": true + } }, "frontendContext": { "isCalcite": true @@ -25,6 +33,10 @@ "role": "trigger", "query": "source={{index}} | dedup firstname consecutive=true" }, + "dedup-consecutive-true-multi-field": { + "role": "trigger", + "query": "source={{index}} | dedup firstname, lastname consecutive=true" + }, "dedup-plain-control": { "role": "control", "query": "source={{index}} | dedup firstname" @@ -38,11 +50,34 @@ "dedup-consecutive-true": { "detectorCount": 1, "severity": "warning", - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "dedup-consecutive-true-multi-field": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } }, "dedup-plain-control": { "detectorCount": 0, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 5e1b33884c8..0303e32e2a5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL — it documents that boundary rather than asserting a gap.", + "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", "grammarSurface": "both", "schedule": "nightly", "wiring": { @@ -14,8 +14,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -26,6 +31,10 @@ "role": "trigger", "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1" }, + "divide-by-decimal-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0.0 | fields ratio | head 1" + }, "divide-by-nonzero-control": { "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" @@ -42,15 +51,44 @@ "divide-by-zero-literal": { "detectorCount": 1, "severity": "warning", - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } + }, + "divide-by-decimal-zero-literal": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } }, "divide-by-nonzero-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } }, "modulo-by-zero-not-flagged": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 695f1b6b550..68cd387a88c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -13,8 +13,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -25,6 +30,10 @@ "role": "trigger", "query": "source={{index}} | head 5" }, + "head-without-sort-after-where": { + "role": "trigger", + "query": "source={{index}} | where age > 20 | head 5" + }, "head-with-sort-control": { "role": "control", "query": "source={{index}} | sort age | head 5" @@ -37,11 +46,34 @@ "head-without-sort": { "detectorCount": 1, "severity": "info", - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "head-without-sort-after-where": { + "detectorCount": 1, + "severity": "info", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } }, "head-with-sort-control": { "detectorCount": 0, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index 3d39fa2e3dc..d96142d62e0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -37,6 +37,10 @@ "role": "trigger", "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" }, + "rex-capture-name-hyphen": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, "rex-capture-name-alphanumeric-control": { "role": "control", "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email, username, domain | head 1" @@ -62,6 +66,21 @@ } } }, + "rex-capture-name-hyphen": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, "backend": { @@ -93,6 +112,21 @@ } } }, + "rex-capture-name-hyphen": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } + } + } + }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, "backend": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 018d086fec3..345742aa9d8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -3,8 +3,11 @@ "ruleId": "multisearch-min-subsearch", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["multisearchCommand", "subSearch"], - "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", + "requiredParserRules": [ + "multisearchCommand", + "subSearch" + ], + "notes": "Query-initial (no leading pipe) on purpose \u2014 see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", "wiring": { "detector": "multisearch-min-subsearch", "enabled": true, @@ -12,11 +15,18 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0" } + "appliesTo": { + "minVersion": "3.4.0" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -27,6 +37,10 @@ "role": "trigger", "query": "multisearch [ search source={{index}} ]" }, + "multisearch-single-subsearch-with-where": { + "role": "trigger", + "query": "multisearch [ search source={{index}} | where age > 30 ]" + }, "multisearch-two-subsearches-control": { "role": "control", "query": "multisearch [ search source={{index}} ] [ search source={{index}} ]" @@ -42,12 +56,39 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + } + }, + "multisearch-single-subsearch-with-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } } }, "multisearch-two-subsearches-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 593bb92023d..c6fa5b162b5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -37,6 +37,10 @@ "role": "trigger", "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname" }, + "replace-wildcard-count-mismatch-reverse": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a_*\" with \"b_*\" in firstname" + }, "replace-symmetric-control": { "role": "control", "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1" @@ -62,6 +66,21 @@ } } }, + "replace-wildcard-count-mismatch-reverse": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, "replace-symmetric-control": { "detectorCount": 0, "backend": { @@ -93,6 +112,21 @@ } } }, + "replace-wildcard-count-mismatch-reverse": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } + } + } + }, "replace-symmetric-control": { "detectorCount": 0, "backend": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index 7f110cc9423..d5eadcc25f5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -3,8 +3,12 @@ "ruleId": "union-min-datasets", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["unionCommand", "unionDataset", "pplCommands"], - "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", + "requiredParserRules": [ + "unionCommand", + "unionDataset", + "pplCommands" + ], + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' \u2014 a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", "wiring": { "detector": "union-min-datasets", "enabled": true, @@ -12,11 +16,19 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.7.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -27,6 +39,10 @@ "role": "trigger", "query": "union [ source={{index}} ]" }, + "union-single-dataset-with-fields": { + "role": "trigger", + "query": "union [ source={{index}} | fields firstname ]" + }, "union-two-datasets-control": { "role": "control", "query": "union [ source={{index}} ] [ source={{index}} ]" @@ -43,12 +59,39 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + } + }, + "union-single-dataset-with-fields": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } } }, "union-two-datasets-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 4f436a383cb..6479fd2b1b1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -11,11 +11,18 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0" } + "appliesTo": { + "minVersion": "3.4.0" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -26,6 +33,10 @@ "role": "trigger", "query": "source={{index}} | eventstats rank() as rank_value" }, + "eventstats-dense-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats dense_rank() as rank_value" + }, "eventstats-avg-control": { "role": "control", "query": "source={{index}} | eventstats avg(age) as avg_age" @@ -51,9 +62,30 @@ } } }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } }, @@ -75,9 +107,30 @@ } } }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } }, @@ -92,13 +145,37 @@ "httpStatus": 400, "body": { "status": 400, - "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } } } }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } From 7fa1bdb213f1616173921b603f07ebeb2849ae8f Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:00:14 -0700 Subject: [PATCH 54/78] test(ci): probe REST client connectivity from a JVM Seven hypotheses for the 2.19 observation-leg timeout have each been refuted by observation: the index-wipe race, a Gradle cluster fallback, HTTP/2 negotiation, FIPS, the bundled plugin set, a port collision, and address family. What is established is narrow and contradictory: the engine is alive and logging throughout, its publish address and network topology are identical to the passing 3.5.0 leg, the Gradle args and task graphs are byte-identical, curl reaches every endpoint the framework calls in 0s from the same runner -- and GET _nodes/plugins from the test JVM never returns. curl has said everything it can. This probe asks the JVM instead, one layer at a time against the same address: raw TCP connect, then HttpURLConnection, then the real OpenSearch RestClient on each endpoint the framework itself calls, each timed and bounded at 15s. Whichever layer stops working localizes the fault -- network, JDK HTTP stack, async client, or a specific response. It deliberately does not extend the framework's base class, since that base class is what hangs; inheriting it would reproduce the symptom instead of isolating it. Reports rather than asserts (the leg is already failing and the evidence is the point), except that a failed TCP connect is fatal because nothing below it would mean anything. Wired into the compiled leg with continue-on-error so a diagnostic can never be what decides the leg. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 32 ++- .../remote/RestClientConnectivityProbeIT.java | 207 ++++++++++++++++++ 2 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 1a8aefbb535..19eae573de0 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -393,12 +393,6 @@ jobs: echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" fi done - # The requests above all succeed over HTTP/1.1 (curl's default) yet the - # test client still times out on the same endpoint. The remaining - # difference is protocol negotiation: RestClientBuilder builds an - # HttpAsyncClient with no version policy, which in HttpClient 5.x means - # h2-with-upgrade. Probe an explicit h2 upgrade to see whether this engine - # completes it. start=$(date +%s) if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then @@ -406,6 +400,14 @@ jobs: else echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" fi + # Response SIZE is the last untested difference. curl streams the body and + # does not care; the test framework calls entityAsMap on it, and + # _nodes/plugins on an engine with many bundled plugins is large. Record the + # sizes so a size-dependent hang is visible rather than inferred. + for path in "_nodes/plugins" "_nodes" "_cat/plugins"; do + bytes=$(curl -sS --max-time 30 "http://localhost:9200/${path}" | wc -c) + echo "size probe: ${path} -> ${bytes} bytes" + done - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 @@ -413,6 +415,24 @@ jobs: distribution: 'temurin' java-version: 21 + # The curl probes above reach this engine instantly, yet the contract IT's own + # client times out on the same endpoint before any test body runs. Everything + # curl can tell us has been exhausted, so run the probe from a JVM: raw TCP, + # then HttpURLConnection, then the real OpenSearch RestClient per endpoint. + # Whichever layer stops working is the answer. + # + # `continue-on-error` because this is a diagnostic: its findings must not be + # what decides the leg, and the contract step below is still the real check. + - name: Probe REST client connectivity from a JVM + continue-on-error: true + run: | + set -uo pipefail + ./gradlew :integ-test:integTestRemote \ + --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD' || true + - name: Run contract observation against engine ${{ matrix.version }} run: | set -euo pipefail diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java new file mode 100644 index 00000000000..0be3228ff4d --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -0,0 +1,207 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.util.Timeout; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; + +/** + * Connectivity probe for {@code tests.rest.cluster}, used to diagnose why the REST test client + * cannot reach some engine versions that plain HTTP clients reach fine. + * + *

Context: on the PPL lint multi-version matrix the 2.19.0 observation leg fails with {@code + * SocketTimeoutException} after the full 60s response timeout, thrown from {@code + * OpenSearchRestTestCase.initClient} on its {@code GET _nodes/plugins} call — before any test body + * runs. From the same runner, {@code curl} against the same endpoint on the same address returns + * HTTP 200 in 0s. The 3.5.0 leg, with byte-identical Gradle args, network topology, publish address + * and task graph, passes. Seven hypotheses (index-wipe race, Gradle cluster fallback, HTTP/2 + * negotiation, FIPS, plugin set, port collision, address family) have each been refuted by + * observation. + * + *

This class deliberately does NOT extend the test framework's base class: that base class is + * what hangs, so inheriting it would reproduce the symptom without isolating the cause. Instead it + * walks up the stack one layer at a time against the same address, so a single run says exactly + * which layer stops working: + * + *

    + *
  1. raw TCP connect — is the port reachable from this JVM at all? + *
  2. {@code HttpURLConnection} — does the JDK's own HTTP stack get a response? + *
  3. {@code RestClient} with default settings — does the OpenSearch async client work? + *
  4. {@code RestClient} on the endpoints the framework itself calls, timed individually. + *
+ * + *

Every step is time-bounded and reports rather than asserts, because the point is to collect + * evidence from a leg that is already failing. The one assertion is that step 1 succeeded: if the + * JVM cannot open a socket, nothing below it means anything. + * + *

Run with: {@code ./gradlew :integ-test:integTestRemote --tests + * '*RestClientConnectivityProbeIT' -Dtests.rest.cluster=localhost:9200} + */ +public class RestClientConnectivityProbeIT { + + /** Bound well below the framework's 60s so a hang is visibly a hang, not a wait. */ + private static final Timeout PROBE_TIMEOUT = Timeout.ofSeconds(15); + + private static final String[] FRAMEWORK_ENDPOINTS = { + // The exact call OpenSearchRestTestCase.initClient makes, and the one that hangs. + "_nodes/plugins", + // What the wipe in OpenSearchSQLRestTestCase.wipeAllOpenSearchIndices calls next. + "_cat/indices?format=json&expand_wildcards=all", + // A trivial response, to separate "any request" from "this request". + "_cluster/health", + // The PPL endpoint the contract actually needs, so a pass here means the leg could work. + "_plugins/_ppl/_grammar", + }; + + @Test + public void probeConnectivity() { + String cluster = System.getProperty("tests.rest.cluster"); + if (cluster == null || cluster.isEmpty()) { + log("SKIP: -Dtests.rest.cluster not set"); + return; + } + String hostPort = cluster.split(",")[0]; + int sep = hostPort.lastIndexOf(':'); + String host = hostPort.substring(0, sep); + int port = Integer.parseInt(hostPort.substring(sep + 1)); + log("probing " + host + ":" + port); + + boolean tcpOk = probeRawSocket(host, port); + probeHttpUrlConnection(host, port); + probeRestClient(host, port); + + // Only a hard failure here is fatal: without a socket the rest is noise. + if (!tcpOk) { + throw new AssertionError("could not open a TCP connection to " + host + ":" + port); + } + } + + /** Layer 1: can this JVM open a socket to the published port? */ + private boolean probeRawSocket(String host, int port) { + long start = System.nanoTime(); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), (int) PROBE_TIMEOUT.toMilliseconds()); + log("tcp connect OK in " + millis(start) + "ms (localAddr=" + socket.getLocalAddress() + ")"); + return true; + } catch (Exception e) { + log("tcp connect FAILED after " + millis(start) + "ms: " + describe(e)); + return false; + } + } + + /** + * Layer 2: the JDK's own blocking HTTP stack. If this works while {@code RestClient} does not, + * the problem is in the async client rather than in the network or the engine. + */ + private void probeHttpUrlConnection(String host, int port) { + long start = System.nanoTime(); + try { + URL url = new URL("http://" + host + ":" + port + "/_nodes/plugins"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + connection.setReadTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + int status = connection.getResponseCode(); + long bytes = drain(connection); + log( + "HttpURLConnection _nodes/plugins OK in " + + millis(start) + + "ms: HTTP " + + status + + ", " + + bytes + + " bytes"); + connection.disconnect(); + } catch (Exception e) { + log("HttpURLConnection _nodes/plugins FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + + /** + * Layer 3: the real {@code RestClient}, built the way the framework builds it (defaults only, no + * credentials or TLS since these legs are plain HTTP), then each framework endpoint in turn. + * + *

Timed per endpoint: a uniform failure means the client cannot talk to this engine at all, + * while one slow endpoint among fast ones means the response itself is the problem. + */ + private void probeRestClient(String host, int port) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + // The framework sets this too; without it a deprecation warning header can turn into a + // failure and confuse the diagnosis. + .setStrictDeprecationMode(false); + + try (RestClient client = builder.build()) { + for (String endpoint : FRAMEWORK_ENDPOINTS) { + long start = System.nanoTime(); + try { + Response response = client.performRequest(new Request("GET", "/" + endpoint)); + long bytes = response.getEntity() == null ? 0 : response.getEntity().getContentLength(); + log( + "RestClient " + + endpoint + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode() + + ", " + + bytes + + " bytes"); + } catch (Exception e) { + log("RestClient " + endpoint + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + } catch (IOException e) { + log("RestClient could not be built/closed: " + describe(e)); + } + } + + private static long drain(HttpURLConnection connection) throws IOException { + byte[] buffer = new byte[8192]; + long total = 0; + try (var stream = connection.getInputStream()) { + int read; + while ((read = stream.read(buffer)) != -1) { + total += read; + } + } + return total; + } + + /** Full cause chain: the outer message alone hides which layer actually gave up. */ + private static String describe(Throwable error) { + List chain = new ArrayList<>(); + for (Throwable current = error; current != null; current = current.getCause()) { + chain.add(current.getClass().getSimpleName() + "(" + current.getMessage() + ")"); + if (current.getCause() == current) { + break; + } + } + return String.join(" <- ", chain); + } + + private static long millis(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } + + private static void log(String message) { + // stdout so it lands in the Gradle test output the CI job already prints. + System.out.println("[rest-connectivity-probe] " + message); + } +} From ca2aeb6f412e49c8f2037effdc0a3fb3f00c5882 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:19:06 -0700 Subject: [PATCH 55/78] test(ppl-lint): run the whole contract corpus on every PR, not just the PR subset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five contracts declared `schedule: "nightly"`, so a pull_request run skipped them: dedup-consecutive-unsupported, disabled-join-type, division-by-zero, field-validation and head-without-sort. That left the required check scoring 3 of 19 rows on a PR, and it hid the triggers the multi-version relaxation rollup depends on — a rule with no scored case contributes no trigger census, so full-vs-partial cannot be judged for it on a PR at all. All 11 contracts now declare `schedule: "pr"`. A PR run reaches the whole corpus: 35 queries, 19 scored on the compiled surface (the other 16 are the runtime-bundle-only contracts, not-applicable there as before), up from 3 of 19. This DOES make the four advisory rules blocking, and that is a deliberate accepted trade rather than an oversight. Neither PplLintRuleValidationIT nor run-frontend-contract.mjs consults the manifest's `enforced` list — a contract that runs is a hard assertion — so `schedule` was the only thing keeping them non-blocking. Their oracles are genuinely weaker than the error rules': an advisory rule's query SUCCEEDS, so the contract can only assert a result shape or plain acceptance, and dedup-consecutive in particular depends on the Calcite-to-v2 fallback staying enabled. If one of them goes red, check the oracle before editing a rule. The manifest description, the IT javadoc and the README all claimed the split controlled blocking. Corrected: `enforced` / `nonEnforcing` record oracle quality and review status — how much to trust a red result — not whether one can occur. The schedule filter itself is kept, since it remains the only way to hold a new contract back from PR runs while its oracle settles. Verified: the full corpus exits 0 on the PR schedule with zero skips, against a live 3.8 engine and the real ACCOUNT fixture. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 12 +++++-- .../dedup-consecutive-unsupported.spec.json | 2 +- .../contracts/disabled-join-type.spec.json | 36 +++++++++++++++---- .../contracts/division-by-zero.spec.json | 2 +- .../contracts/field-validation.spec.json | 2 +- .../contracts/head-without-sort.spec.json | 2 +- .../ppl-lint/contracts/manifest.json | 10 +++--- scripts/ppl-lint/README.md | 19 ++++++++-- 8 files changed, 66 insertions(+), 19 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index e9d1ad6a947..6a2de542ab6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -61,8 +61,16 @@ * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. * - *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): PR runs only the - * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus. + *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips + * contracts declaring {@code schedule: "nightly"}, while nightly runs the full corpus. Every + * contract in the corpus currently declares {@code schedule: "pr"}, so the two are equivalent + * today; the filter stays because it is the only mechanism for holding a new contract back from + * PR runs while its oracle is still settling. + * + *

Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's + * {@code enforced} list — that list records oracle quality and review status, not blocking + * behavior. Adding a contract, or moving one onto the PR schedule, makes it capable of failing the + * required check. */ public class PplLintRuleValidationIT extends PPLIntegTestCase { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index fb9d44de7c3..fffdce393dd 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "dedup-consecutive-unsupported", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "dedup-consecutive-unsupported", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index 25cd3232e88..d2d5df24260 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "disabled-join-type", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "disabled-join-type", "enabled": true, @@ -13,8 +13,14 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false, "allJoinTypesAllowed": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false, + "allJoinTypesAllowed": false + } }, "frontendContext": { "isCalcite": true @@ -44,7 +50,13 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } } }, "cross-join-disabled": { @@ -53,12 +65,24 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } } }, "inner-join-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 0303e32e2a5..e1063eddc7d 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -3,7 +3,7 @@ "ruleId": "division-by-zero", "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "division-by-zero", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 984fcf851c1..5814d837249 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "field-validation", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "field-validation", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 68cd387a88c..56c272abc9c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "head-without-sort", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "head-without-sort", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index a8b031315d6..d316c1e8de1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,6 +1,6 @@ { "schemaVersion": 3, - "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", + "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus. EVERY contract now declares `schedule: \"pr\"`, so every contract runs \u2014 and asserts \u2014 on every pull request: neither reader consults `enforced`, so any contract that runs is a hard assertion. The `enforced` / `nonEnforcing` lists below therefore describe oracle QUALITY and review status, not whether a mismatch blocks (design \u00a75.1, \u00a75.2). They are what a reviewer should read when judging how much to trust a red result.", "contracts": [ "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", @@ -38,9 +38,9 @@ "dedup-consecutive-unsupported.spec.json" ], "notes": { - "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required single-version validation-result check.", - "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog — the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", - "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", - "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. The most trustworthy oracles in the corpus \u2014 a mismatch here is almost certainly a real drift.", + "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog \u2014 the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design \u00a75.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", + "nonEnforcing": "Warning / info / advisory / result-shape rules. Their oracle is weaker than a clean rejection (an advisory rule's query SUCCEEDS, so the contract asserts a result shape or mere acceptance), which makes them likelier to move for reasons unrelated to the lint rule \u2014 dedup-consecutive, for instance, depends on the Calcite-to-v2 fallback staying enabled. They ran nightly-only until every contract moved to the PR schedule so the multi-version rollup sees a full trigger census on each PR; they now block like any other contract, and a red result here warrants checking the oracle before editing a rule." } } diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index e1e6a1439d6..688d3439cc7 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -49,6 +49,12 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j | `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | | `schedule` (nightly) | full corpus + coverage | `main` | No | +Every contract declares `schedule: "pr"`, so a PR run exercises the **whole corpus** +— 11 rules, 35 queries. A contract that runs also asserts: neither the IT nor the +detector runner consults the manifest's `enforced` list, so any contract on the PR +schedule can fail the required check. Keep that in mind when adding one; a new +contract whose oracle has not settled should say `schedule: "nightly"` until it has. + `workflow_dispatch` inputs: - `osd_repo` — the OSD repository to check out, for validating an unmerged change @@ -184,8 +190,17 @@ rule cannot be validated end to end. single-version `enforced` set because their backend oracle is a semantic `Field [...] not found.` rejection they share with each other rather than a rule-unique grammar rejection. -- `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the - nightly schedule for coverage and never block a PR. +- `nonEnforcing` — warning/info/advisory/result-shape rules. Their oracle is weaker + than a clean rejection: an advisory rule's query *succeeds*, so the contract can + only assert a result shape or plain acceptance, which is likelier to move for + reasons unrelated to the lint rule (`dedup-consecutive` depends on the + Calcite-to-v2 fallback staying on). These ran nightly-only until every contract + moved to the PR schedule, so they now block like any other. A red result here is + worth checking against the oracle before editing a rule. + +The `enforced` / `nonEnforcing` split therefore describes **oracle quality and review +status, not blocking behavior** — it tells a reviewer how much to trust a red result, +not whether one can occur. ## Multi-version validation From 719cc4691a86649eb933ae0a95751d066e06d55a Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:48:22 -0700 Subject: [PATCH 56/78] fix(ci): make the connectivity probe actually run The probe reported nothing: its step logged BUILD SUCCESSFUL in 2m45s with no probe output at all. It used JUnit 4's org.junit.Test while this module runs useJUnitPlatform(), so the class was collected as zero tests and the task succeeded vacuously -- the same shape of failure the PPL lint contract itself guards against, in the diagnostic meant to explain it. Switch to org.junit.jupiter.api.Test, matching PplLintRuleValidationIT. Also stop the step's grep from hiding evidence: add --info and match 'tests completed' and 'No tests found' so a zero-test run is visible next time instead of reading as a pass. Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-multiversion-validation.yml | 3 ++- .../sql/calcite/remote/RestClientConnectivityProbeIT.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 19eae573de0..d7ec99ba51d 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -431,7 +431,8 @@ jobs: --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ -Dtests.rest.cluster=localhost:9200 \ -Dtests.cluster=localhost:9200 \ - -Dtests.clustername=docker-cluster 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD' || true + -Dtests.clustername=docker-cluster \ + --info 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD|tests? completed|No tests found' || true - name: Run contract observation against engine ${{ matrix.version }} run: | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 0be3228ff4d..6ca497986ce 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -14,7 +14,7 @@ import java.util.List; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.util.Timeout; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; From cdf730fe322b5eb379dbfbbcafd0e06af75a054f Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:00:42 -0700 Subject: [PATCH 57/78] fix(ppl-lint): scope the trigger differential to rules the engine actually rejects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving every contract onto the PR schedule turned the required check red with 8 failures, 7 of them this: the trigger cross-check paired the detector against `be.rejected`, which is only meaningful for a `rejection`-kind rule. An advisory rule flags a query the engine runs happily — head-without-sort marks non-determinism, division-by-zero marks a silent null, dedup-consecutive succeeds via the Calcite-to-v2 fallback. For those, "detector flagged, backend accepted" is the rule working as designed, so the check failed every advisory trigger unconditionally, including ones that predate this branch. That, not runtime cost, is the structural reason those contracts could only ever run nightly; I had attributed it to cost and weaker oracles, which was wrong. The check now runs only when the contract declares `backend.kind: "rejection"`. The contracts already carry that distinction, so this reads data that exists rather than adding a flag, and rejection rules are completely unaffected. Advisory triggers keep full coverage from the two other assertions, which is why relaxing the pairing is safe rather than merely convenient: - the backend-kind check still fires if the engine starts REJECTING a query the contract pinned as accepted; - the `detectorCount` assertion still fires if the detector stops flagging it. Every trigger in the corpus pins detectorCount: 1 regardless of kind, so a silent advisory detector is still caught. Verified by replaying the failed CI run's own artifacts (backend-report.json, target.json, ppl-grammar-bundle.json downloaded from run 30289514275): same inputs go from 6 failures to 0 on the compiled surface. Confirmed still selective by tampering the backend report to make `disabled-join-type/right-join-disabled` — a rejection rule — look accepted: that fails with both the backend-kind and the trigger assertion. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/run-frontend-contract.mjs | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 6f7ecd6de50..dcdfe5e4a1d 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -685,14 +685,38 @@ function main() { `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` ); } - // Trigger/control cross-check against the detector's own verdict. + // Trigger cross-check: a trigger the detector flags must be one the engine + // ALSO objects to — but only where the contract claims the engine objects + // at all. + // + // For a `rejection` rule the two coincide: detector flags <-> engine + // rejects, and a disagreement means one side drifted. That is the original + // check and it is unchanged. + // + // An ADVISORY rule is different by design. It flags a query the engine + // runs happily: `head-without-sort` marks non-determinism, + // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via + // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that + // rule working, not drift — so pairing the detector against `be.rejected` + // failed every advisory trigger unconditionally. That, not runtime cost, + // is the structural reason those contracts could only run nightly. + // + // The contracts already carry the distinction in `backend.kind`, so this + // reads data that exists rather than adding a flag. Advisory triggers keep + // full coverage from the other two assertions: the backend-kind check above + // fires if the engine starts REJECTING a query pinned as accepted, and the + // `detectorCount` assertion fires if the detector stops flagging it. Only + // the pairing rule is scoped to the rules it makes sense for. const detectorFlagged = actual > 0; - if (role === 'trigger' && detectorFlagged !== !!be.rejected) { + if (role === 'trigger' && expectRejected && detectorFlagged !== !!be.rejected) { failures.push( `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` ); } + // A control must pass on both sides regardless of kind: it is a valid + // query the rule has to stay quiet on. Unlike a trigger, that claim does + // not vary with `backend.kind`. if (role === 'control' && (detectorFlagged || be.rejected)) { failures.push( `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + From c3bee56bc26f0051d41f6013a6a4b4aefd756bff Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:00:59 -0700 Subject: [PATCH 58/78] fix(test): make the connectivity probe discoverable Second vacuous pass from the same probe, different cause. The JUnit 5 switch was wrong: integTestRemote runs the default JUnit 4 runner -- only integJdbcTest calls useJUnitPlatform() -- so the jupiter @Test made the class undiscoverable and Gradle reported 'No tests found for given includes: [**/*IT.class]' while the step still looked fine. But the JUnit 4 annotation alone was not enough either. Gradle only discovers an IT that inherits a runner from a framework base class; a standalone class has none and is collected as zero tests. That is why the FIRST version reported nothing despite compiling, matching the include pattern, and having its .class file in place. Extend OpenSearchTestCase: it supplies the randomized-testing runner but builds no REST client, so the probe is discovered without inheriting the client setup that hangs -- which was the whole reason for not extending the REST base class. Verified locally against a live cluster, all four layers reporting: tcp connect OK in 2ms HttpURLConnection _nodes/plugins OK in 8ms: HTTP 200, 9456 bytes RestClient _nodes/plugins OK in 54ms: HTTP 200, 9456 bytes RestClient _plugins/_ppl/_grammar OK in 20ms: HTTP 200, 248625 bytes Note for anyone running it by hand: the opensearch.rest-test plugin requires tests.rest.cluster, tests.cluster and tests.clustername to be all-null or all-non-null, or the project fails to configure before any test runs. Signed-off-by: Hanyu Wei --- .../remote/RestClientConnectivityProbeIT.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 6ca497986ce..0a800fe94a1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -14,11 +14,12 @@ import java.util.List; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.util.Timeout; -import org.junit.jupiter.api.Test; +import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; +import org.opensearch.test.OpenSearchTestCase; /** * Connectivity probe for {@code tests.rest.cluster}, used to diagnose why the REST test client @@ -33,10 +34,14 @@ * negotiation, FIPS, plugin set, port collision, address family) have each been refuted by * observation. * - *

This class deliberately does NOT extend the test framework's base class: that base class is - * what hangs, so inheriting it would reproduce the symptom without isolating the cause. Instead it - * walks up the stack one layer at a time against the same address, so a single run says exactly - * which layer stops working: + *

It extends {@link OpenSearchTestCase}, NOT the REST base class. Two constraints meet here: + * {@code integTestRemote} runs the default JUnit 4 runner (only {@code integJdbcTest} calls {@code + * useJUnitPlatform()}), and Gradle only discovers an IT that inherits a runner from a framework + * base class — a standalone class is silently collected as ZERO tests, which is how the first + * version of this probe "passed" while reporting nothing. {@code OpenSearchTestCase} supplies that + * runner but builds no REST client, so discovery works without inheriting the hang under + * investigation. Instead of the framework's client setup, this walks up the stack one layer at a + * time against the same address, so a single run says exactly which layer stops working: * *

    *
  1. raw TCP connect — is the port reachable from this JVM at all? @@ -52,7 +57,7 @@ *

    Run with: {@code ./gradlew :integ-test:integTestRemote --tests * '*RestClientConnectivityProbeIT' -Dtests.rest.cluster=localhost:9200} */ -public class RestClientConnectivityProbeIT { +public class RestClientConnectivityProbeIT extends OpenSearchTestCase { /** Bound well below the framework's 60s so a hang is visibly a hang, not a wait. */ private static final Timeout PROBE_TIMEOUT = Timeout.ofSeconds(15); From 624933c8d98e96c8f3de6e571fcce011f3f46d4c Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:14:53 -0700 Subject: [PATCH 59/78] test(ci): validate the latest patch of each line, and probe client variants Two fixes. 1. Test the LATEST PATCH, not .0. The compiled matrix pinned 2.19.0 while the published line is at 2.19.6 -- six patches stale, so it validated an engine no user runs and would have attributed any bug fixed in between to the whole 2.19 line. Checked every line on Docker Hub: 2.19.6 is the only stale pin; 3.0.0 / 3.5.0 / 3.6.0 / 3.7.0 already are their lines' latest patch. Comments now say to keep them latest rather than .0. 2. Probe client variants. The previous probe localized the 2.19 timeout precisely: tcp connect OK 4ms HttpURLConnection _nodes/plugins OK 27ms HTTP 200, 15844 bytes RestClient _nodes/plugins FAILED 15396ms SocketTimeoutException RestClient _cluster/health FAILED 15025ms SocketTimeoutException Every endpoint fails, including a 459-byte health response, while the JDK's own HTTP stack succeeds against the same URL. So the fault is in how the async client speaks to this engine -- not the network, engine, response size, or any one endpoint, which is why seven log-derived theories all missed it. Rather than guess again, run candidate configurations side by side against the same endpoint: FORCE_HTTP_1, NEGOTIATE, FORCE_HTTP_2, and a fresh single- connection manager. Whichever succeeds names the fix; if none do, the client cannot be configured around it and the answer is a client/engine version constraint instead. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 9 ++- .../remote/RestClientConnectivityProbeIT.java | 78 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index d7ec99ba51d..f765c0d132e 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -84,6 +84,9 @@ env: # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must # be >= 3.6.0 (the _grammar endpoint floor) and must have a published # distribution image. + # Latest patch of each line (3.6.0 and 3.7.0 ARE the latest patches today; bump + # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch + # exists — that would validate an engine no user runs). ENGINE_VERSIONS: '["3.6.0","3.7.0"]' # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. # @@ -97,7 +100,11 @@ env: # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are # reported not-applicable. Nightly only — see the `compiled_versions` input to # run one ad hoc. - COMPILED_ENGINE_VERSIONS: '["2.19.0","3.0.0","3.5.0"]' + # + # Always the LATEST PATCH of each line, never `.0`. A user on 2.19 is on + # 2.19.6, so validating 2.19.0 tests an engine nobody runs and attributes any + # bug fixed in between to the whole line. + COMPILED_ENGINE_VERSIONS: '["2.19.6","3.0.0","3.5.0"]' jobs: # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 0a800fe94a1..517503cc148 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -89,6 +89,7 @@ public void probeConnectivity() { boolean tcpOk = probeRawSocket(host, port); probeHttpUrlConnection(host, port); probeRestClient(host, port); + probeClientVariants(host, port); // Only a hard failure here is fatal: without a socket the rest is noise. if (!tcpOk) { @@ -201,6 +202,83 @@ private static String describe(Throwable error) { return String.join(" <- ", chain); } + /** + * Layer 4: candidate fixes, each a one-line change from the default client, all against the same + * endpoint on the same engine. + * + *

    The default async client times out on EVERY endpoint here — including a 459-byte {@code + * _cluster/health} — while {@code HttpURLConnection} against the same URL returns 200 in 27ms. So + * the fault is in how the async client speaks to this engine, not in the network, the engine, the + * response size, or any one endpoint. Each variant isolates one suspect; whichever succeeds names + * the fix, and if none do, the client cannot be configured around it. + */ + private void probeClientVariants(String host, int port) { + // Forcing HTTP/1.1 up front, rather than letting HttpClient 5.x negotiate h2. + variant( + host, + port, + "FORCE_HTTP_1", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_1))); + // Same, but negotiating h2 explicitly, to tell "policy matters" from "1.1 specifically works". + variant( + host, + port, + "NEGOTIATE", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.NEGOTIATE))); + // FORCE_HTTP_2, to confirm the direction of any protocol effect rather than assume it. + variant( + host, + port, + "FORCE_HTTP_2", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_2))); + // A fresh connection manager: rules out connection reuse/pooling against this engine. + variant( + host, + port, + "fresh-conn-manager", + b -> + b.setHttpClientConfigCallback( + c -> + c.setConnectionManager( + org.apache.hc.client5.http.impl.nio + .PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(1) + .setMaxConnTotal(1) + .build()))); + } + + /** Run one client variant against {@code _cluster/health} — the smallest response available. */ + private void variant( + String host, + int port, + String name, + java.util.function.UnaryOperator tune) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + .setStrictDeprecationMode(false); + long start = System.nanoTime(); + try (RestClient client = tune.apply(builder).build()) { + Response response = client.performRequest(new Request("GET", "/_cluster/health")); + log( + "variant " + + name + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode()); + } catch (Exception e) { + log("variant " + name + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + private static long millis(long startNanos) { return (System.nanoTime() - startNanos) / 1_000_000; } From b64d2f320fe23f105440fa36f1c70d6a3e8db9d9 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:36:07 -0700 Subject: [PATCH 60/78] fix(test): tolerate a pre-Calcite engine in observe-only mode With the version pin corrected to 2.19.6, the leg gets past the client entirely and fails on the real incompatibility instead: PUT /_cluster/settings -> 400 persistent setting [plugins.calcite.enabled], not recognized Calcite is a 3.x feature and SQLIntegTestCase.init() sets that setting unconditionally, so every 2.x leg aborts before seeding a fixture or running a query. In observe-only mode, catch exactly that failure and continue without the setting: a pre-Calcite engine is a legitimate thing to observe, and each contract's own frontendContext.isCalcite already states what the linter should assume there. super.init() aborts partway when it throws, so redo the version-independent half (increaseMaxCompilationsRate). Matched narrowly -- on the setting name plus "not recognized", not on any 400 -- so a genuinely broken settings call on a Calcite-capable engine still fails rather than being waved through as "old engine". Asserting mode is unchanged: the required check runs against the PR's own build, where a missing Calcite setting is a real problem. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 6a2de542ab6..336bff4bf68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -64,8 +64,8 @@ *

    The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips * contracts declaring {@code schedule: "nightly"}, while nightly runs the full corpus. Every * contract in the corpus currently declares {@code schedule: "pr"}, so the two are equivalent - * today; the filter stays because it is the only mechanism for holding a new contract back from - * PR runs while its oracle is still settling. + * today; the filter stays because it is the only mechanism for holding a new contract back from PR + * runs while its oracle is still settling. * *

    Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's * {@code enforced} list — that list records oracle quality and review status, not blocking @@ -111,8 +111,29 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { @Override public void init() throws Exception { - super.init(); - enableCalcite(); + // Calcite is a 3.x engine feature: on 2.x the cluster rejects + // `plugins.calcite.enabled` outright with "not recognized", and the base class's + // init() sets it unconditionally. In observe-only mode (the multi-version + // matrix) that must not abort the leg — a pre-Calcite engine is a legitimate + // thing to observe, and the contracts' own `frontendContext.isCalcite` already + // describes what the linter should assume there. + // + // Asserting mode keeps the strict behavior: the required check runs against the + // PR's own build, where a missing Calcite setting is a real problem. + try { + super.init(); + enableCalcite(); + } catch (Exception e) { + if (!observeOnly || !isUnrecognizedCalciteSetting(e)) { + throw e; + } + System.err.println( + "[ppl-lint] engine does not support the Calcite setting; observing without it: " + + e.getMessage()); + // super.init() aborted partway, so redo the part that is version-independent. + increaseMaxCompilationsRate(); + } + // Fall through to fixture seeding either way. // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { try { @@ -639,6 +660,29 @@ private void exportGrammarArtifacts(List failures) { } /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ + /** + * True when a failure is the cluster rejecting {@code plugins.calcite.enabled} because it does + * not know that setting — i.e. a pre-Calcite (2.x) engine. + * + *

    Deliberately narrow: matched on the setting name plus "not recognized" rather than on any + * 400, so a genuinely broken settings call on a Calcite-capable engine still fails the run + * instead of being waved through as "old engine". + */ + private static boolean isUnrecognizedCalciteSetting(Throwable error) { + for (Throwable current = error; current != null; current = current.getCause()) { + String message = current.getMessage(); + if (message != null + && message.contains(Settings.Key.CALCITE_ENGINE_ENABLED.getKeyValue()) + && message.contains("not recognized")) { + return true; + } + if (current.getCause() == current) { + break; + } + } + return false; + } + private void writeTargetManifest(String grammarHash, List failures) { writeTargetManifest(grammarHash, "", failures); } From d8a70b114c7717bd40a523d2f3c2d4da1056860f Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:49:03 -0700 Subject: [PATCH 61/78] fix(test): skip the whole Calcite settings family on a pre-Calcite engine The init() tolerance worked -- the log shows 'observing without it' -- but the leg still failed, because applyClusterSettings re-applies the same setting per contract from backendFixture.clusterSettings, undoing it once per contract. Every setting in that block is Calcite-family (calcite, calciteFallback, allJoinTypesAllowed), and a pre-Calcite engine rejects all of them identically. So record support once in init() and skip the block, rather than wrapping each call in the same catch. Verified in the same run that the earlier fix landed: 2.19.6's REST client is healthy (_nodes/plugins 312ms, _cluster/health 6ms, FORCE_HTTP_1 / NEGOTIATE / fresh-conn-manager all OK; only FORCE_HTTP_2 fails, correctly, since 2.19 has no h2). There was never a client bug -- 2.19.0 was six patches stale. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 336bff4bf68..7ed5e6ad989 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -102,6 +102,14 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + /** + * Whether this cluster recognizes the Calcite settings at all. False on a pre-Calcite (2.x) + * engine, where every {@code plugins.calcite.*} write is rejected as "not recognized". + * Established once in {@code init()} and honored by {@code applyClusterSettings}, so the whole + * family is skipped rather than retried and failed once per contract. + */ + private boolean calciteSettingsSupported = true; + /** * Index fixtures that could not be created on this engine (observe-only mode only). Contracts * that need one are reported as {@code outcome: "error"} instead of as engine behavior, because @@ -127,8 +135,9 @@ public void init() throws Exception { if (!observeOnly || !isUnrecognizedCalciteSetting(e)) { throw e; } + calciteSettingsSupported = false; System.err.println( - "[ppl-lint] engine does not support the Calcite setting; observing without it: " + "[ppl-lint] engine does not support the Calcite settings; observing without them: " + e.getMessage()); // super.init() aborted partway, so redo the part that is version-independent. increaseMaxCompilationsRate(); @@ -738,6 +747,13 @@ private List applyClusterSettings(JSONObject fixture) throws IOException if (settings == null) { return applied; } + // Every setting below is Calcite-family, and a pre-Calcite engine rejects all of + // them the same way. `init()` already established whether this cluster knows + // them, so skip the whole block rather than fail per contract — otherwise the + // tolerance added there is undone here, once per contract. + if (!calciteSettingsSupported) { + return applied; + } if (settings.has("calcite")) { if (settings.getBoolean("calcite")) { enableCalcite(); From e7ce252072b120dac3c67412ba06274e520e04b1 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 13:07:09 -0700 Subject: [PATCH 62/78] revert(test): un-pin the REST client from HTTP/1.1 Reverts 4c80f7b91. That commit claimed HTTP/2 negotiation as the cause of the 2.19 observation-leg timeout. It was not, and the pin fixed nothing. The real cause was the version pin: the leg tested opensearchproject/opensearch: 2.19.0, six patches behind the 2.19.6 the line actually ships. On 2.19.6 the unmodified client is healthy -- _nodes/plugins in 312ms, _cluster/health in 6ms -- and the variant probe confirms it is not protocol-related at all: FORCE_HTTP_1 OK 18ms NEGOTIATE OK 21ms fresh-conn-manager OK 12ms FORCE_HTTP_2 FAILED (correctly: 2.19 has no h2) Since NEGOTIATE -- the default -- works, forcing 1.1 was a no-op dressed as a fix, and leaving it in would have suggested a protocol constraint that does not exist. Signed-off-by: Hanyu Wei --- .../sql/legacy/OpenSearchSQLRestTestCase.java | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 0abe1f4ab7c..267adea43c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; @@ -29,7 +28,6 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; -import org.apache.hc.core5.http2.HttpVersionPolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.LogManager; @@ -257,39 +255,12 @@ protected static void configureClient(RestClientBuilder builder, Settings settin credentialsProvider.setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password.toCharArray())); - return forceHttp11( - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); }); - } else { - builder.setHttpClientConfigCallback(OpenSearchSQLRestTestCase::forceHttp11); } OpenSearchRestTestCase.configureClient(builder, settings); } - /** - * Pin a client to HTTP/1.1. - * - *

    {@code RestClientBuilder} builds its async client with no version policy, which in - * HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against - * one that does not, the async I/O reactor stalls instead of falling back, so every request fails - * with {@code SocketTimeoutException} after the full response timeout — thrown from {@code - * AbstractSingleCoreIOReactor.execute} before a single test runs. - * - *

    Live-verified on the PPL lint multi-version matrix: an {@code --http2} probe against engine - * 3.5.0 negotiated HTTP/2 and that leg PASSED, while the same probe against 2.19.0 reported - * HTTP/1.1 and the leg timed out at exactly 60s. curl falls back cleanly; this client does not. - * These tests never need h2, so asking for 1.1 up front removes the negotiation and works across - * every supported engine line. - * - *

    Applied INSIDE each config callback rather than as its own {@code - * setHttpClientConfigCallback} call, because that setter replaces rather than accumulates: a - * separate call would silently drop the credentials or TLS configuration set here, and only on - * the paths where it matters. - */ - private static HttpAsyncClientBuilder forceHttp11(HttpAsyncClientBuilder httpClientBuilder) { - return httpClientBuilder.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1); - } - protected static void configureHttpsClient( RestClientBuilder builder, Settings settings, HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); @@ -321,13 +292,12 @@ protected static void configureHttpsClient( .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .build(); - return forceHttp11( - httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy(tlsStrategy) - .build())); + return httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build()); } catch (Exception e) { throw new RuntimeException(e); } From 4bdd24619998b132c06977f28932b7d2846cc02d Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 13:23:49 -0700 Subject: [PATCH 63/78] test(ppl-lint): pin dense_rank's 3.7 rejection wording The multi-version check flagged engine-message-changed on 3.7.0: error.reason "There was internal problem at backend" -> "Unexpected window function: dense_rank" The >=3.7.0 <3.8.0 expectation for eventstats-dense-rank was copied from the pre-3.7 epoch and never updated, while its sibling eventstats-rank in the SAME epoch already pins the function-naming wording. 3.7 names the offending function; only the pre-3.7 engines emit the generic message. Detector-side verdict is unaffected, so this is the update-contract remediation the classifier recommended -- not a rule change. Signed-off-by: Hanyu Wei --- .../unsupported-window-function-in-eventstats.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 6479fd2b1b1..75e45299c0f 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -117,7 +117,7 @@ "status": 500, "error": { "type": "UnsupportedOperationException", - "reason": "There was internal problem at backend" + "reason": "Unexpected window function: dense_rank" } } } From 39a9c79aaf115779598f7deb98f0e04a7d23743f Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 12:35:48 -0700 Subject: [PATCH 64/78] feat(ci): observe PPL lint contracts on analytics engine Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 105 +- .../workflows/ppl-lint-rule-validation.yml | 9 + ...ppl-lint-analytics-engine-ci-validation.md | 811 +++++++++++++ integ-test/build.gradle | 54 +- .../remote/PplLintRuleValidationIT.java | 1048 +++++++++++++++-- .../contracts/division-by-zero.spec.json | 11 +- scripts/ppl-lint-rule-validation.sh | 40 +- scripts/ppl-lint/README.md | 65 +- .../__tests__/aggregate-versions.test.mjs | 752 +++++++++++- scripts/ppl-lint/__tests__/annotate.test.mjs | 23 + .../__tests__/assemble-run-manifest.test.mjs | 174 +++ .../__tests__/contract-schema.test.mjs | 469 ++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 74 ++ scripts/ppl-lint/aggregate-versions.mjs | 982 ++++++++++++--- scripts/ppl-lint/annotate.mjs | 33 +- scripts/ppl-lint/assemble-run-manifest.mjs | 158 ++- scripts/ppl-lint/contract-schema.mjs | 397 +++++++ scripts/ppl-lint/drift.mjs | 278 ++++- scripts/ppl-lint/probe-discovery-backend.mjs | 7 +- scripts/ppl-lint/run-frontend-contract.mjs | 348 ++++-- 20 files changed, 5459 insertions(+), 379 deletions(-) create mode 100644 docs/dev/ppl-lint-analytics-engine-ci-validation.md create mode 100644 scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs create mode 100644 scripts/ppl-lint/__tests__/contract-schema.test.mjs create mode 100644 scripts/ppl-lint/contract-schema.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f765c0d132e..b02a2a5ecf7 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -55,7 +55,10 @@ on: # point is to see the multi-version effect of the change. pull_request: paths: + - 'integ-test/build.gradle' + - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint-rule-validation.sh' - 'scripts/ppl-lint/**' - '.github/workflows/ppl-lint-multiversion-validation.yml' workflow_dispatch: @@ -80,6 +83,10 @@ on: permissions: contents: read +concurrency: + group: ppl-lint-multiversion-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must # be >= 3.6.0 (the _grammar endpoint floor) and must have a published @@ -273,6 +280,8 @@ jobs: -Dtests.clustername=docker-cluster \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ -Dppl.lint.target="$(pwd)/leg/target.json" @@ -454,6 +463,8 @@ jobs: -Dtests.clustername=docker-cluster \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ -Dppl.lint.target="$(pwd)/leg/target.json" # Mark the leg so the detect job knows to lint it on the compiled surface. @@ -513,6 +524,8 @@ jobs: --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ -Dppl.lint.report=$(pwd)/leg/backend-report.json \ -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/leg/target.json" @@ -533,6 +546,63 @@ jobs: name: ppl-lint-leg-pr-build-logs path: | integ-test/build/reports/** + integ-test/build/test-results/** + integ-test/build/testclusters/*/logs/* + + # The PR build through the full composite/Parquet + DataFusion stack. This is + # an observation leg: route/identity/infrastructure failures are fatal, while + # backend oracles are promoted only after their captured behavior is reviewed. + observe-pr-build-analytics: + name: Observe engine pr-build (analytics) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 25 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 25 + + - name: Run analytics contract observation against the PR build + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Upload analytics leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build-analytics + path: leg + if-no-files-found: error + + - name: Upload analytics failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-analytics-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** integ-test/build/testclusters/*/logs/* # Lint each engine's exported grammar with the OSD detectors. Separate from the @@ -549,6 +619,7 @@ jobs: - observe-released - observe-compiled - observe-pr-build + - observe-pr-build-analytics if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 40 @@ -639,7 +710,11 @@ jobs: echo "skipping $leg (no grammar bundle and no compiled-surface marker)" continue fi - env "${surface_env[@]}" \ + observe_env=(PPL_LINT_OBSERVE_ONLY=1) + if [ "$(jq -r '.executionBackend // empty' "$leg/target.json")" = 'analytics' ]; then + observe_env+=(PPL_LINT_OBSERVE_ANALYTICS=1) + fi + env "${surface_env[@]}" "${observe_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ @@ -693,7 +768,7 @@ jobs: import json,sys print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) ") - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build; do + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build pr-build-analytics; do found=no for have in "${present[@]}"; do [ "$have" = "$want" ] && found=yes && break @@ -708,6 +783,7 @@ jobs: --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ --summary "$GITHUB_STEP_SUMMARY" \ + --observe-analytics \ "${args[@]}" - name: Upload drift report @@ -876,8 +952,13 @@ jobs: ") python3 -c " import json - json.dump({'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', - 'grammarHash': '$hash'}, + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '$hash', + 'grammarBundle': 'discovery-bundle.json', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, open('$GITHUB_WORKSPACE/discovery-target.json','w')) " echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" @@ -888,6 +969,17 @@ jobs: # The surface is recorded in the report, so a reader can see which ran. echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" + python3 -c " + import json + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '', + 'grammarBundle': '', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " fi - name: Run the detectors over the discovery corpus @@ -900,10 +992,11 @@ jobs: # expanding an empty array as "${a[@]}" is an unbound-variable error in bash # before 4.4, which would crash the compiled-surface fallback — the very # path that only runs when something else already went wrong. - extra=(PPL_LINT_DISCOVERY=1) + extra=(PPL_LINT_DISCOVERY=1 + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") if [ "$SURFACE" = 'runtime-bundle' ]; then extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" - PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") + ) fi # A non-zero exit is EXPECTED and ignored: the generated specs carry # placeholder expectations, so the runner reports a "failure" for every diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 543139e3ebb..078fd285494 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,5 +1,12 @@ name: PPL lint rule validation +permissions: + contents: read + +concurrency: + group: ppl-lint-rule-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + # Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and # the SQL backend must agree on the SAME candidate runtime grammar. A shared, # reviewed corpus of contract files pins each rule's OSD detector diagnostic @@ -134,6 +141,8 @@ jobs: su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ -Dppl.lint.report=$(pwd)/backend-report.json \ -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/target.json" diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md new file mode 100644 index 00000000000..3777f561dc7 --- /dev/null +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -0,0 +1,811 @@ +# Analytics Engine Coverage for PPL Lint CI Validation + +- **Status:** Validated for phased implementation +- **Last updated:** 2026-07-28 +- **Scope:** PPL lint contract validation in + `.github/workflows/ppl-lint-rule-validation.yml` and + `.github/workflows/ppl-lint-multiversion-validation.yml` + +## 1. Summary + +The PPL lint CI contract currently compares OpenSearch Dashboards (OSD) +detectors with the standard SQL execution route only. It does not prove that +the same lint diagnostics are correct when a query is routed through the +analytics engine and executed by DataFusion over composite/Parquet storage. + +This design adds analytics-engine coverage by: + +1. Running the existing `PplLintRuleValidationIT` corpus against a dedicated, + full-stack analytics-engine test cluster. +2. Making execution backend an explicit contract and artifact dimension, + separate from OpenSearch version, Calcite applicability, and grammar + surface. +3. Failing if the analytics lane silently falls back to the standard route. +4. Running the OSD detector comparison against both standard and analytics + backend reports while bootstrapping OSD only once. +5. Shipping the lane as non-enforcing observation first, then adding it to the + stable required result after its artifacts, expectations, and reliability + meet the promotion criteria in this document. + +The initial implementation covers the SQL pull request build on one shard. It +does not add a Cartesian product of analytics backends, released OpenSearch +versions, grammar surfaces, and shard counts. + +## 2. Current State + +### 2.1 Required PPL lint validation + +`.github/workflows/ppl-lint-rule-validation.yml` is a three-job pipeline: + +```text +backend-validation + -> detector-validation + -> validation-result +``` + +- `backend-validation` runs `PplLintRuleValidationIT` against the ordinary + Gradle `integTest` cluster. That cluster installs SQL, Job Scheduler, and + Geospatial, but not the analytics-engine stack. +- The integration test executes every scheduled trigger and control query + against `POST /_plugins/_ppl`, then exports: + - `ppl-grammar-bundle.json` + - `target.json` + - `backend-report.json` +- `detector-validation` bootstraps OSD, runs its production headless PPL lint + API against the exported grammar, and compares detector output with the + backend report. +- `validation-result` uses `if: always()` and fails unless both producer jobs + succeeded. This is the stable branch-protection check. + +The multi-version companion workflow repeats the same contract against +released standard engines and the pull request build. Its current dimensions +are OpenSearch version and grammar surface. + +### 2.2 Existing analytics-engine support + +The repository already contains most of the required test infrastructure: + +- `integ-test/build.gradle` can download the analytics engine, Arrow, + composite engine, Parquet data format, and Lucene/DataFusion backend plugin + ZIPs. +- The full analytics stack is already configured for + `analyticsEngineProfileIT` and `analyticsEngineSecurityIT`. +- `-Dtests.analytics.parquet_indices=true` makes helper-created fixtures use + composite/Parquet storage. +- `SQLIntegTestCase` applies the corresponding cluster defaults before fixture + creation. +- `PPLIntegTestCase.isAnalyticsParquetIndicesEnabled()` exposes the active + route to tests. +- `integTestRemote` already forwards the analytics fixture properties. +- `CalciteAnalyticsDatetimeWireFormatIT` demonstrates route attestation using + explain output: analytics plans contain + `LogicalTableScan(table=[[opensearch,` and not + `CalciteLogicalIndexScan`. + +### 2.3 Gap in the existing analytics workflow + +`.github/workflows/analytics-engine-compat.yml` runs only +`AnalyticsEngineCompatIT`. Its purpose is plugin coexistence. Its PPL assertion +uses the `rest` row source, which is explicitly excluded from analytics +routing. The workflow can therefore pass without executing a PPL query through +DataFusion. + +The `analyticsEngineCompat` cluster is also intentionally smaller than the +stack required for real analytics execution. It does not install the composite +engine, Parquet data format, or both analytics backends. + +### 2.4 Terminology + +The following dimensions must remain independent: + +| Dimension | Examples | Meaning | +| --- | --- | --- | +| Engine version | `3.7.0`, `3.8.0-SNAPSHOT` | OpenSearch/SQL product version | +| Grammar surface | `runtime-bundle`, `compiled-simplified` | Grammar used by OSD lint | +| Lint/planner applicability | `engine: "calcite"` | Existing OSD rule applicability | +| Execution backend | `standard`, `analytics` | SQL execution route selected at runtime | +| Storage | `lucene`, `composite-parquet` | Fixture storage that drives routing | + +Analytics uses Calcite planning, so treating `analytics` as another value of +the existing `engine` field would be incorrect. Treating it as another engine +version would also cause the drift analyzer to recommend version scoping for a +backend-specific difference. + +## 3. Problem Statement + +A lint rule is presented to users before query execution. OSD currently has no +analytics-route signal in the lint context, so the same detector result applies +whether the selected index later uses the standard or analytics route. + +The current CI can miss these failures: + +1. A detector reports an error for a query that the analytics backend accepts. + This is a false positive for analytics users. +2. A detector is silent for a query rejected only by the analytics route. This + is a false negative for analytics users. +3. A control query passes on the standard route but fails on analytics. +4. An analytics test is configured incorrectly and silently executes on the + standard route, producing a vacuous green result. +5. Standard and analytics observations are stored under the same product + version, causing aggregation to overwrite or misclassify one of them. +6. A required job consumes mutable `feature-datafusion/latest` artifacts, so a + rerun can test a different stack without recording that change. + +## 4. Goals and Non-Goals + +### 4.1 Goals + +- Run every scheduled PPL lint trigger and control against the pull request's + analytics route. +- Reuse the existing contract corpus and Java integration-test oracle. +- Use byte-identical query text, the same SQL commit, the same runtime grammar, + the same OSD commit, and the same frontend lint context for both backends. +- Represent execution backend in contracts, reports, manifests, summaries, and + aggregation keys. +- Prove that the analytics plugin stack is installed, fixtures are + composite/Parquet, routing selected analytics, and DataFusion executed a + canary query. +- Distinguish backend-route divergence from version drift. +- Fail closed on missing reports, missing expectations, route fallback, + incomplete matrices, or inconsistent grammar identity. +- Produce enough artifacts to reproduce infrastructure and semantic failures. +- Keep pull request wall-clock growth bounded by running backend jobs in + parallel and bootstrapping OSD once. + +### 4.2 Non-goals + +- Replacing the existing broad analytics compatibility, security, or profile + suites. +- Running the entire PPL integration-test suite in the lint validation job. +- Adding browser, Monaco, or a running OSD server. +- Performance or benchmark validation. +- Testing every released OpenSearch version with every analytics stack in the + first release. +- Adding multi-shard analytics coverage to the required lint check. +- Automatically accepting known analytics limitations through broad Gradle + exclusions or JUnit assumptions. +- Changing production routing solely to make the test easier. + +## 5. Design Invariants + +The implementation must preserve these invariants: + +1. **Same SQL candidate:** both backend lanes build the same checked-out SQL + commit. +2. **Same grammar:** both lanes export a runtime bundle. Their engine version + and grammar hash must match before detector validation starts. +3. **Same OSD candidate:** both detector comparisons use one resolved OSD SHA + and one OSD bootstrap. +4. **Same queries:** standard, analytics, and detector passes read the same + contract files and substitute the same index names. +5. **Explicit identity:** every target and report names its execution backend. + Missing or conflicting identity is an infrastructure failure. +6. **Proven route:** setting `tests.analytics.parquet_indices=true` is not + sufficient evidence. The analytics lane must attest the installed plugins, + index settings, explain plan, and a profiled execution. +7. **No semantic retry:** downloads and cluster startup may be retried within + bounded limits. Contract queries and assertions are executed once. +8. **No vacuous pass:** missing queries, reports, detector rows, route evidence, + or planned matrix legs fail or become an explicit non-applicable result. +9. **No implicit fallback:** the analytics lane must never count a standard + route result as analytics coverage. +10. **One detector oracle:** detector count and severity remain route + independent until OSD exposes an execution-backend lint context. +11. **Complete contracts:** every selected expectation names exactly the same + query keys as the contract's top-level `queries` map. Duplicate or missing + report rows are infrastructure failures. +12. **Strict artifacts:** requested targets and reports must exist, parse, and + agree on execution identity. Writers and consumers fail rather than degrade + to an identity-free or differential-free run. + +## 6. Target Identity + +`target.json` currently records only engine version, grammar hash, and bundle +name. It will move to schema version 2 and include execution identity: + +```json +{ + "schemaVersion": 2, + "sqlSha": "...", + "engineVersion": "3.8.0-SNAPSHOT", + "grammarHash": "sha256:...", + "grammarBundle": "ppl-grammar-bundle.json", + "executionBackend": "analytics", + "storage": "composite-parquet", + "shardCount": 1, + "analyticsStack": { + "source": "immutable feature-build URL", + "buildId": "...", + "components": [ + { + "name": "analytics-engine", + "version": "3.8.0-SNAPSHOT", + "sha256": "..." + } + ] + }, + "routeAttestation": { + "pluginsVerified": true, + "clusterSettingsVerified": true, + "fixtureIndicesVerified": true, + "explainVerified": true, + "profiledExecutionVerified": true + } +} +``` + +For the standard route: + +```json +{ + "executionBackend": "standard", + "storage": "lucene", + "shardCount": 1 +} +``` + +The backend report, detector report, drift report, and run manifest will also +carry `executionBackend`. Aggregation keys become: + +```text +(leg label, engine version, grammar surface, execution backend) +``` + +The leg label remains the presentation key because multiple legs can share the +same engine version. + +## 7. Contract Schema + +### 7.1 Schema version 4 + +Detector expectations are shared, while backend oracles are keyed by execution +backend: + +```json +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException" + } + } + } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} +``` + +The existing `engine` field keeps its current meaning. It is not renamed to +avoid mixing this work with an unrelated contract migration. + +### 7.2 Compatibility and migration + +- A schema version 3 `backend` object is read as `backends.standard`. It is not + used as an implicit analytics oracle. +- Observation can begin before every analytics oracle is reviewed. In + observation mode, a missing analytics oracle executes the query once and + records `coverage-missing` plus the raw backend result; it does not score that + result against the standard oracle. Infrastructure and route-attestation + failures still fail the lane. +- Enforcement requires `backends.analytics` for every selected query. +- An unknown execution backend is a contract error. +- An unknown schema version is a contract error. +- More than one version/planner expectation match remains an error. +- A selected expectation must name exactly the top-level contract query set. +- The Java test and Node runner must implement identical selection behavior. + +An explicit non-applicable form is permitted only when the fixture cannot +meaningfully exercise analytics: + +```json +{ + "kind": "not-applicable", + "reason": "Fixture field type cannot be represented by composite/Parquet storage", + "owner": "@analytics-team", + "issue": "https://github.com/opensearch-project/sql/issues/..." +} +``` + +Rules in the required `defaultError` set cannot be promoted while their +analytics oracle is non-applicable. For other rules, non-applicable entries +remain visible in the report and require an owner and issue. + +### 7.3 Differential policy + +| Case | Detector requirement | Backend requirement | +| --- | --- | --- | +| Control | Zero diagnostics | Every applicable backend accepts | +| Rejection trigger | Expected diagnostic count and severity | Every applicable backend rejects with its reviewed error shape | +| Advisory trigger | Expected diagnostic count and severity | Backend matches its reviewed acceptance/result-shape oracle | +| Missing backend oracle | Not scored | Coverage failure | +| Backend transport error | Not scored | Infrastructure/inconclusive failure, never acceptance | + +If an error rule fires while analytics accepts the trigger, the result is +`execution-backend-divergence`. The remediation must not recommend changing an +OpenSearch version range. Because OSD currently lacks backend context, the +choices are to make the rule valid for both routes, narrow the detector to +behavior common to both, disable it, or first add a reliable backend signal to +the OSD lint context. + +## 8. Analytics Test Cluster and Gradle Task + +### 8.1 Chosen approach + +Add a dedicated Gradle-managed cluster and task: + +```text +testClusters.analyticsEnginePplLint +:integ-test:analyticsEnginePplLintIT +``` + +The cluster will install: + +- Job Scheduler +- Arrow Base +- Arrow Flight RPC +- Analytics Engine +- Composite Engine +- Parquet Data Format +- Analytics Backend Lucene +- Analytics Backend DataFusion +- The SQL plugin built from the current checkout + +It will reuse the native-access, Netty, and experimental feature settings used +by the existing full-stack profile/security clusters. Shared cluster +configuration should be extracted into a small Gradle helper if that can be +done without changing those tasks' behavior. + +The task will: + +- Depend on all analytics plugin downloads and SQL `bundlePlugin`. +- Filter to `PplLintRuleValidationIT`. +- Set `tests.analytics.parquet_indices=true`. +- Set `tests.analytics.num_shards=1`. +- Set `ppl.lint.execution_backend=analytics`. +- Forward the existing `ppl.lint.*` paths and schedule. +- Run as a non-root user in CI. + +Example invocation: + +```bash +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.execution_backend=analytics \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$PWD/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$PWD/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$PWD/leg/target.json" +``` + +The task sets the analytics fixture properties itself so a caller cannot +accidentally request an analytics report while creating Lucene fixtures. + +### 8.2 Why not the alternatives + +**Reuse `analyticsEngineCompatIT`:** rejected because its cluster lacks the full +execution stack and its test intentionally avoids analytics routing. + +**Provision an external cluster and use `integTestRemote`:** the remote task is +a valid future path for released analytics stacks, but it requires separate +cluster lifecycle, plugin installation, and SQL-plugin provenance checks. A +managed cluster is simpler and guarantees that the SQL plugin comes from the +current checkout. + +**Run the full PPL integration suite:** rejected for the required lint check. +It adds unrelated capability exclusions, runtime, and flakiness without +improving the detector contract. + +## 9. Route Attestation + +The analytics integration test will perform attestation before scoring any +contract: + +1. Query `/_cat/plugins?format=json` and require every plugin in the full stack. +2. Verify plugin versions are compatible with the OpenSearch/SQL version. +3. Read cluster settings and require composite data format defaults. +4. Read settings for every fixture index and require: + - `index.pluggable.dataformat.enabled=true` + - `index.pluggable.dataformat=composite` + - `index.composite.primary_data_format=parquet` +5. Run one valid explain canary per fixture index: + + ```text + source= | head 1 + ``` + + Require `LogicalTableScan(table=[[opensearch,` and reject + `CalciteLogicalIndexScan`. +6. Run the same canary with `profile=true` and require at least one successful + execution stage. Record every `execution_type`. Before required promotion, + pin and require the exact DataFusion-specific marker exposed by the locked + analytics stack. A generic non-empty profile is sufficient only for the + observation lane. +7. Write the attestation outcome into `target.json`. + +Invalid trigger queries may fail before DataFusion execution. Such results are +observations from an analytics-configured, route-attested environment, not +claims that DataFusion executed the invalid query. The canary proves that each +fixture is capable of analytics execution. A static +`cluster.pluggable.dataformat=composite` startup setting is also required so +query-initial parse failures see the same routing configuration as valid +queries. + +Attestation uses assertions, not JUnit assumptions. A missing plugin or legacy +explain plan fails the lane. + +## 10. CI Workflow + +### 10.1 Final required topology + +```text +Get-CI-Image-Tag + |-------------------------------| + v v +standard-backend-validation analytics-backend-validation + | | + |---- standard artifacts |---- analytics artifacts + \ / + v v + detector-validation + (one OSD checkout/bootstrap, + two backend comparisons) + | + v + validation-result +``` + +The backend jobs run in parallel. The analytics job uses JDK 25 to match the +existing analytics compatibility workflow; the standard job keeps its current +JDK. + +Artifacts use distinct names and directories: + +```text +ppl-lint-backend-standard/ +ppl-lint-backend-analytics/ +ppl-lint-backend-standard-logs/ +ppl-lint-backend-analytics-logs/ +``` + +Before linting, `detector-validation` verifies: + +- Both target manifests exist. +- Both backend reports are non-empty. +- Both targets report the expected execution backend. +- Both targets report the same engine version and grammar hash. +- Analytics route attestation is complete. +- Every requested report exists, is non-empty, contains no duplicate identities, + and agrees with its target's execution backend. + +It then invokes `run-frontend-contract.mjs` twice against the same OSD checkout +and runtime grammar: + +```text +standard backend report -> detector-standard-report.json +analytics backend report -> detector-analytics-report.json +``` + +The duplicate detector pass costs seconds; the OSD bootstrap dominates the +job. Two explicit invocations are lower risk than redesigning the runner to +accept an arbitrary report collection. The result job also compares normalized +detector rows: rule/query identity, count, severity, and any asserted message +match. Equal counts alone are not sufficient parity. + +`validation-result` continues to use `if: always()` and becomes red unless all +three validation jobs succeeded. A skipped detector caused by either backend +failure therefore cannot appear green. + +### 10.2 Multi-version workflow + +The first analytics leg is `pr-build-analytics`. It is not added to every +released version: + +| Leg | Version | Grammar surface | Execution backend | +| --- | --- | --- | --- | +| Existing released legs | Released matrix | Runtime/compiled as configured | Standard | +| `pr-build` | Pull request build | Runtime bundle | Standard | +| `pr-build-analytics` | Pull request build | Runtime bundle | Analytics | + +`aggregate-versions.mjs` must understand the backend dimension before this leg +is added. It reports backend divergence separately and never turns an +analytics-only difference into version-scoping advice. + +The discovery corpus remains standard-only in the initial implementation. It +has no reviewed oracle and should not expand the analytics rollout's cost or +diagnostic surface. + +### 10.3 Local entry point + +`scripts/ppl-lint-rule-validation.sh` will gain an opt-in analytics mode, for +example `RUN_ANALYTICS=1`. It will support the existing local ZIP override +properties. Standard local behavior remains unchanged. + +## 11. Artifact Provenance + +The current Gradle default uses a mutable +`feature-datafusion/latest/linux/x64` URL. This is acceptable for early +observation but not for a required check. + +Before promotion: + +1. Add a checked-in compatibility lock describing the immutable analytics + feature build for the current OpenSearch line. +2. Add a Gradle property such as `analyticsFeatureBuildBase` so CI can pass the + immutable base while local development can retain the current default. +3. Verify SHA-256 for every downloaded plugin ZIP before cluster startup. +4. Record the immutable source, build ID, component versions, and hashes in + `target.json`. +5. Fail if installed plugin versions do not match the locked tuple. + +If an immutable artifact source cannot be provided, the analytics lane remains +non-enforcing. + +## 12. Failure Semantics + +| Failure | Classification | CI behavior | +| --- | --- | --- | +| Plugin download or checksum failure | Infrastructure | Retry download at most three times, then fail lane | +| Cluster does not become healthy | Infrastructure | Fail and upload cluster logs/thread dump | +| Required plugin absent or wrong version | Infrastructure | Fail before contracts | +| Fixture is not composite/Parquet | Route attestation | Fail before contracts | +| Explain/profile canary uses standard route | Route attestation | Fail before contracts | +| Standard and analytics grammar hashes differ | Candidate identity | Fail detector job | +| Missing/empty backend or detector report | Incomplete run | Fail; never aggregate survivors only | +| Missing analytics expectation | Coverage hole | Fail once analytics enforcement is enabled | +| Contract query transport timeout | Inconclusive run | Fail; never treat as backend acceptance | +| Trigger/control behavior differs from oracle | Semantic drift | Report backend, query, observed status/type, and remediation | +| Standard and analytics behavior differ | Execution-backend divergence | Report separately; do not suggest version scoping | +| Detector output differs between backend passes | Harness/context defect | Fail detector job | + +Semantic assertions are never retried. A retry could hide a nondeterministic +backend or detector defect. + +## 13. Diagnostics and Resource Bounds + +The analytics job will use: + +- A 30-minute GitHub job timeout. +- A bounded OpenSearch heap consistent with current workflows. +- One Netty direct arena and the existing native-access flags. +- One primary shard for required contract coverage. +- No credentials or fork secrets. +- `permissions: contents: read`. + +Always upload on failure: + +- `target.json` and analytics stack identity. +- Backend and detector reports. +- JUnit XML and HTML reports. +- Gradle test reports. +- Installed plugin list. +- Effective cluster and fixture index settings. +- Fixture mapping hashes and any fields stripped by the analytics fixture + helper. +- OpenSearch and test-cluster logs. +- Detector logs. +- Thread dumps for startup or query timeout. + +Reports must distinguish `accepted`, `rejected`, `error`, and +`not-applicable`. An absent `rejected` field is not equivalent to acceptance. + +## 14. Test Plan + +### 14.1 Harness unit tests + +Add Node tests for: + +- Schema version 3 compatibility and schema version 4 backend selection. +- Unknown or missing execution backend. +- Missing analytics oracle. +- Duplicate target identities. +- Same version with standard and analytics legs. +- Standard/analytics grammar mismatch. +- Backend transport error not being read as acceptance. +- Analytics divergence producing backend remediation, not version scoping. +- Detector parity between standard and analytics passes. +- Non-applicable handling and required-rule coverage holes. +- Summary and annotation output naming the execution backend. + +### 14.2 Java integration coverage + +Verify: + +- The standard `PplLintRuleValidationIT` behavior is unchanged. +- The analytics task installs the full stack. +- ACCOUNT and FLAT_OBJECT fixtures are composite/Parquet or fail explicitly. +- Explain and profile canaries attest the analytics route. +- Every scheduled contract emits one backend result per expected query. +- Report entries include `executionBackend`. +- A forced missing-plugin or standard-route configuration fails attestation. + +### 14.3 Workflow validation + +Use `workflow_dispatch` to validate: + +- Canonical OSD `main`. +- An explicit OSD branch/SHA. +- A successful dual-backend run. +- An intentionally wrong analytics oracle. +- An intentionally missing analytics artifact. +- A backend failure that skips detector work but still makes the final result + red. + +No production branch-protection change is made during this validation. + +## 15. Rollout + +### Phase 1: Identity and observation + +- Add execution-backend identity to targets and reports. +- Add the schema version 4 reader with version 3 compatibility. +- Make artifact consumers fail closed on missing, malformed, duplicate, or + conflicting identities. +- Add backend-aware aggregation and divergence remediation before introducing + an analytics leg. +- Add the managed analytics Gradle task and route attestation. +- Add `pr-build-analytics` to the non-required multi-version workflow. +- Missing analytics oracles are recorded as unscored coverage gaps during + observation. Infrastructure, identity, completeness, and attestation failures + remain red. Do not use `continue-on-error` inside the producer lane. + +### Phase 2: Baseline and review + +- Capture real analytics observations for the full contract corpus. +- Add reviewed analytics oracles. +- Resolve every default-error non-applicable case. +- Pin immutable analytics artifacts and verify their checksums. +- Pin the DataFusion-specific profile execution marker. +- Measure runtime and infrastructure reliability. + +Promotion requires: + +- Every scheduled contract has a reviewed analytics oracle. +- No `defaultError` contract is non-applicable. +- No unexplained semantic divergence remains. +- At least 25 consecutive green observation runs. +- At least 50 total runs with less than 1% infrastructure failure. +- Analytics job p95 runtime is at most 15 minutes. +- Artifact provenance is immutable and recorded. + +### Phase 3: Required check + +- Add `analytics-backend-validation` to the required single-version workflow. +- Make detector validation require both backend artifacts. +- Make `validation-result` require standard backend, analytics backend, and + detector success. +- Update the run manifest and PR summary to show both routes. + +There is no silent repository-variable bypass after promotion. An emergency +rollback requires an explicit workflow/branch-protection change and a tracking +issue. + +### Phase 4: Optional expansion + +After the required lane is stable, evaluate: + +- Matching released analytics stacks. +- A scheduled three-shard analytics leg. +- Analytics execution for the discovery corpus. +- Consolidating or retiring redundant parts of + `analytics-engine-compat.yml`. + +These are separate changes and are not prerequisites for initial enforcement. + +## 16. Planned File Changes + +| File | Change | +| --- | --- | +| `integ-test/build.gradle` | Add the full-stack analytics lint cluster/task and artifact lock inputs | +| `PplLintRuleValidationIT.java` | Select backend-specific oracles, attest route, and emit backend identity | +| `integ-test/src/test/resources/ppl-lint/contracts/*.spec.json` | Migrate to schema version 4 and add analytics oracles | +| `integ-test/src/test/resources/ppl-lint/contracts/manifest.json` | Bump schema metadata and document analytics coverage | +| `scripts/ppl-lint/run-frontend-contract.mjs` | Select the active backend oracle and emit backend identity | +| `scripts/ppl-lint/contract-schema.mjs` | Share strict Node schema, identity, and backend-oracle selection | +| `scripts/ppl-lint/aggregate-versions.mjs` | Key and render legs by execution backend | +| `scripts/ppl-lint/drift.mjs` | Add execution-backend divergence and remediation | +| `scripts/ppl-lint/annotate.mjs` | Attach backend-specific findings to contract declarations | +| `scripts/ppl-lint/assemble-run-manifest.mjs` | Record both targets and job results | +| `scripts/ppl-lint/__tests__/*` | Cover schema, identity, aggregation, and remediation changes | +| `.github/workflows/ppl-lint-multiversion-validation.yml` | Add the observation leg | +| `.github/workflows/ppl-lint-rule-validation.yml` | Add the required lane after promotion | +| `scripts/ppl-lint-rule-validation.sh` | Add opt-in local analytics reproduction | +| `scripts/ppl-lint/README.md` | Document backend-aware contracts and commands | +| Analytics compatibility lock (path TBD) | Pin immutable plugin URLs, versions, and SHA-256 values before required promotion | + +## 17. Success Criteria + +The work is complete when: + +1. A pull request can produce standard and analytics observations from the same + SQL commit and grammar. +2. CI proves the analytics route instead of relying on a configuration flag. +3. Every scheduled contract has an explicit analytics result. +4. Reports cannot confuse backend divergence with version drift. +5. Missing analytics coverage cannot pass as agreement. +6. The required result fails when either backend or the OSD detector contract + fails. +7. A failed run includes enough immutable identity and logs to reproduce the + target that was tested. + +## 18. Open Questions + +1. Which system owns publishing and retaining immutable analytics feature-build + tuples for required CI? +2. Should the artifact compatibility lock live in this repository or be + generated by the OpenSearch feature-build pipeline? +3. Which current contract queries produce intentional analytics behavior + differences once the first observation run is available? +4. Will OSD eventually expose a reliable execution-backend signal to lint + context? If so, detector expectations may later become backend-aware. +5. After the full semantic lane is required, does the smaller coexistence smoke + workflow still provide enough independent value to keep? diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 822b44cde2b..1364ecd3f6f 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -294,9 +294,12 @@ def getGeoSpatialPlugin() { } } -// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead). +// Fetch from the mutable feature-build artifact for observation (linux/x64 only). CI can +// select a specific build with -PanalyticsFeatureBuildBase, and local development can pass +// individual plugin ZIP overrides such as -PanalyticsEngineZip=/path. ext.pluginVersion = opensearch_version.tokenize('-')[0] -ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: + "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -448,6 +451,35 @@ testClusters { // Composite-default cluster: PPL queries route to the analytics engine unless excluded. setting 'cluster.pluggable.dataformat', 'composite' } + analyticsEnginePplLintIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getCompositeEnginePlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin ":opensearch-sql-plugin" + setting 'cluster.pluggable.dataformat.enabled', 'true' + setting 'cluster.pluggable.dataformat', 'composite' + setting 'cluster.composite.primary_data_format', 'parquet' + setting 'cluster.composite.secondary_data_formats', '[lucene]' + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // Native library path for DataFusion/parquet -- pass via -PnativeLibPath=/path/to/release/ + if (project.findProperty('nativeLibPath')) { + systemProperty 'java.library.path', project.findProperty('nativeLibPath') + } + } } def isPrometheusRunning() { @@ -507,6 +539,24 @@ task analyticsEngineCompatIT(type: RestIntegTestTask) { } } +task analyticsEnginePplLintIT(type: RestIntegTestTask) { + useCluster testClusters.analyticsEnginePplLintIT + dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, + downloadCompositeEngineZip, downloadParquetDataFormatZip, + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.analytics.parquet_indices', 'true' + systemProperty 'tests.analytics.num_shards', '1' + systemProperty 'ppl.lint.execution_backend', 'analytics' + systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase + systemProperty 'tests.security.manager', 'false' + + filter { + includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' + } +} + task analyticsEngineSecurityIT(type: RestIntegTestTask) { dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip dependsOn ':opensearch-sql-plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 7ed5e6ad989..01012b7406d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -12,7 +12,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -29,14 +32,14 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the schema-v3 PPL lint rule validation contract. + * Backend half of the schema-v3/schema-v4 PPL lint rule validation contract. * *

    This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from * the current checkout. For every contract (see {@code * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} * entry that matches the candidate backend version (exactly one must match, or the contract fails - * before any query runs), applies the contract's cluster settings, and asserts, per query's {@code - * backend.kind}: + * before any query runs), applies the contract's cluster settings, and asserts the oracle selected + * for {@code ppl.lint.execution_backend}: * *

      *
    • {@code rejection} — the query returns the contracted HTTP status and structured error body @@ -77,10 +80,27 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; private static final String GRAMMAR_API_ENDPOINT = "/_plugins/_ppl/_grammar"; + private static final String EXECUTION_BACKEND_PROPERTY = "ppl.lint.execution_backend"; + private static final String ANALYTICS_SHARD_COUNT_PROPERTY = "tests.analytics.num_shards"; + private static final String[] REQUIRED_ANALYTICS_PLUGIN_COMPONENTS = { + "job-scheduler", + "arrow-base", + "arrow-flight-rpc", + "analytics-engine", + "analytics-backend-lucene", + "analytics-backend-datafusion", + "parquet-data-format", + "composite-engine", + "opensearch-sql" + }; /** Which contracts to run this session; PR is the fast blocking subset. */ private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + /** Execution route whose backend oracle and artifact identity this run represents. */ + private final ExecutionBackend executionBackend = + ExecutionBackend.parse(System.getProperty(EXECUTION_BACKEND_PROPERTY, "standard")); + /** * Observe-only mode, used by the multi-version workflow ({@code * .github/workflows/ppl-lint-multiversion-validation.yml}). @@ -101,6 +121,13 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + private final JSONObject analyticsRouteAttestation = + new JSONObject() + .put("pluginsVerified", false) + .put("clusterSettingsVerified", false) + .put("fixtureIndicesVerified", false) + .put("explainVerified", false) + .put("profiledExecutionVerified", false); /** * Whether this cluster recognizes the Calcite settings at all. False on a pre-Calcite (2.x) @@ -175,18 +202,32 @@ public void testValidatesLintRuleContracts() throws IOException { List contracts = loadScheduledContracts(); List failures = new ArrayList<>(); JSONArray report = new JSONArray(); + if (contracts.isEmpty()) { + failures.add("[contracts] no contracts were selected for schedule \"" + schedule + "\""); + } + + boolean routeAttested = + executionBackend != ExecutionBackend.ANALYTICS || attestAnalyticsRoute(failures); // Export the candidate grammar bundle + target manifest while the cluster is // alive. Runs before the contract loop so the artifacts are emitted even if a // contract later fails. exportGrammarArtifacts(failures); - for (JSONObject contract : contracts) { - String ruleId = contract.getString("ruleId"); - runContract(contract, ruleId, failures, report); + // A failed route attestation is infrastructure failure, not backend behavior. + // Do not score any contract against a route that was not proven. + if (routeAttested) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + runContract(contract, ruleId, failures, report); + } } - writeReport(report); + try { + writeReport(report); + } catch (IOException e) { + failures.add("[report] failed to write backend report: " + e.getMessage()); + } if (!failures.isEmpty()) { fail( @@ -200,12 +241,53 @@ public void testValidatesLintRuleContracts() throws IOException { private void runContract( JSONObject contract, String ruleId, List failures, JSONArray report) throws IOException { + int schemaVersion = contract.getInt("schemaVersion"); + if (schemaVersion != 3 && schemaVersion != 4) { + failures.add( + "[" + ruleId + "] unsupported schemaVersion " + schemaVersion + " (expected 3 or 4)"); + return; + } + String index = contract.getString("index"); JSONObject queries = contract.getJSONObject("queries"); JSONArray expectations = contract.getJSONArray("expectations"); JSONObject fixture = contract.optJSONObject("backendFixture"); boolean calciteOn = fixtureCalciteEnabled(fixture); + if (expectations.length() == 0) { + failures.add("[" + ruleId + "] expectations must not be empty"); + return; + } + + if (!validateAllExpectations(ruleId, queries, expectations, schemaVersion, failures)) { + return; + } + + List matches = matchingExpectations(expectations, calciteOn); + if (matches.size() > 1) { + failures.add( + "[" + + ruleId + + "] " + + matches.size() + + " expectations match backend version " + + backendVersionLabel() + + " (exactly one required)"); + return; + } + + JSONObject selected = matches.isEmpty() ? null : matches.get(0); + if (selected == null && !observeOnly) { + failures.add( + "[" + + ruleId + + "] no version expectation matches backend version " + + backendVersionLabel()); + return; + } + + JSONObject expectedQueries = selected == null ? null : selected.getJSONObject("queries"); + // A contract whose fixture index never got created cannot produce a meaningful // observation: every query would fail with IndexNotFoundException regardless of // the rule. Report each case as an error so the aggregator counts it as @@ -216,48 +298,54 @@ private void runContract( return; } + if (!observeOnly + && recordEnforcementCoverageGaps( + ruleId, index, queries, expectedQueries, schemaVersion, failures, report)) { + return; + } + List applied = applyClusterSettings(fixture); try { - // In observe-only mode, "no expectation matches this version" is information, - // not a failure — a rule the corpus does not pin for THIS engine is exactly - // what the multi-version matrix is here to learn. selectExpectation records - // into whatever list it is handed, so hand it a scratch list we discard; - // otherwise the leg both records the observation AND fails, which is what - // kept union-min-datasets (a >=3.7 rule) failing the 3.6 leg. - List selectionFailures = observeOnly ? new ArrayList<>() : failures; - JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, selectionFailures); if (selected == null) { - if (!observeOnly) { - return; // no/ambiguous version expectation — failure already recorded. - } // Record the raw behavior of every query and let the aggregator decide // whether the gap matters (out-of-scope rule vs a real coverage hole). - observeAllQueries(ruleId, index, queries, report); + observeAllQueries(ruleId, index, queries, failures, report); return; } - JSONObject expectedQueries = selected.getJSONObject("queries"); - for (String queryName : expectedQueries.keySet()) { - if (!queries.has(queryName)) { - failures.add( - "[" - + ruleId - + "] expectation references unknown query \"" - + queryName - + "\" (not in the top-level queries map)"); - continue; - } + + for (String queryName : queries.keySet()) { JSONObject queryDef = queries.getJSONObject(queryName); String role = queryDef.optString("role", "trigger"); String query = queryDef.getString("query").replace("{{index}}", index); JSONObject expected = expectedQueries.getJSONObject(queryName); - JSONObject backend = expected.getJSONObject("backend"); + JSONObject backend = resolveBackendOracle(schemaVersion, expected); + if (backend == null) { + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + continue; + } + String kind = backend.getString("kind"); JSONObject entry = reportEntry(ruleId, queryName, role, query, kind); + if ("not-applicable".equals(kind)) { + recordNotApplicable(ruleId, queryName, backend, entry, report); + continue; + } + try { verifyCase(kind, queryName, query, backend, entry); entry.put("outcome", "pass"); log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); + } catch (IOException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend query transport failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR (" + kind + "): " + e.getMessage()); } catch (AssertionError | RuntimeException e) { entry.put("outcome", observeOnly ? "observed-mismatch" : "fail"); entry.put("error", String.valueOf(e.getMessage())); @@ -327,7 +415,7 @@ private void recordUnusableContract( * a blank row that would read as agreement. */ private void observeAllQueries( - String ruleId, String index, JSONObject queries, JSONArray report) { + String ruleId, String index, JSONObject queries, List failures, JSONArray report) { for (String queryName : queries.keySet()) { JSONObject queryDef = queries.getJSONObject(queryName); String role = queryDef.optString("role", "trigger"); @@ -344,6 +432,13 @@ private void observeAllQueries( // A transport-level problem is a broken run, not an engine verdict; mark it // so the aggregator does not read the absence of a rejection as acceptance. entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); log(ruleId, queryName, "ERROR: " + e.getMessage()); } report.put(entry); @@ -351,42 +446,412 @@ private void observeAllQueries( } /** - * Select the single {@code expectations[]} entry that applies to the candidate backend version - * and engine. Exactly one must match: zero means the rule test does not cover this version - * (design §9), and more than one means overlapping ranges — both fail before execution (§5.3). + * A missing backend oracle is a coverage result, not an invitation to borrow another backend's + * expectation. Observation executes the query exactly once and records its raw behavior; + * enforcement records the gap without executing or scoring the query. */ - private JSONObject selectExpectation( - String ruleId, JSONArray expectations, boolean calciteOn, List failures) { - List matches = new ArrayList<>(); + private void recordMissingOracle( + String ruleId, + String queryName, + String role, + String query, + int schemaVersion, + List failures, + JSONArray report) { + String reason = + schemaVersion == 3 + ? "schema v3 provides only a standard backend oracle" + : "schema v4 has no " + executionBackend.id + " entry in expected query backends"; + JSONObject entry = + reportEntry(ruleId, queryName, role, query, "coverage-missing") + .put("coverage", "missing") + .put("reason", reason); + + if (!observeOnly) { + entry.put("outcome", "coverage-missing"); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] missing " + + executionBackend.id + + " backend oracle: " + + reason); + report.put(entry); + log(ruleId, queryName, "COVERAGE MISSING (" + executionBackend.id + ")"); + return; + } + + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "coverage-missing"); + log( + ruleId, + queryName, + "COVERAGE MISSING; OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + + /** + * Enforcement must establish complete backend-oracle coverage before executing any query in the + * contract. This avoids producing partially scored evidence when a later query has no oracle. + */ + private boolean recordEnforcementCoverageGaps( + String ruleId, + String index, + JSONObject queries, + JSONObject expectedQueries, + int schemaVersion, + List failures, + JSONArray report) { + boolean missing = false; + for (String queryName : queries.keySet()) { + JSONObject expected = expectedQueries.getJSONObject(queryName); + if (resolveBackendOracle(schemaVersion, expected) != null) { + continue; + } + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + missing = true; + } + return missing; + } + + /** Record an explicit schema-v4 non-applicable oracle without executing the query. */ + private void recordNotApplicable( + String ruleId, String queryName, JSONObject backend, JSONObject entry, JSONArray report) { + String reason = backend.getString("reason"); + entry + .put("outcome", "not-applicable") + .put("reason", reason) + .put("owner", backend.getString("owner")) + .put("issue", backend.getString("issue")); + report.put(entry); + log(ruleId, queryName, "NOT APPLICABLE (" + executionBackend.id + ")"); + } + + /** + * Resolve the execution backend oracle without fallback. Schema v3 is standard-only; schema v4 + * requires an explicit entry in {@code backends}. + */ + private JSONObject resolveBackendOracle(int schemaVersion, JSONObject expected) { + if (schemaVersion == 3) { + return executionBackend == ExecutionBackend.STANDARD + ? expected.getJSONObject("backend") + : null; + } + if (schemaVersion == 4) { + JSONObject backends = expected.optJSONObject("backends"); + return backends != null && backends.has(executionBackend.id) + ? backends.getJSONObject(executionBackend.id) + : null; + } + throw new IllegalArgumentException("unsupported contract schemaVersion " + schemaVersion); + } + + /** + * Validate every expectation before version selection or query execution. Observation mode may + * tolerate a missing route oracle, but it must never turn a malformed oracle into observed drift. + */ + private boolean validateAllExpectations( + String ruleId, + JSONObject declaredQueries, + JSONArray expectations, + int schemaVersion, + List failures) { + Set declared = new LinkedHashSet<>(declaredQueries.keySet()); + boolean valid = true; + if (declared.isEmpty()) { + failures.add("[" + ruleId + "] queries must not be empty"); + valid = false; + } for (int i = 0; i < expectations.length(); i++) { - JSONObject exp = expectations.getJSONObject(i); - if (!versionMatchesRange(exp.optString("version", null))) { + String expectationPath = "expectations[" + i + "]"; + Object expectationValue = expectations.opt(i); + if (!(expectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + " must be an object"); + valid = false; continue; } - String engine = exp.optString("engine", ""); - if ("calcite".equals(engine) && !calciteOn) { + JSONObject expectation = (JSONObject) expectationValue; + Object expectationQueriesValue = expectation.opt("queries"); + if (!(expectationQueriesValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + ".queries must be an object"); + valid = false; continue; } - matches.add(exp); + JSONObject expectationQueries = (JSONObject) expectationQueriesValue; + Set expected = new LinkedHashSet<>(expectationQueries.keySet()); + if (!declared.equals(expected)) { + Set missingFromExpectation = new LinkedHashSet<>(declared); + missingFromExpectation.removeAll(expected); + Set unknownInExpectation = new LinkedHashSet<>(expected); + unknownInExpectation.removeAll(declared); + failures.add( + "[" + + ruleId + + "] " + + expectationPath + + " query keys must exactly match top-level queries" + + "; missing from expectation=" + + missingFromExpectation + + "; unknown in expectation=" + + unknownInExpectation); + valid = false; + } + + for (String queryName : expected) { + String queryPath = expectationPath + ".queries." + queryName; + Object queryExpectationValue = expectationQueries.opt(queryName); + if (!(queryExpectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + " must be an object"); + valid = false; + continue; + } + JSONObject queryExpectation = (JSONObject) queryExpectationValue; + if (schemaVersion == 3) { + Object backendValue = queryExpectation.opt("backend"); + if (!(backendValue instanceof JSONObject)) { + failures.add( + "[" + ruleId + "] " + queryPath + ".backend must be a schema-v3 oracle object"); + valid = false; + continue; + } + valid &= + validateBackendOracle( + ruleId, queryPath + ".backend", (JSONObject) backendValue, failures); + continue; + } + + if (!queryExpectation.has("backends")) { + continue; + } + Object backendsValue = queryExpectation.opt("backends"); + if (!(backendsValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + ".backends must be an object"); + valid = false; + continue; + } + JSONObject backends = (JSONObject) backendsValue; + for (String backend : backends.keySet()) { + String backendPath = queryPath + ".backends." + backend; + if (!"standard".equals(backend) && !"analytics".equals(backend)) { + failures.add( + "[" + + ruleId + + "] " + + queryPath + + " declares unknown execution backend \"" + + backend + + "\""); + valid = false; + continue; + } + Object oracleValue = backends.opt(backend); + if (!(oracleValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + backendPath + " must be an oracle object"); + valid = false; + continue; + } + valid &= validateBackendOracle(ruleId, backendPath, (JSONObject) oracleValue, failures); + } + } + } + return valid; + } + + private boolean validateBackendOracle( + String ruleId, String path, JSONObject oracle, List failures) { + int initialFailureCount = failures.size(); + String kind = requireNonBlankString(ruleId, path + ".kind", oracle.opt("kind"), failures); + if (kind == null) { + return false; } - String versionLabel = engineVersionRaw == null ? "unknown" : engineVersionRaw; - if (matches.size() == 1) { - return matches.get(0); + + if ("not-applicable".equals(kind)) { + requireNonBlankString(ruleId, path + ".reason", oracle.opt("reason"), failures); + requireNonBlankString(ruleId, path + ".owner", oracle.opt("owner"), failures); + requireNonBlankString(ruleId, path + ".issue", oracle.opt("issue"), failures); + return failures.size() == initialFailureCount; } - if (matches.isEmpty()) { - failures.add( - "[" + ruleId + "] no version expectation matches backend version " + versionLabel); - } else { + + Integer httpStatus = + requireInteger(ruleId, path + ".httpStatus", oracle.opt("httpStatus"), 100, 599, failures); + switch (kind) { + case "rejection": + validateRejectionOracle(ruleId, path, oracle, httpStatus, failures); + break; + case "result-shape": + requireHttpOk(ruleId, path, httpStatus, failures); + validateResultShapeOracle(ruleId, path, oracle, failures); + break; + case "advisory": + requireHttpOk(ruleId, path, httpStatus, failures); + validateAdvisoryOracle(ruleId, path, oracle, failures); + break; + default: + failures.add("[" + ruleId + "] " + path + ".kind is unknown: \"" + kind + "\""); + break; + } + return failures.size() == initialFailureCount; + } + + private void validateRejectionOracle( + String ruleId, String path, JSONObject oracle, Integer httpStatus, List failures) { + Object bodyValue = oracle.opt("body"); + if (!(bodyValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body must be an object"); + return; + } + JSONObject body = (JSONObject) bodyValue; + Integer bodyStatus = + requireInteger(ruleId, path + ".body.status", body.opt("status"), 100, 599, failures); + if (httpStatus != null && bodyStatus != null && !httpStatus.equals(bodyStatus)) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must equal " + path + ".body.status"); + } + + if (!body.has("error")) { + return; + } + Object errorValue = body.opt("error"); + if (!(errorValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body.error must be an object"); + return; + } + JSONObject error = (JSONObject) errorValue; + if (error.has("type")) { + requireNonBlankString(ruleId, path + ".body.error.type", error.opt("type"), failures); + } + if (error.has("reason")) { + requireNonBlankString(ruleId, path + ".body.error.reason", error.opt("reason"), failures); + } + } + + private void validateResultShapeOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("datarowsNonEmpty") && !(expect.opt("datarowsNonEmpty") instanceof Boolean)) { + failures.add("[" + ruleId + "] " + path + ".expect.datarowsNonEmpty must be a boolean"); + } + if (expect.has("datarowsCount")) { + requireInteger( + ruleId, + path + ".expect.datarowsCount", + expect.opt("datarowsCount"), + 0, + Integer.MAX_VALUE, + failures); + } + if (expect.has("columnAllNull")) { + requireNonBlankString( + ruleId, path + ".expect.columnAllNull", expect.opt("columnAllNull"), failures); + } + } + + private void validateAdvisoryOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("accepted") && !Boolean.TRUE.equals(expect.opt("accepted"))) { + failures.add("[" + ruleId + "] " + path + ".expect.accepted must be true"); + } + } + + private void requireHttpOk( + String ruleId, String path, Integer httpStatus, List failures) { + if (httpStatus != null && httpStatus != 200) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must be 200"); + } + } + + private String requireNonBlankString( + String ruleId, String path, Object value, List failures) { + if (!(value instanceof String) || ((String) value).trim().isEmpty()) { + failures.add("[" + ruleId + "] " + path + " must be a non-blank string"); + return null; + } + return (String) value; + } + + private Integer requireInteger( + String ruleId, String path, Object value, int minimum, int maximum, List failures) { + if (!(value instanceof Number)) { + failures.add("[" + ruleId + "] " + path + " must be an integer"); + return null; + } + double numeric = ((Number) value).doubleValue(); + if (!Double.isFinite(numeric) + || numeric != Math.rint(numeric) + || numeric < minimum + || numeric > maximum) { failures.add( "[" + ruleId + "] " - + matches.size() - + " expectations match backend version " - + versionLabel - + " (exactly one required)"); + + path + + " must be an integer from " + + minimum + + " through " + + maximum); + return null; } - return null; + return ((Number) value).intValue(); + } + + /** + * Find the expectations that apply to the candidate version and planner. The caller treats zero + * matches as raw-observation-only and multiple matches as fatal in every mode. + */ + private List matchingExpectations(JSONArray expectations, boolean calciteOn) { + List matches = new ArrayList<>(); + for (int i = 0; i < expectations.length(); i++) { + JSONObject exp = expectations.getJSONObject(i); + if (!versionMatchesRange(exp.optString("version", null))) { + continue; + } + String engine = exp.optString("engine", ""); + if ("calcite".equals(engine) && !calciteOn) { + continue; + } + matches.add(exp); + } + return matches; + } + + private String backendVersionLabel() { + return engineVersionRaw == null ? "unknown" : engineVersionRaw; } private void verifyCase( @@ -428,8 +893,7 @@ private BackendObservation observeBackend(String query) throws IOException { try { body = new JSONObject(getResponseBody(e.getResponse(), true)); } catch (IOException ioe) { - throw new RuntimeException( - "failed to read rejection response body for query: " + query, ioe); + throw new IOException("failed to read rejection response body for query: " + query, ioe); } return BackendObservation.rejected(status, body); } @@ -624,15 +1088,403 @@ static BackendObservation rejected(int status, JSONObject body) { JSONObject toJson() { JSONObject o = new JSONObject().put("httpStatus", status).put("rejected", rejected); if (body != null) { + o.put("body", body); JSONObject err = body.optJSONObject("error"); if (err != null) { o.put("type", err.opt("type")).put("reason", err.opt("reason")); } } + if (response != null) { + o.put("response", response); + } return o; } } + // --- analytics route attestation ------------------------------------------ + + /** + * Prove the analytics route before any contract is scored. Each check is retained in the target + * manifest, including failures, so a missing route cannot be mistaken for backend coverage. + */ + private boolean attestAnalyticsRoute(List failures) { + boolean plugins = + runAnalyticsAttestationCheck( + "pluginsVerified", "required plugins", this::verifyAnalyticsPlugins, failures); + boolean clusterSettings = + runAnalyticsAttestationCheck( + "clusterSettingsVerified", + "cluster settings", + this::verifyAnalyticsClusterSettings, + failures); + boolean fixtureIndices = + runAnalyticsAttestationCheck( + "fixtureIndicesVerified", + "fixture index settings", + this::verifyAnalyticsFixtureIndices, + failures); + boolean explain = + runAnalyticsAttestationCheck( + "explainVerified", "explain route", this::verifyAnalyticsExplainCanaries, failures); + boolean profile = + runAnalyticsAttestationCheck( + "profiledExecutionVerified", + "profiled execution", + this::verifyAnalyticsProfileCanaries, + failures); + return plugins && clusterSettings && fixtureIndices && explain && profile; + } + + private boolean runAnalyticsAttestationCheck( + String targetField, String label, AttestationCheck check, List failures) { + try { + check.run(); + analyticsRouteAttestation.put(targetField, true); + log("route-attestation", label, "PASS"); + return true; + } catch (Exception | AssertionError e) { + analyticsRouteAttestation.put(targetField, false); + failures.add( + "[route-attestation/" + + label + + "] " + + (e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage())); + log("route-attestation", label, "FAIL: " + e.getMessage()); + return false; + } + } + + private void verifyAnalyticsPlugins() throws IOException { + Response response = + client() + .performRequest(new Request("GET", "/_cat/plugins?format=json&h=component,version")); + JSONArray plugins = new JSONArray(getResponseBody(response, true)); + List installed = new ArrayList<>(); + for (int i = 0; i < plugins.length(); i++) { + installed.add(plugins.getJSONObject(i).getString("component")); + } + analyticsRouteAttestation.put("plugins", plugins); + + requireAttestation( + engineVersionRaw != null && !engineVersionRaw.trim().isEmpty(), + "cluster engine version is unavailable for plugin compatibility checks"); + String expectedVersionPrefix = engineVersionRaw.split("-")[0]; + for (String required : REQUIRED_ANALYTICS_PLUGIN_COMPONENTS) { + JSONObject matched = null; + for (int i = 0; i < plugins.length(); i++) { + JSONObject plugin = plugins.getJSONObject(i); + if (pluginComponentMatches(plugin.getString("component"), required)) { + matched = plugin; + break; + } + } + requireAttestation( + matched != null, + "required plugin component matching \"" + + required + + "\" is missing; installed=" + + installed); + String version = matched.optString("version", ""); + requireAttestation( + version.equals(expectedVersionPrefix) + || version.startsWith(expectedVersionPrefix + ".") + || version.startsWith(expectedVersionPrefix + "-"), + "plugin " + + matched.getString("component") + + " version " + + version + + " is incompatible with engine " + + engineVersionRaw); + } + } + + private boolean pluginComponentMatches(String component, String required) { + return component.equals(required) || component.endsWith("-" + required); + } + + private void verifyAnalyticsClusterSettings() throws IOException { + Response nodesResponse = + client().performRequest(new Request("GET", "/_nodes/settings?flat_settings=true")); + JSONObject nodes = new JSONObject(getResponseBody(nodesResponse, true)).getJSONObject("nodes"); + requireAttestation(nodes.length() > 0, "node settings response contained no nodes"); + for (String nodeId : nodes.keySet()) { + String startupDataFormat = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat", ""); + String startupEnabled = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat.enabled", ""); + requireAttestation( + "composite".equals(startupDataFormat), + "node " + + nodeId + + " startup cluster.pluggable.dataformat must be composite but was \"" + + startupDataFormat + + "\""); + requireAttestation( + "true".equals(startupEnabled), + "node " + + nodeId + + " startup cluster.pluggable.dataformat.enabled must be true but was \"" + + startupEnabled + + "\""); + } + + Response response = + client() + .performRequest( + new Request("GET", "/_cluster/settings?flat_settings=true&include_defaults=true")); + JSONObject settings = new JSONObject(getResponseBody(response, true)); + + requireEffectiveSetting(settings, "cluster.pluggable.dataformat", "composite"); + requireEffectiveSetting(settings, "cluster.pluggable.dataformat.enabled", "true"); + requireEffectiveSetting(settings, "cluster.composite.primary_data_format", "parquet"); + requireEffectiveSettingContains(settings, "cluster.composite.secondary_data_formats", "lucene"); + analyticsRouteAttestation.put("clusterSettings", settings); + } + + private void verifyAnalyticsFixtureIndices() throws IOException { + int expectedShards = analyticsShardCount(); + JSONObject documentCounts = new JSONObject(); + JSONObject fixtureIndices = new JSONObject(); + analyticsRouteAttestation + .put("fixtureDocumentCounts", documentCounts) + .put("fixtureIndices", fixtureIndices); + for (String indexEnum : requiredIndexEnums()) { + String indexName = Index.valueOf(indexEnum).getName(); + Response response = + client() + .performRequest( + new Request( + "GET", + "/" + indexName + "/_settings?flat_settings=true&include_defaults=true")); + JSONObject body = new JSONObject(getResponseBody(response, true)); + JSONObject settings = body.getJSONObject(indexName).getJSONObject("settings"); + JSONObject fixtureEvidence = new JSONObject().put("settings", settings); + fixtureIndices.put(indexName, fixtureEvidence); + + Response mappingResponse = + client().performRequest(new Request("GET", "/" + indexName + "/_mapping")); + JSONObject mappingBody = new JSONObject(getResponseBody(mappingResponse, true)); + JSONObject mapping = mappingBody.getJSONObject(indexName).getJSONObject("mappings"); + fixtureEvidence.put("mappingHash", sha256(canonicalJson(mapping))).put("mapping", mapping); + + requireIndexSetting(indexName, settings, "index.pluggable.dataformat.enabled", "true"); + requireIndexSetting(indexName, settings, "index.pluggable.dataformat", "composite"); + requireIndexSetting(indexName, settings, "index.composite.primary_data_format", "parquet"); + requireIndexSettingContains( + indexName, settings, "index.composite.secondary_data_formats", "lucene"); + requireIndexSetting( + indexName, settings, "index.number_of_shards", Integer.toString(expectedShards)); + + Response countResponse = + client().performRequest(new Request("GET", "/" + indexName + "/_count")); + long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + requireAttestation( + count > 0, + "fixture " + indexName + " contains no documents; fixture ingestion did not complete"); + documentCounts.put(indexName, count); + fixtureEvidence.put("documentCount", count); + } + } + + private String canonicalJson(Object value) { + if (value == null || value == JSONObject.NULL) { + return "null"; + } + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + List keys = new ArrayList<>(object.keySet()); + Collections.sort(keys); + StringBuilder canonical = new StringBuilder("{"); + for (int i = 0; i < keys.size(); i++) { + if (i > 0) { + canonical.append(','); + } + String key = keys.get(i); + canonical.append(JSONObject.quote(key)).append(':').append(canonicalJson(object.get(key))); + } + return canonical.append('}').toString(); + } + if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + StringBuilder canonical = new StringBuilder("["); + for (int i = 0; i < array.length(); i++) { + if (i > 0) { + canonical.append(','); + } + canonical.append(canonicalJson(array.get(i))); + } + return canonical.append(']').toString(); + } + if (value instanceof String) { + return JSONObject.quote((String) value); + } + if (value instanceof Number || value instanceof Boolean) { + return value.toString(); + } + throw new IllegalArgumentException( + "unsupported JSON value type in fixture mapping: " + value.getClass().getName()); + } + + private String sha256(String value) { + try { + byte[] digest = + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder("sha256:"); + for (byte octet : digest) { + hex.append(String.format(Locale.ROOT, "%02x", octet & 0xff)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 digest is unavailable", e); + } + } + + private void verifyAnalyticsExplainCanaries() throws IOException { + for (String indexEnum : requiredIndexEnums()) { + String query = analyticsCanaryQuery(indexEnum); + String explained = explainQueryToString(query); + requireAttestation( + explained.contains("LogicalTableScan(table=[[opensearch,"), + "fixture " + indexEnum + " did not use LogicalTableScan(opensearch): " + explained); + requireAttestation( + !explained.contains("CalciteLogicalIndexScan"), + "fixture " + indexEnum + " fell back to CalciteLogicalIndexScan: " + explained); + } + } + + private void verifyAnalyticsProfileCanaries() throws IOException { + JSONArray executionTypes = new JSONArray(); + for (String indexEnum : requiredIndexEnums()) { + JSONObject response = runProfiledPplQuery(analyticsCanaryQuery(indexEnum)); + JSONObject profile = response.getJSONObject("profile"); + JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); + requireAttestation( + stages.length() > 0, "fixture " + indexEnum + " profile returned no execution stages"); + for (int i = 0; i < stages.length(); i++) { + JSONObject stage = stages.getJSONObject(i); + requireAttestation( + "SUCCEEDED".equals(stage.optString("state")), + "fixture " + indexEnum + " profile stage " + i + " was not successful: " + stage); + requireAttestation( + !stage.optString("execution_type", "").trim().isEmpty(), + "fixture " + indexEnum + " profile stage " + i + " has no execution_type: " + stage); + executionTypes.put(stage.getString("execution_type")); + } + } + analyticsRouteAttestation.put("profileExecutionTypes", executionTypes); + } + + private String analyticsCanaryQuery(String indexEnum) { + String indexName = Index.valueOf(indexEnum).getName(); + switch (indexEnum) { + case "ACCOUNT": + return "source=" + indexName + " | fields account_number, firstname | head 1"; + case "FLAT_OBJECT": + return "source=" + indexName + " | fields name, status | head 1"; + default: + throw new IllegalArgumentException( + "no fixture-safe analytics canary projection is defined for " + indexEnum); + } + } + + private JSONObject runProfiledPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).put("profile", true).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private void requireEffectiveSetting(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + expected.equals(actual), + "effective " + key + " must be " + expected + " but was \"" + actual + "\""); + } + + private void requireEffectiveSettingContains(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + actual.contains(expected), + "effective " + key + " must contain " + expected + " but was \"" + actual + "\""); + } + + private String effectiveSetting(JSONObject settings, String key) { + String transientValue = settingInSection(settings, "transient", key); + if (!transientValue.isEmpty()) { + return transientValue; + } + String persistentValue = settingInSection(settings, "persistent", key); + if (!persistentValue.isEmpty()) { + return persistentValue; + } + return settingInSection(settings, "defaults", key); + } + + private String settingInSection(JSONObject settings, String section, String key) { + JSONObject values = settings.optJSONObject(section); + return values == null ? "" : values.optString(key, ""); + } + + private void requireIndexSetting( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + expected.equals(actual), + "fixture " + + indexName + + " setting " + + key + + " must be " + + expected + + " but was \"" + + actual + + "\""); + } + + private void requireIndexSettingContains( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + actual.contains(expected), + "fixture " + + indexName + + " setting " + + key + + " must contain " + + expected + + " but was \"" + + actual + + "\""); + } + + private int analyticsShardCount() { + int shardCount = Integer.parseInt(System.getProperty(ANALYTICS_SHARD_COUNT_PROPERTY, "1")); + requireAttestation(shardCount > 0, ANALYTICS_SHARD_COUNT_PROPERTY + " must be positive"); + return shardCount; + } + + private static void requireAttestation(boolean condition, String message) { + if (!condition) { + throw new IllegalStateException(message); + } + } + + @FunctionalInterface + private interface AttestationCheck { + void run() throws Exception; + } + // --- grammar bundle export ------------------------------------------------- /** @@ -653,22 +1505,26 @@ private void exportGrammarArtifacts(List failures) { writeTargetManifest("", failures); return; } + String grammarHash = ""; + String bundleName = ""; try { Response response = client().performRequest(new Request("GET", GRAMMAR_API_ENDPOINT)); String bundleBody = getResponseBody(response, true); - Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); - JSONObject bundle = new JSONObject(bundleBody); - String grammarHash = bundle.optString("grammarHash", ""); - writeTargetManifest(grammarHash, Paths.get(bundlePath).getFileName().toString(), failures); + grammarHash = bundle.optString("grammarHash", ""); + Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); + bundleName = Paths.get(bundlePath).getFileName().toString(); log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); } catch (Exception e) { failures.add( "[grammar-export] failed to fetch/write " + GRAMMAR_API_ENDPOINT + ": " + e.getMessage()); + } finally { + // Route and attestation identity remain available even when the grammar + // endpoint or bundle write fails. + writeTargetManifest(grammarHash, bundleName, failures); } } - /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ /** * True when a failure is the cluster rejecting {@code plugins.calcite.enabled} because it does * not know that setting — i.e. a pre-Calcite (2.x) engine. @@ -697,9 +1553,8 @@ private void writeTargetManifest(String grammarHash, List failures) { } /** - * Write {@code ppl.lint.target}: the engine version, the grammar hash when there is one, and the - * bundle filename when one was exported. Every consumer keys on {@code engineVersion}, so this is - * written whether or not a bundle exists. + * Write target schema v2 with engine, grammar, execution route, storage, shard count, and (for + * analytics) route attestation identity. */ private void writeTargetManifest(String grammarHash, String bundleName, List failures) { String targetPath = System.getProperty("ppl.lint.target"); @@ -709,9 +1564,24 @@ private void writeTargetManifest(String grammarHash, String bundleName, List loadScheduledContracts() throws IOException { List result = new ArrayList<>(); + Set ruleIds = new LinkedHashSet<>(); for (String fileName : manifestContractNames()) { JSONObject contract = loadContractFile(CONTRACT_DIR + "/" + fileName); + String ruleId = contract.getString("ruleId"); + if (!ruleIds.add(ruleId)) { + throw new IOException("contract manifest contains duplicate ruleId \"" + ruleId + "\""); + } String contractSchedule = contract.optString("schedule", "pr"); if ("pr".equals(schedule) && !"pr".equals(contractSchedule)) { continue; // PR runs only PR-scheduled contracts; nightly runs all. @@ -908,8 +1783,13 @@ private List manifestContractNames() throws IOException { JSONObject manifest = loadContractFile(MANIFEST); JSONArray contracts = manifest.getJSONArray("contracts"); List names = new ArrayList<>(); + Set unique = new LinkedHashSet<>(); for (int i = 0; i < contracts.length(); i++) { - names.add(contracts.getString(i)); + String name = contracts.getString(i); + if (!unique.add(name)) { + throw new IOException("contract manifest contains duplicate file \"" + name + "\""); + } + names.add(name); } return names; } @@ -950,19 +1830,16 @@ private JSONObject reportEntry( .put("queryName", queryName) .put("role", role) .put("query", query) - .put("kind", kind); + .put("kind", kind) + .put("executionBackend", executionBackend.id); } - private void writeReport(JSONArray report) { + private void writeReport(JSONArray report) throws IOException { String target = System.getProperty("ppl.lint.report"); if (target == null || target.isEmpty()) { return; } - try { - Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); - } catch (IOException e) { - System.err.println("[ppl-lint] could not write backend report to " + target + ": " + e); - } + Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); } private void log(String ruleId, String caseId, String message) { @@ -970,4 +1847,27 @@ private void log(String ruleId, String caseId, String message) { String.format( Locale.ROOT, "[ppl-lint-backend-contract] %s/%s: %s", ruleId, caseId, message)); } + + private enum ExecutionBackend { + STANDARD("standard", "lucene"), + ANALYTICS("analytics", "composite-parquet"); + + private final String id; + private final String storage; + + ExecutionBackend(String id, String storage) { + this.id = id; + this.storage = storage; + } + + private static ExecutionBackend parse(String value) { + for (ExecutionBackend backend : values()) { + if (backend.id.equals(value)) { + return backend; + } + } + throw new IllegalArgumentException( + EXECUTION_BACKEND_PROPERTY + " must be standard or analytics but was \"" + value + "\""); + } + } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index e1063eddc7d..bc30f5ecb77 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", + "note": "The detector flags both division and modulo by a literal zero because both operations return null silently. The backend result-shape oracle verifies that behavior independently for each operator.", "grammarSurface": "both", "schedule": "pr", "wiring": { @@ -39,8 +39,8 @@ "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" }, - "modulo-by-zero-not-flagged": { - "role": "control", + "modulo-by-zero-literal": { + "role": "trigger", "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" } }, @@ -80,8 +80,9 @@ } } }, - "modulo-by-zero-not-flagged": { - "detectorCount": 0, + "modulo-by-zero-literal": { + "detectorCount": 1, + "severity": "warning", "backend": { "kind": "result-shape", "httpStatus": 200, diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh index 295168f8aad..11e77f1e4ba 100755 --- a/scripts/ppl-lint-rule-validation.sh +++ b/scripts/ppl-lint-rule-validation.sh @@ -36,6 +36,13 @@ # # # Run the full nightly corpus (all rules + coverage assertion) # PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh +# +# # Run the same corpus through composite/Parquet + DataFusion +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Pass local analytics plugin ZIP overrides through to Gradle +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ +# -PanalyticsEngineZip=/path/to/analytics-engine.zip set -euo pipefail @@ -50,6 +57,7 @@ DETECTOR_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" # pr (fast, blocking subset) or nightly (full corpus + coverage assertion). PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" +RUN_ANALYTICS="${RUN_ANALYTICS:-0}" # Candidate artifacts the backend half exports and the detector half consumes. GRAMMAR_BUNDLE="$SQL_ROOT/ppl-grammar-bundle.json" @@ -60,12 +68,30 @@ DETECTOR_REPORT="$SQL_ROOT/detector-report.json" log() { echo "[ppl-lint-rule-validation] $*"; } run_backend() { - log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" - ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ - -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ - -Dppl.lint.report="$BACKEND_REPORT" \ - -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" \ + local backend="standard" + local gradle_args=( + :integ-test:integTest + --tests "$IT_CLASS" + ) + if [[ "$RUN_ANALYTICS" == "1" ]]; then + backend="analytics" + gradle_args=(:integ-test:analyticsEnginePplLintIT) + # The checked-in schema-v3 contracts intentionally have no analytics + # oracles yet. Execute them once and retain their raw observations without + # borrowing the standard route's oracle. + gradle_args+=(-Dppl.lint.observe.only=true) + fi + + log "Running $backend backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + gradle_args+=( + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" + -Dppl.lint.execution_backend="$backend" + -Dppl.lint.sql_sha="$(git rev-parse HEAD)" + -Dppl.lint.report="$BACKEND_REPORT" + -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" -Dppl.lint.target="$TARGET_MANIFEST" + ) + ./gradlew "${gradle_args[@]}" "$@" log "Backend integration test passed. Exported: $(basename "$GRAMMAR_BUNDLE"), $(basename "$TARGET_MANIFEST")." } @@ -93,13 +119,15 @@ run_detector() { PPL_LINT_TARGET_MANIFEST="$TARGET_MANIFEST" \ PPL_LINT_BACKEND_REPORT="$BACKEND_REPORT" \ PPL_LINT_REPORT="$DETECTOR_REPORT" \ + PPL_LINT_OBSERVE_ONLY="$RUN_ANALYTICS" \ + PPL_LINT_OBSERVE_ANALYTICS="$RUN_ANALYTICS" \ node -r ./src/setup_node_env "$DETECTOR_SCRIPT" ) log "Detector validation passed." } if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then - run_backend + run_backend "$@" else log "SKIP_BACKEND=1 — skipping the SQL backend integration test (using existing artifacts)." fi diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 688d3439cc7..db60cbe23ca 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -10,6 +10,7 @@ or a semantic change makes a flagged query valid) without touching OSD. Neither repository's own unit tests catch that. This check does. - **Design:** `ppl-lint-ci-validation-design.md` +- **Analytics rollout:** [`docs/dev/ppl-lint-analytics-engine-ci-validation.md`](../../docs/dev/ppl-lint-analytics-engine-ci-validation.md) - **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) - **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) @@ -26,8 +27,8 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j the Gradle test cluster, runs each contract's trigger/control queries against `POST /_plugins/_ppl`, and — while the cluster is alive — exports: - `ppl-grammar-bundle.json` — the candidate runtime grammar (`GET /_plugins/_ppl/_grammar`); - - `target.json` — `{ engineVersion, grammarHash, grammarBundle }`; - - `backend-report.json` — the observed HTTP behavior per query. + - `target.json` — schema-v2 engine, grammar, execution-backend, storage, and route identity; + - `backend-report.json` — the observed HTTP behavior and execution backend per query. 2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a Node code dependency (no OSD server, no Monaco, no browser), then runs [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner @@ -87,6 +88,14 @@ OSD_REF= ./scripts/ppl-lint-rule-validation.sh # Full nightly corpus + coverage assertion. PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh +# Run the corpus through the full composite/Parquet + DataFusion stack. +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh + +# Use locally built analytics plugins (all trailing arguments pass to Gradle). +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ + -PanalyticsEngineZip=/path/to/analytics-engine.zip \ + -PnativeLibPath=/path/to/native/release + # Re-run only one half (detector needs the backend artifacts to exist). SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh @@ -106,12 +115,12 @@ writes `detector-report.json`. | `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | | `PPL_LINT_SCHEDULE` | `pr` or `nightly` | | `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | -| `PPL_LINT_TARGET_MANIFEST` | `target.json` (engine version + grammar hash) | +| `PPL_LINT_TARGET_MANIFEST` | schema-v2 `target.json` (engine, grammar, execution backend, and storage identity) | | `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | | `PPL_LINT_REPORT` | where to write `detector-report.json` | | `PPL_LINT_CONTRACT_FILE` | (optional) run a single spec instead of the dir | -## Contract format (schema v3) +## Contract format (schema v3 and v4) One JSON file per rule under `contracts/`, listed in `manifest.json`. Each file has a top-level `queries` map (each `{ role: "trigger"|"control", query }`) and a @@ -120,7 +129,7 @@ backend version (zero or more than one fails before any query runs). ```jsonc { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "union-min-datasets", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -139,11 +148,17 @@ backend version (zero or more than one fails before any query runs). "queries": { "union-single-dataset": { "detectorCount": 1, "severity": "error", - "backend": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + "backends": { + "standard": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } }, + "analytics": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + } }, "union-two-datasets-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backends": { + "standard": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } }, + "analytics": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } } } } @@ -151,9 +166,18 @@ backend version (zero or more than one fails before any query runs). } ``` -`backend.kind` is one of `rejection` (contracted 4xx + error type/reason), +Schema v3's `backend` is read only as `backends.standard`; it is never an +implicit analytics oracle. Schema v4's `backends` selects the configured +`standard` or `analytics` execution backend. Missing analytics oracles are +recorded as unscored coverage during observation, including the raw backend +response needed to review a schema-v4 oracle, and are fatal before promotion to +the required check. The selected expectation must contain exactly the same query +names as the top-level `queries` map. + +Each backend `kind` is one of `rejection` (contracted 4xx + error type/reason), `result-shape` (200 with datarow expectations), or `advisory` (soft 200-only -oracle). When a behavior changes in a new version, keep **both** version-scoped +oracle). `not-applicable` requires a reason, owner, and tracking issue. When a behavior +changes in a new version, keep **both** version-scoped expectations so the nightly matrix proves the rule still fires on the old version while the candidate check proves the fix on the new one. @@ -214,7 +238,7 @@ validates every `defaultError` rule against several engine versions at once, and reports **what to change in the linter** when one disagrees. ``` -observe (matrix: 3.6.0, 3.7.0 released images + pr-build) +observe (matrix: released images + pr-build standard + pr-build analytics) └── each leg exports the same 4 artifacts as the single-version check detect (one OSD bootstrap, one detector pass per leg's grammar) └── aggregate-versions.mjs → drift-report.json + remediation report @@ -222,7 +246,10 @@ detect (one OSD bootstrap, one detector pass per leg's grammar) Released legs run the official `opensearchproject/opensearch:` image, which bundles the matching `opensearch-sql` plugin, so no old branch is built. The -`pr-build` leg is the same Gradle test cluster the single-version check uses. Both +`pr-build` leg is the same Gradle test cluster the single-version check uses. The +`pr-build-analytics` leg installs the full Arrow, analytics, composite, Parquet, +Lucene-backend, and DataFusion-backend stack and fails unless fixture settings, +explain output, and a profiled canary attest the route. These legs run the **same** contract oracle (`PplLintRuleValidationIT`) with `-Dppl.lint.observe.only=true`, which records real behavior instead of asserting against expectations — on an older engine a mismatch is the signal being @@ -263,7 +290,8 @@ cells read `n/a (surface)`, and a rule whose every case is inert is `n/a` — no is nothing to re-run). Two legs may share an engine version while validating different surfaces, so the -matrix is keyed on the **leg label**, not the version. +matrix is keyed on the **leg label**, grammar surface, and execution backend, not +the version alone. Each contract declares the surface(s) it was verified against, and a contract is only scored on a matching leg — `"both"` opts into either. Judged on a surface it @@ -304,6 +332,7 @@ Every finding names a drift class, the evidence, and one remediation action: | `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | | `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | | `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | +| `align-execution-backends` | standard and analytics disagree for the same SQL version and grammar | reconcile the detector with both routes or add a reliable backend signal to OSD | Drift classes: `grammar-rule-missing` (a parser rule the detector walks was renamed or removed — the finding names the closest current rule names), @@ -311,7 +340,9 @@ renamed or removed — the finding names the closest current rule names), verdict flipped), `engine-message-changed` (same verdict, reworded error), `detector-silent` / `detector-noisy` (false negative / false positive), `version-scope-too-narrow` (the engine rejects but the rule is scoped away from -that version, so users see no diagnostic), and `severity-mismatch`. +that version, so users see no diagnostic), `execution-backend-divergence` (same +version, different route verdict), and `severity-mismatch`. Backend divergence +never recommends changing a version range. #### Full vs partial relaxation: scope the rule, or narrow the detector? @@ -410,6 +441,14 @@ mkdir -p legs/3.7.0 -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ -Dppl.lint.target=$PWD/legs/3.7.0/target.json +# Observe the PR build through composite/Parquet + DataFusion. +mkdir -p legs/pr-build-analytics +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$PWD/legs/pr-build-analytics/backend-report.json \ + -Dppl.lint.grammar.bundle=$PWD/legs/pr-build-analytics/ppl-grammar-bundle.json \ + -Dppl.lint.target=$PWD/legs/pr-build-analytics/target.json + # Lint each leg's grammar from an OSD checkout (writes detector-report.json), # then compare every version at once: node scripts/ppl-lint/aggregate-versions.mjs \ diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 8100829eb9d..76f64e31348 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -114,12 +114,41 @@ function writeLeg({ version, cases, parserRuleNames = ['unionCommand', 'unionDataset'], - defaultErrorRules, + defaultErrorRules = [SPEC.ruleId], + executionBackend = 'standard', + grammarHash = `sha256:${version}`, + surface = 'runtime-bundle', + explicitIdentity = true, }) { const dir = makeTmp(`ppl-lint-leg-${version}-`); + const target = { + engineVersion: version, + grammarHash, + ...(explicitIdentity + ? { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + } + : {}), + }; fs.writeFileSync( path.join(dir, 'target.json'), - JSON.stringify({ engineVersion: version, grammarHash: `sha256:${version}` }) + JSON.stringify(target) ); fs.writeFileSync( path.join(dir, 'ppl-grammar-bundle.json'), @@ -137,6 +166,9 @@ function writeLeg({ expected: role === 'trigger' ? 1 : 0, actual: c.detector, severities: c.severities || (c.detector > 0 ? ['error'] : []), + severityMatched: c.severityMatched ?? true, + messageMatched: c.messageMatched ?? true, + ...(explicitIdentity ? { executionBackend } : {}), }); backend.push({ ruleId: SPEC.ruleId, @@ -144,15 +176,31 @@ function writeLeg({ role, rejected: !!c.rejected, observed: { - httpStatus: c.rejected ? 400 : 200, + httpStatus: c.httpStatus || (c.rejected ? 400 : 200), rejected: !!c.rejected, ...(c.rejected ? { type: c.type || REJECTION.type, reason: c.reason || REJECTION.reason } : {}), }, + ...(c.outcome ? { outcome: c.outcome } : {}), + ...(c.error ? { error: c.error } : {}), + ...(explicitIdentity ? { executionBackend } : {}), }); } + const detectorIdentity = explicitIdentity + ? { + schemaVersion: 2, + executionBackend, + engineVersion: version, + grammarHash, + } + : {}; fs.writeFileSync( path.join(dir, 'detector-report.json'), - JSON.stringify({ results, ...(defaultErrorRules ? { defaultErrorRules } : {}) }) + JSON.stringify({ + ...detectorIdentity, + surface, + results, + ...(defaultErrorRules !== null ? { defaultErrorRules } : {}), + }) ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); return dir; @@ -163,7 +211,8 @@ function run({ contracts, legs, extraArgs = [] }) { const outDir = makeTmp('ppl-lint-out-'); const out = path.join(outDir, 'drift-report.json'); const args = [SCRIPT, '--contracts', contracts, '--out', out]; - for (const [version, dir] of Object.entries(legs)) { + const entries = Array.isArray(legs) ? legs : Object.entries(legs); + for (const [version, dir] of entries) { args.push('--leg', `${version}=${dir}`); } args.push(...extraArgs); @@ -186,6 +235,43 @@ function healthyLegs() { }; } +function writeSchema4Contracts({ includeAnalytics = true } = {}) { + const routeOracles = (standard, analytics) => ({ + standard, + ...(includeAnalytics ? { analytics } : {}), + }); + return writeContracts({ + schemaVersion: 4, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backends: routeOracles( + { + kind: 'rejection', + httpStatus: 400, + body: { status: 400, error: REJECTION }, + }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + control: { + detectorCount: 0, + backends: routeOracles( + { kind: 'result-shape', httpStatus: 200 }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + }, + }, + ], + }); +} + test('all versions agreeing exits 0 and reports no drift', () => { const { status, report, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); assert.equal(status, 0); @@ -197,6 +283,460 @@ test('all versions agreeing exits 0 and reports no drift', () => { assert.ok(report.matrix.every((m) => m.status === 'agree')); }); +test('same-version standard and analytics verdicts are classified as backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report, stdout } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + + assert.equal(status, 1); + assert.equal(report.schemaVersion, 2); + assert.equal(report.legs.length, 2); + assert.equal(new Set(report.legs.map((leg) => leg.key)).size, 2); + assert.deepEqual( + report.legs.map((leg) => leg.executionBackend).sort(), + ['analytics', 'standard'] + ); + assert.equal(report.backendPairs.length, 1); + assert.ok(report.matrix.every((row) => row.status === 'drift')); + assert.ok(report.matrix.every((row) => row.key.includes(row.executionBackend))); + + const divergence = report.drifts.find( + (drift) => drift.driftClass === 'execution-backend-divergence' + ); + assert.ok(divergence); + assert.deepEqual(divergence.executionBackends, ['standard', 'analytics']); + assert.match(divergence.key, /standard-vs-analytics/); + assert.equal(divergence.remediation.action, 'align-execution-backends'); + assert.doesNotMatch(divergence.remediation.detail, /maxVersion|minVersion|scope/i); + assert.equal( + report.drifts.filter( + (drift) => + drift.driftClass === 'engine-relaxed' || drift.driftClass === 'engine-tightened' + ).length, + 0, + 'route differences must not be rendered as product-version drift' + ); + assert.match(stdout, /`3\.8\.0`
      standard/); + assert.match(stdout, /`3\.8\.0`
      analytics/); +}); + +test('schema-v3 analytics has explicit backend-oracle coverage holes', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report, stdout } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.executionBackend === 'analytics')); + assert.ok(report.coverageHoles.every((hole) => hole.kind === 'backend-oracle')); + assert.ok(report.coverageHoles.every((hole) => /standard-only/.test(hole.reason))); + assert.equal(report.matrix[0].status, 'uncovered'); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /schema-v3 backend oracles are standard-only/); +}); + +test('analytics observation mode reports schema-v3 coverage without failing', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal(report.result.enforcedCoverageHoles, 0); + assert.equal(report.result.observedAnalyticsFindings, 2); + assert.ok(report.coverageHoles.every((hole) => hole.blocking === false)); +}); + +test('schema-v3 raw analytics observations still expose backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ...entry, + kind: 'coverage-missing', + outcome: 'coverage-missing', + coverage: 'missing', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal( + report.drifts.filter((drift) => drift.driftClass === 'execution-backend-divergence') + .length, + 1 + ); + assert.equal(report.result.observedAnalyticsFindings, 3); +}); + +test('backend divergence cannot hide a standard-route regression', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.ok( + report.drifts.some( + (drift) => + drift.executionBackend === 'standard' && + drift.driftClass === 'engine-relaxed' + ), + 'the blocking standard-route regression must survive paired-route classification' + ); + assert.ok( + report.drifts.some( + (drift) => drift.driftClass === 'execution-backend-divergence' + ) + ); + assert.ok(report.result.enforcedDriftCount > 0); +}); + +test('not-applicable cannot waive an enforced analytics backend oracle', () => { + const contracts = writeSchema4Contracts(); + const contractFile = path.join(contracts, 'union.spec.json'); + const contract = JSON.parse(fs.readFileSync(contractFile, 'utf8')); + for (const query of Object.values(contract.expectations[0].queries)) { + query.backends.analytics = { + kind: 'not-applicable', + reason: 'analytics fixture is not supported yet', + owner: '@analytics-team', + issue: 'https://example.test/issues/42', + }; + } + fs.writeFileSync(contractFile, JSON.stringify(contract)); + + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + kind: 'not-applicable', + outcome: 'not-applicable', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts, + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.issue.endsWith('/42'))); + assert.equal(report.matrix[0].status, 'uncovered'); +}); + +test('analytics coverage gaps cannot hide missing raw observations', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: 'analytics', + kind: 'coverage-missing', + outcome: 'error', + error: 'connect timeout', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(report.inconclusive[0].reasons.join(' '), /no engine verdict/); +}); + +test('paired detector reports must be identical across execution backends', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(analytics, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.find((entry) => entry.queryName === 'trigger').actual = 0; + detector.results.find((entry) => entry.queryName === 'trigger').severities = []; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 2); + assert.match(stderr, /detector parity failed for union-min-datasets::trigger/); +}); + +test('target and detector execution identities must match', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.executionBackend = 'standard'; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /detector report executionBackend "standard" does not match target "analytics"/); +}); + +test('unknown target execution backends are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + cases: { trigger: { detector: 1, rejected: true } }, + }); + const targetFile = path.join(dir, 'target.json'); + const target = JSON.parse(fs.readFileSync(targetFile, 'utf8')); + target.executionBackend = 'experimental'; + fs.writeFileSync(targetFile, JSON.stringify(target)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['pr-build', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /must be "standard" or "analytics"/); +}); + +test('every backend row must match its target execution identity', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend[0].executionBackend = 'standard'; + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /backend report row .* does not match target "analytics"/); +}); + +test('duplicate backend row keys are rejected instead of overwritten', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend.push({ ...backend[0] }); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate backend report key "union-min-datasets::trigger"/); +}); + +test('duplicate detector row keys are rejected instead of selecting the first', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate detector report key "union-min-datasets::trigger"/); +}); + +test('duplicate backend-qualified leg identities are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [ + ['3.8.0', dir], + ['3.8.0', dir], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate leg identity/); +}); + +test('paired standard and analytics legs require the same runtime grammar hash', () => { + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash: 'sha256:standard', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash: 'sha256:analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /different grammar hashes/); +}); + test('a version where only one engine relaxed is red, and names just that version', () => { const legs = healthyLegs(); // 3.8 now accepts what 3.7 still rejects, while the detector keeps flagging. @@ -215,6 +755,54 @@ test('a version where only one engine relaxed is red, and names just that versio assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); }); +test('a changed rejection HTTP status is semantic drift, not agreement', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true, httpStatus: 500 }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /HTTP status changed from 400 to 500/); + assert.equal(drift.remediation.action, 'review-backend-oracle'); +}); + +test('a same-verdict result-shape mismatch cannot pass aggregation', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { + detector: 0, + rejected: false, + outcome: 'observed-mismatch', + error: 'expected non-empty datarows', + }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /expected non-empty datarows/); +}); + // --- partial vs full relaxation, end to end --------------------------------- // // Driven through the real script because the bug this guards is in the AGGREGATION: @@ -460,12 +1048,55 @@ test('a census matching the manifest keeps the check green', () => { assert.equal(report.result.missingContractCount, 0); }); -test('a legacy detector report without a census warns instead of failing', () => { - // Older detector builds do not emit defaultErrorRules; the aggregator must say - // so out loud rather than quietly reporting full coverage. - const { status, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); - assert.equal(status, 0); - assert.match(stdout, /no detector leg reported a defaultErrorRules census/); +test('a schema-v2 detector report without a census fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: null, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules must be a JSON array/); +}); + +test('a schema-v2 detector report with an unknown grammar surface fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + surface: 'unknown-surface', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /surface must be "runtime-bundle" or "compiled-simplified"/); +}); + +test('a schema-v2 detector census rejects duplicate rule identities', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: [SPEC.ruleId, SPEC.ruleId], + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules contains duplicate rule/); }); // --- "we don't know" must never render as "it's fine" ------------------------- @@ -478,7 +1109,14 @@ function writeLegWithTransportError({ version, erroredQuery, cases }) { entry.queryName === erroredQuery ? // Exactly what the IT writes on a transport failure: an `error` outcome and // NO `rejected` field, because no verdict was ever received. - { ruleId: entry.ruleId, queryName: entry.queryName, role: entry.role, outcome: 'error', error: 'connect timeout' } + { + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + outcome: 'error', + error: 'connect timeout', + } : { ...entry, outcome: 'observed' } ); fs.writeFileSync(file, JSON.stringify(backend)); @@ -523,11 +1161,19 @@ test('losing every trigger is inconclusive even when a control still compares', fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', + executionBackend: 'standard', rejected: false, outcome: 'observed', observed: { httpStatus: 200, rejected: false }, @@ -549,8 +1195,22 @@ test('a leg where nothing could be compared is inconclusive, not agreement', () fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, - { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, ]) ); const { status, report, stdout } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); @@ -622,18 +1282,29 @@ test('an errored trigger on an out-of-scope rule does not silently pass', () => fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', + executionBackend: 'standard', rejected: false, outcome: 'observed', observed: { httpStatus: 200, rejected: false }, }, ]) ); - const { report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); // The point is that an unobserved trigger yields no CLAIM either way: it must // not be reported as a confident out-of-scope agreement... assert.equal( @@ -643,6 +1314,37 @@ test('an errored trigger on an out-of-scope rule does not silently pass', () => ); // ...nor may it invent linter advice from a verdict that never arrived. assert.equal(report.drifts.length, 0); + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(report.result.enforcedInconclusive, 1); +}); + +test('a missing detector and backend row is inconclusive even when the rule is out of scope', () => { + const dir = writeLeg({ + version: '3.6.0', + cases: { + trigger: { detector: 0, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results = detector.results.filter((entry) => entry.queryName !== 'trigger'); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).filter( + (entry) => entry.queryName !== 'trigger' + ); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(report.inconclusive[0].reasons.join(' '), /trigger \(no detector result\)/); }); test('an errored control cannot fail open into "widen appliesTo" advice', () => { @@ -657,11 +1359,21 @@ test('an errored control cannot fail open into "widen appliesTo" advice', () => const backend = JSON.parse(fs.readFileSync(path.join(dir, 'backend-report.json'), 'utf8')).map( (e) => e.role === 'control' - ? { ruleId: e.ruleId, queryName: e.queryName, role: e.role, outcome: 'error', error: 'timeout' } + ? { + ruleId: e.ruleId, + queryName: e.queryName, + role: e.role, + executionBackend: e.executionBackend, + outcome: 'error', + error: 'timeout', + } : { ...e, outcome: 'observed' } ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); - const { report, stdout } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const { status, report, stdout } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); const scoped = report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow'); assert.equal( scoped.length, @@ -669,6 +1381,8 @@ test('an errored control cannot fail open into "widen appliesTo" advice', () => 'with the control unobserved there is no evidence the command is supported, so no widening advice' ); assert.ok(!/Widen "/.test(stdout)); + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); }); test('a bad --leg argument is rejected', () => { diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs index dc55503a6ad..357b046d47c 100644 --- a/scripts/ppl-lint/__tests__/annotate.test.mjs +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -129,6 +129,29 @@ test('a non-enforced drift is a warning so it cannot be read as blocking', () => assert.equal(annotations[0].level, 'warning'); }); +test('backend divergence annotations name both execution routes', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + executionBackend: 'analytics', + executionBackends: ['standard', 'analytics'], + driftClass: 'execution-backend-divergence', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'standard rejected while analytics accepted', + remediation: { action: 'align-execution-backends', detail: 'Align route behavior.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.match(annotations[0].title, /standard vs analytics/); + assert.match(annotations[0].title, /execution-backend-divergence/); +}); + test('an unvalidated rule has no file to point at', () => { const annotations = buildAnnotations( { missingContracts: [{ ruleId: 'sort-on-eval-field', reason: 'has no contract file' }] }, diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs new file mode 100644 index 00000000000..1c8e1f4b21d --- /dev/null +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'assemble-run-manifest.mjs'); +const tmpDirs = []; + +function makeRun() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-manifest-')); + tmpDirs.push(dir); + fs.mkdirSync(path.join(dir, 'artifacts')); + return dir; +} + +function writeJson(dir, name, value) { + fs.writeFileSync(path.join(dir, 'artifacts', name), JSON.stringify(value)); +} + +function validArtifacts(dir) { + writeJson(dir, 'target.json', { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + rejected: false, + observed: { httpStatus: 200, rejected: false, response: { datarows: [] } }, + outcome: 'pass', + }, + ]); + writeJson(dir, 'detector-report.json', { + schemaVersion: 2, + executionBackend: 'standard', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + surface: 'runtime-bundle', + defaultErrorRules: ['advisory-rule'], + results: [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + expected: 1, + actual: 1, + severities: ['warning'], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }, + ], + }); +} + +function run(dir, extraEnv = {}) { + return spawnSync(process.execPath, [SCRIPT], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + BACKEND_RESULT: 'success', + DETECTOR_RESULT: 'success', + SQL_SHA: 'candidate-sql-sha', + ...extraEnv, + }, + }); +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('valid standard artifacts produce a passing schema-v2 manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + const result = run(dir); + assert.equal(result.status, 0, result.stderr); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.schemaVersion, 2); + assert.equal(manifest.executionBackend, 'standard'); + assert.equal(manifest.result.passed, true); + assert.deepEqual(manifest.result.artifactErrors, []); +}); + +test('a missing report fails closed while still writing the manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + fs.rmSync(path.join(dir, 'artifacts', 'backend-report.json')); + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /required artifact is missing/); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.result.passed, false); +}); + +test('detector rows must have unique identities matching the target', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(file, JSON.stringify(detector)); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /duplicate row advisory-rule::trigger/); +}); + +test('a backend row without a real verdict cannot render as acceptance', () => { + const dir = makeRun(); + validArtifacts(dir); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'connect timeout', + }, + ]); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not pass its oracle/); +}); + +test('an accepted advisory trigger is summarized from its backend outcome, not its role', () => { + const dir = makeRun(); + validArtifacts(dir); + const summary = path.join(dir, 'summary.md'); + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.equal(result.status, 0, result.stderr); + const markdown = fs.readFileSync(summary, 'utf8'); + assert.match(markdown, /advisory-rule.*accepted.*Pass/); +}); + +test('detector severity and message mismatches fail the manifest and summary', () => { + for (const field of ['severityMatched', 'messageMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`did not match its ${field === 'severityMatched' ? 'severity' : 'message'}`)); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); + } +}); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs new file mode 100644 index 00000000000..6dce090d45d --- /dev/null +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -0,0 +1,469 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from '../contract-schema.mjs'; + +const QUERY = { + detectorCount: 1, + severity: 'error', + matchMessage: 'bad query', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, +}; + +function spec(schemaVersion, queryExpectation = QUERY) { + return { + schemaVersion, + ruleId: 'example-rule', + queries: { + trigger: { role: 'trigger', query: 'source={{index}} | bad' }, + control: { role: 'control', query: 'source={{index}} | head 1' }, + }, + expectations: [ + { + version: '>=3.7.0', + queries: { + trigger: queryExpectation, + control: { + detectorCount: 0, + ...(schemaVersion === 3 + ? { backend: { kind: 'result-shape', httpStatus: 200 } } + : { + backends: { + standard: { kind: 'result-shape', httpStatus: 200 }, + analytics: { kind: 'result-shape', httpStatus: 200 }, + }, + }), + }, + }, + }, + ], + }; +} + +test('target schema v2 requires and preserves explicit standard or analytics identity', () => { + for (const executionBackend of ['standard', 'analytics']) { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:abc', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + }); + assert.equal(target.executionBackend, executionBackend); + assert.equal(target.legacy, false); + } +}); + +test('unversioned targets cannot infer a standard execution identity', () => { + assert.throws( + () => + normalizeTarget({ + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + }), + /target\.schemaVersion is required/ + ); +}); + +test('unknown target schema and execution backend are rejected', () => { + assert.throws( + () => + normalizeTarget({ + schemaVersion: 3, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + }), + /Unsupported target schemaVersion 3/ + ); + assert.throws( + () => + normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'experimental', + }), + /must be "standard" or "analytics"/ + ); +}); + +test('schema v3 resolves a standard oracle and never falls back for analytics', () => { + const contract = spec(3); + const standard = resolveBackendOracle(contract, QUERY, 'standard'); + const analytics = resolveBackendOracle(contract, QUERY, 'analytics'); + + assert.equal(standard.status, 'applicable'); + assert.equal(standard.oracle, QUERY.backend); + assert.deepEqual(standard.detector, analytics.detector); + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /standard-only/); +}); + +test('analytics targets fail closed on storage and route attestation', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }; + assert.equal(normalizeTarget(base).executionBackend, 'analytics'); + assert.throws( + () => normalizeTarget({ ...base, storage: 'lucene' }), + /storage must be "composite-parquet"/ + ); + const withoutStack = { ...base }; + delete withoutStack.analyticsStack; + assert.throws( + () => normalizeTarget(withoutStack), + /analyticsStack must be a JSON object/ + ); + assert.throws( + () => + normalizeTarget({ + ...base, + routeAttestation: { ...base.routeAttestation, explainVerified: false }, + }), + /explainVerified must be true/ + ); +}); + +test('schema-v2 standard targets require explicit storage and shard identity', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; + assert.equal(normalizeTarget(base).executionBackend, 'standard'); + assert.throws( + () => normalizeTarget({ ...base, storage: undefined }), + /storage must be "lucene"/ + ); + assert.throws( + () => normalizeTarget({ ...base, shardCount: undefined }), + /shardCount must be a positive integer/ + ); +}); + +test('schema v4 selects only the requested backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'warning', + backends: { + standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + analytics: { kind: 'advisory', httpStatus: 200 }, + }, + }; + const contract = spec(4, queryExpectation); + + const standard = resolveBackendOracle(contract, queryExpectation, 'standard'); + const analytics = resolveBackendOracle(contract, queryExpectation, 'analytics'); + assert.equal(standard.oracle, queryExpectation.backends.standard); + assert.equal(analytics.oracle, queryExpectation.backends.analytics); + assert.deepEqual(standard.detector, analytics.detector); +}); + +test('schema v4 reports missing route coverage without using another backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'error', + backends: { + standard: { kind: 'rejection', httpStatus: 400 }, + }, + }; + const analytics = resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'); + + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /no analytics backend oracle/); +}); + +test('not-applicable is explicit while an absent oracle is coverage-missing', () => { + const notApplicable = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + owner: '@analytics-team', + issue: 'https://example.test/issues/1', + }, + }, + }; + const missing = { + detectorCount: 1, + backends: {}, + }; + + assert.equal( + resolveBackendOracle(spec(4, notApplicable), notApplicable, 'analytics').status, + 'not-applicable' + ); + assert.equal( + resolveBackendOracle(spec(4, missing), missing, 'analytics').status, + 'coverage-missing' + ); + assert.throws( + () => + resolveBackendOracle( + spec(4, { + detectorCount: 1, + backends: { analytics: { kind: 'not-applicable' } }, + }), + { detectorCount: 1, backends: { analytics: { kind: 'not-applicable' } } }, + 'analytics' + ), + /backend oracle\.reason/ + ); + assert.throws( + () => { + const oracle = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + issue: 'https://example.test/issues/1', + }, + }, + }; + return resolveBackendOracle(spec(4, oracle), oracle, 'analytics'); + }, + /backend oracle\.owner/ + ); +}); + +test('unknown contract schema and backend oracle kind are rejected', () => { + assert.throws(() => assertContractSchema(spec(5)), /expected 3 or 4/); + const queryExpectation = { + detectorCount: 1, + backends: { analytics: { kind: 'maybe' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + /unknown analytics backend oracle.kind/ + ); + const unknownBackend = { + detectorCount: 1, + backends: { experimental: { kind: 'advisory' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, unknownBackend), unknownBackend, 'analytics'), + /backends key must be "standard" or "analytics"/ + ); +}); + +test('backend oracle payloads fail closed when required shapes are malformed', () => { + const cases = [ + { + oracle: { kind: 'rejection', body: { status: 400 } }, + expected: /httpStatus/, + }, + { + oracle: { kind: 'rejection', httpStatus: 400 }, + expected: /\.body must be a JSON object/, + }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: '400' }, + }, + expected: /\.body\.status must be an integer/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsNonEmpty: 'yes' }, + }, + expected: /datarowsNonEmpty must be a boolean/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsCount: -1 }, + }, + expected: /datarowsCount must be a non-negative integer/, + }, + ]; + + for (const { oracle, expected } of cases) { + const queryExpectation = { + detectorCount: 1, + backends: { analytics: oracle }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + expected + ); + } +}); + +test('selected expectation query keys must exactly equal top-level query keys', () => { + const contract = spec(3); + assert.deepEqual( + assertExactQueryCoverage(contract, contract.expectations[0]), + ['control', 'trigger'] + ); + + const missing = structuredClone(contract.expectations[0]); + delete missing.queries.control; + assert.throws( + () => assertExactQueryCoverage(contract, missing), + /missing from expectation: control/ + ); + + const extra = structuredClone(contract.expectations[0]); + extra.queries.unknown = QUERY; + assert.throws( + () => assertExactQueryCoverage(contract, extra), + /not present in contract\.queries: unknown/ + ); + + assert.throws( + () => + assertExactQueryCoverage( + { ...contract, queries: {} }, + { ...contract.expectations[0], queries: {} } + ), + /contract\.queries must not be empty/ + ); +}); + +test('non-verdict backend states are never coerced to acceptance', () => { + assert.deepEqual(classifyBackendReportRow({ rejected: false }), { + status: 'observed', + rejected: false, + }); + for (const outcome of ['not-applicable', 'coverage-missing', 'error']) { + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).status, + outcome + ); + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).rejected, + undefined + ); + } + assert.equal(classifyBackendReportRow({ outcome: 'pass' }).status, 'error'); +}); + +test('backend report indexing rejects duplicate keys', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const row = { + ruleId: 'example-rule', + queryName: 'trigger', + executionBackend: 'analytics', + rejected: true, + }; + assert.throws(() => indexBackendReport([row, { ...row }], target), /duplicate backend report key/); + assert.throws(() => indexBackendReport({}, target), /must be a JSON array/); +}); + +test('every schema-v2 backend report row must carry identity matching the target', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const base = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([base], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...base, executionBackend: 'standard' }], target), + /does not match target "analytics"/ + ); + assert.equal( + indexBackendReport([{ ...base, executionBackend: 'analytics' }], target).get( + 'example-rule::trigger' + ).rejected, + true + ); +}); + +test('schema-v2 standard backend rows cannot omit identity', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + const row = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([row], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...row, executionBackend: 'analytics' }], target), + /does not match target "standard"/ + ); +}); diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 251ecb3c9bf..e8e3eecfde6 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -23,6 +23,7 @@ import { DRIFT_CLASSES, REMEDIATIONS, classifyDrift, + classifyExecutionBackendDivergence, classifyRelaxationScope, formatDriftReport, parseVersion, @@ -136,6 +137,39 @@ test('grammar-rule check is skipped when the contract declares no required rules // --- engine behavior flips ---------------------------------------------------- +test('same-candidate route differences use backend remediation, never version scoping', () => { + const drift = classifyExecutionBackendDivergence({ + ruleId: 'union-min-datasets', + version: '3.8.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + standardObserved: { backendRejected: true, backendType: 'IllegalArgumentException' }, + analyticsObserved: { backendRejected: false }, + standardLeg: 'pr-build', + analyticsLeg: 'pr-build-analytics', + grammarHash: 'sha256:same', + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE); + assert.equal(drift.remediation.action, REMEDIATIONS.ALIGN_EXECUTION_BACKENDS); + assert.deepEqual(drift.executionBackends, ['standard', 'analytics']); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); + assert.match(drift.evidence, /standard REJECTED.*analytics ACCEPTED/); +}); + +test('analytics oracle flips are not labeled as product-version relaxation', () => { + const drift = classifyDrift( + agreeingTrigger({ + executionBackend: 'analytics', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.REVIEW_BACKEND_ORACLE); + assert.notEqual(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); +}); + test('engine relaxation with a still-firing detector demands version scoping', () => { const drift = classifyDrift( agreeingTrigger({ @@ -357,6 +391,46 @@ test('a noisy detector the engine agrees with points at the expectation', () => assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); }); +test('a nonzero detector count mismatch is not reduced to flagged versus silent', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 2, + severity: 'error', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + detectorCount: 1, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /expected exactly 2.*emitted 1/); +}); + +test('a detector message mismatch is classified independently of count and severity', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 1, + severity: 'error', + matchMessage: 'requires at least two datasets', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + severityMatched: true, + messageMatched: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /requires at least two datasets/); +}); + // --- severity ---------------------------------------------------------------- test('a downgraded severity is caught even when the count is right', () => { diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index d7d6b25731f..e5dd0510ca6 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -34,8 +34,18 @@ import fs from 'fs'; import path from 'path'; import { emitAnnotations } from './annotate.mjs'; +import { + assertContractSchema, + assertExactQueryCoverage, + assertExecutionBackend, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; import { classifyDrift, + classifyExecutionBackendDivergence, classifyGrammarDrift, classifyRelaxationScope, DRIFT_CLASSES, @@ -55,7 +65,14 @@ function fatal(message) { } function parseArgs(argv) { - const args = { legs: [], contracts: '', out: 'drift-report.json', summary: '', allRules: false }; + const args = { + legs: [], + contracts: '', + out: 'drift-report.json', + summary: '', + allRules: false, + observeAnalytics: false, + }; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; const next = () => { @@ -76,6 +93,8 @@ function parseArgs(argv) { args.summary = next(); } else if (arg === '--all-rules') { args.allRules = true; + } else if (arg === '--observe-analytics') { + args.observeAnalytics = true; } else { fatal(`unknown argument "${arg}"`); } @@ -99,12 +118,206 @@ function readJson(file, { optional = false } = {}) { return undefined; } +function artifactFatal(file, error) { + fatal(`invalid ${file}: ${error.message}`); +} + +function reportRowKey(entry, label) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new TypeError(`${label} row must be a JSON object`); + } + if (typeof entry.ruleId !== 'string' || entry.ruleId.length === 0) { + throw new TypeError(`${label} row.ruleId must be a non-empty string`); + } + if (typeof entry.queryName !== 'string' || entry.queryName.length === 0) { + throw new TypeError(`${label} row.queryName must be a non-empty string`); + } + return `${entry.ruleId}::${entry.queryName}`; +} + +function rowExecutionBackend(entry, target, label, key) { + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error(`${label} row ${key} is missing executionBackend for a schema-v2 target`); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(entry.executionBackend, `${label} row ${key}.executionBackend`) + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `${label} row ${key} executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + return executionBackend; +} + +function validateOptionalRowIdentity(entry, target, label, key) { + for (const field of ['engineVersion', 'grammarHash']) { + if ( + Object.prototype.hasOwnProperty.call(entry, field) && + entry[field] !== target[field] + ) { + throw new Error( + `${label} row ${key} ${field} ${JSON.stringify(entry[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } +} + +function normalizeDetectorReport(detector, target) { + if (!detector || typeof detector !== 'object' || Array.isArray(detector)) { + throw new TypeError('detector report must be a JSON object'); + } + + const hasIdentity = Object.prototype.hasOwnProperty.call(detector, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error('detector report is missing executionBackend for a schema-v2 target'); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(detector.executionBackend, 'detector report.executionBackend') + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `detector report executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + if (!target.legacy && detector.schemaVersion !== 2) { + throw new Error( + `detector report schemaVersion ${JSON.stringify(detector.schemaVersion)} does not match ` + + 'schema-v2 target' + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + const hasField = Object.prototype.hasOwnProperty.call(detector, field); + if (!hasField && !target.legacy) { + throw new Error(`detector report is missing ${field} for a schema-v2 target`); + } + if (hasField && detector[field] !== target[field]) { + throw new Error( + `detector report ${field} ${JSON.stringify(detector[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } + if (!Array.isArray(detector.results)) { + throw new TypeError('detector report.results must be a JSON array'); + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + throw new Error( + `detector report.surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + throw new TypeError('detector report.defaultErrorRules must be a JSON array'); + } + const census = new Set(); + for (const ruleId of detector.defaultErrorRules) { + if (typeof ruleId !== 'string' || ruleId.length === 0) { + throw new TypeError('detector report.defaultErrorRules entries must be non-empty strings'); + } + if (census.has(ruleId)) { + throw new Error(`detector report.defaultErrorRules contains duplicate rule "${ruleId}"`); + } + census.add(ruleId); + } + + const results = new Map(); + for (const entry of detector.results) { + const key = reportRowKey(entry, 'detector report'); + rowExecutionBackend(entry, target, 'detector report', key); + validateOptionalRowIdentity(entry, target, 'detector report', key); + if (results.has(key)) { + throw new Error(`duplicate detector report key "${key}"`); + } + if (!entry.notApplicable && entry.outcome !== 'not-applicable') { + if (!Number.isInteger(entry.expected) || entry.expected < 0) { + throw new TypeError(`detector report row ${key}.expected must be a non-negative integer`); + } + if (!Number.isInteger(entry.actual) || entry.actual < 0) { + throw new TypeError(`detector report row ${key}.actual must be a non-negative integer`); + } + if (!Array.isArray(entry.severities)) { + throw new TypeError(`detector report row ${key}.severities must be a JSON array`); + } + if (typeof entry.severityMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.severityMatched must be a boolean`); + } + if (typeof entry.messageMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.messageMatched must be a boolean`); + } + } + results.set(key, entry); + } + return { ...detector, executionBackend, resultsByKey: results }; +} + +function makeLegKey({ label, version, surface, executionBackend }) { + return [label, version, surface, executionBackend] + .map((part) => encodeURIComponent(part)) + .join('::'); +} + +function legFields(leg) { + return { + version: leg.version, + leg: leg.label, + legKey: leg.key, + executionBackend: leg.executionBackend, + }; +} + +function findingKey(finding) { + const backend = Array.isArray(finding.executionBackends) + ? finding.executionBackends.join('-vs-') + : finding.executionBackend || 'standard'; + return [ + finding.legKey || finding.leg || finding.version, + backend, + finding.ruleId, + finding.queryName || '', + finding.driftClass, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + +function reportItemKey(item, kind) { + return [ + item.legKey || item.leg || item.version, + item.executionBackend || 'standard', + item.ruleId, + item.queryName || '', + kind, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + /** Load the contract corpus, keyed by ruleId, plus the manifest's enforced sets. */ function loadContracts(dir) { const manifest = readJson(path.join(dir, 'manifest.json')); const specs = new Map(); for (const name of manifest.contracts || []) { const spec = readJson(path.join(dir, name)); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + } catch (error) { + artifactFatal(path.join(dir, name), error); + } specs.set(spec.ruleId, { spec, file: name }); } // `defaultError` is the multi-version enforced set: every rule that ships @@ -125,14 +338,45 @@ function loadContracts(dir) { * exists to prevent. */ function loadLeg({ version, dir }) { - const target = readJson(path.join(dir, 'target.json')); - const detector = readJson(path.join(dir, 'detector-report.json')); + const targetFile = path.join(dir, 'target.json'); + const detectorFile = path.join(dir, 'detector-report.json'); + const backendFile = path.join(dir, 'backend-report.json'); + const targetRaw = readJson(targetFile); + const detectorRaw = readJson(detectorFile); const backendRaw = readJson(path.join(dir, 'backend-report.json')); const bundle = readJson(path.join(dir, 'ppl-grammar-bundle.json'), { optional: true }); - const backend = new Map(); - for (const entry of Array.isArray(backendRaw) ? backendRaw : []) { - backend.set(`${entry.ruleId}::${entry.queryName}`, entry); + let target; + let detector; + let backend; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactFatal(targetFile, error); + } + try { + detector = normalizeDetectorReport(detectorRaw, target); + } catch (error) { + artifactFatal(detectorFile, error); + } + try { + backend = indexBackendReport(backendRaw, target); + for (const [key, entry] of backend) { + validateOptionalRowIdentity(entry, target, 'backend report', key); + } + } catch (error) { + artifactFatal(backendFile, error); + } + if ( + bundle && + Object.prototype.hasOwnProperty.call(bundle, 'grammarHash') && + bundle.grammarHash !== target.grammarHash + ) { + fatal( + `grammar bundle ${path.join(dir, 'ppl-grammar-bundle.json')} reports ` + + `${JSON.stringify(bundle.grammarHash)} but target reports ` + + `${JSON.stringify(target.grammarHash)}` + ); } // The engine's self-reported version wins over the matrix label, so a matrix @@ -145,11 +389,15 @@ function loadLeg({ version, dir }) { ); } - return { + const leg = { version: reported || version, label: version, dir, grammarHash: target.grammarHash || '', + sqlSha: target.sqlSha || '', + executionBackend: target.executionBackend, + targetSchemaVersion: target.schemaVersion, + legacyTarget: target.legacy, // Which of OSD's two lint surfaces this leg validated. Older detector reports // predate the field; they were all runtime-bundle runs. surface: detector.surface || 'runtime-bundle', @@ -157,6 +405,221 @@ function loadLeg({ version, dir }) { detector, backend, }; + leg.key = makeLegKey(leg); + return leg; +} + +function pairBackendLegs(legs) { + const identities = new Set(); + for (const leg of legs) { + if (identities.has(leg.key)) { + fatal(`duplicate leg identity "${leg.key}"`); + } + identities.add(leg.key); + } + + const runtimeLegs = legs.filter((leg) => leg.surface === 'runtime-bundle'); + const standards = runtimeLegs.filter((leg) => leg.executionBackend === 'standard'); + const analyticsLegs = runtimeLegs.filter((leg) => leg.executionBackend === 'analytics'); + const usedStandards = new Set(); + const pairs = []; + const neutralLabel = (label) => String(label).replace(/[-_](?:standard|analytics)$/i, ''); + + for (const analytics of analyticsLegs) { + const labelPeers = standards.filter( + (standard) => neutralLabel(standard.label) === neutralLabel(analytics.label) + ); + const candidates = + labelPeers.length > 0 + ? labelPeers + : standards.filter((standard) => standard.version === analytics.version); + if (candidates.length === 0) { + if (standards.length > 0) { + fatal( + `analytics leg "${analytics.key}" has no standard peer for engine ` + + `${analytics.version}` + ); + } + continue; + } + + const sameLabel = candidates.filter((standard) => standard.label === analytics.label); + const sameGrammar = candidates.filter( + (standard) => standard.grammarHash === analytics.grammarHash + ); + let standard; + if (sameLabel.length === 1) { + standard = sameLabel[0]; + } else if (sameGrammar.length === 1) { + standard = sameGrammar[0]; + } else if (candidates.length === 1) { + standard = candidates[0]; + } else { + fatal( + `analytics leg "${analytics.key}" has ${candidates.length} possible standard peers for ` + + `${analytics.version}; use an unambiguous label/grammar identity` + ); + } + + if (standard.version !== analytics.version) { + fatal( + `paired standard/analytics legs report different engine versions: ` + + `${standard.label}=${JSON.stringify(standard.version)}, ` + + `${analytics.label}=${JSON.stringify(analytics.version)}` + ); + } + if (!standard.sqlSha || !analytics.sqlSha) { + fatal( + `paired standard/analytics legs must both report a non-empty SQL SHA: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (standard.sqlSha !== analytics.sqlSha) { + fatal( + `paired standard/analytics legs report different SQL SHAs: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (!standard.grammarHash || !analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} must both report a runtime grammar hash` + ); + } + if (standard.grammarHash !== analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} have different grammar hashes: ` + + `${standard.label}=${JSON.stringify(standard.grammarHash)}, ` + + `${analytics.label}=${JSON.stringify(analytics.grammarHash)}` + ); + } + if (usedStandards.has(standard.key)) { + fatal( + `standard leg "${standard.key}" matches more than one analytics leg; duplicate backend leg identity` + ); + } + usedStandards.add(standard.key); + const pair = { + key: `${standard.key}::${analytics.key}`, + standard, + analytics, + engineVersion: analytics.version, + grammarHash: analytics.grammarHash, + }; + assertDetectorParity(pair); + pairs.push(pair); + } + return pairs; +} + +function detectorParityValue(entry) { + return { + role: entry.role || 'trigger', + query: entry.query || '', + expected: entry.expected, + actual: entry.actual, + severities: [...(entry.severities || [])].sort(), + severityMatched: + typeof entry.severityMatched === 'boolean' ? entry.severityMatched : undefined, + messageMatched: + typeof entry.messageMatched === 'boolean' ? entry.messageMatched : undefined, + }; +} + +/** + * Both detector passes use the same OSD checkout, grammar, contracts, and lint + * context. Any route-qualified difference is therefore a harness defect, not a + * backend observation. + */ +function assertDetectorParity(pair) { + const standard = pair.standard.detector.resultsByKey; + const analytics = pair.analytics.detector.resultsByKey; + const keys = new Set([...standard.keys(), ...analytics.keys()]); + for (const key of keys) { + const standardRow = standard.get(key); + const analyticsRow = analytics.get(key); + if (!standardRow || !analyticsRow) { + fatal( + `detector parity failed for ${key}: standard row=${!!standardRow}, ` + + `analytics row=${!!analyticsRow}` + ); + } + const standardValue = detectorParityValue(standardRow); + const analyticsValue = detectorParityValue(analyticsRow); + if (JSON.stringify(standardValue) !== JSON.stringify(analyticsValue)) { + fatal( + `detector parity failed for ${key}: standard=${JSON.stringify(standardValue)}, ` + + `analytics=${JSON.stringify(analyticsValue)}` + ); + } + } +} + +function backendVerdict(entry) { + if (!entry) { + return { + backendRejected: undefined, + backendType: undefined, + backendReason: undefined, + }; + } + const state = classifyBackendReportRow(entry); + const observedBackend = entry && entry.observed; + const rowRejected = + typeof entry.rejected === 'boolean' ? entry.rejected : undefined; + const observedRejected = + observedBackend && typeof observedBackend.rejected === 'boolean' + ? observedBackend.rejected + : undefined; + if ( + typeof rowRejected === 'boolean' && + typeof observedRejected === 'boolean' && + rowRejected !== observedRejected + ) { + fatal( + `backend report row ${reportRowKey(entry, 'backend report')} has conflicting ` + + `rejected verdicts` + ); + } + const explicitRejected = + typeof observedRejected === 'boolean' ? observedRejected : rowRejected; + const usableRawObservation = + state.status === 'observed' || state.status === 'coverage-missing'; + return { + backendRejected: + usableRawObservation && typeof explicitRejected === 'boolean' + ? explicitRejected + : undefined, + backendStatus: observedBackend ? observedBackend.httpStatus : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + backendOutcome: entry.outcome, + backendMismatch: entry.error, + }; +} + +function indexDivergentCases(pairs) { + const cases = new Map(); + for (const pair of pairs) { + for (const [rowKey, standardEntry] of pair.standard.backend) { + const analyticsEntry = pair.analytics.backend.get(rowKey); + if (!analyticsEntry) continue; + const standardObserved = backendVerdict(standardEntry); + const analyticsObserved = backendVerdict(analyticsEntry); + if ( + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + continue; + } + const value = { pair, rowKey, standardObserved, analyticsObserved }; + cases.set(`${pair.standard.key}::${rowKey}`, value); + cases.set(`${pair.analytics.key}::${rowKey}`, value); + } + } + return cases; } /** @@ -216,23 +679,22 @@ function auditDefaultErrorCensus(legs, specs, enforcedRules) { * comparable engine verdict, so the caller can refuse to call it agreement. */ function readBackendObservation(backendEntry, detectorResult) { - const observedBackend = (backendEntry && backendEntry.observed) || undefined; - const outcome = backendEntry && backendEntry.outcome; - // `observed`/`error` are the observe-only outcomes; `pass`/`fail` come from the - // asserting mode. Only those carry a real verdict. - const hasVerdict = - !!backendEntry && - outcome !== 'error' && - (typeof backendEntry.rejected === 'boolean' || !!observedBackend); + const verdict = backendVerdict(backendEntry); + const hasVerdict = typeof verdict.backendRejected === 'boolean'; return { usable: hasVerdict && !!detectorResult, observed: { detectorCount: detectorResult ? detectorResult.actual : 0, severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: hasVerdict ? !!backendEntry.rejected : undefined, - backendType: observedBackend ? observedBackend.type : undefined, - backendReason: observedBackend ? observedBackend.reason : undefined, + backendRejected: verdict.backendRejected, + backendStatus: verdict.backendStatus, + backendType: verdict.backendType, + backendReason: verdict.backendReason, + backendOutcome: verdict.backendOutcome, + backendMismatch: verdict.backendMismatch, + severityMatched: detectorResult ? detectorResult.severityMatched : undefined, + messageMatched: detectorResult ? detectorResult.messageMatched : undefined, }, }; } @@ -247,28 +709,31 @@ function readBackendObservation(backendEntry, detectorResult) { * expectation to read on this path), and the backend observation from this leg's * report; `classifyDrift` decides, so the "too narrow" wording stays in one place. */ -function classifyOutOfScope({ spec, ruleId, leg, classify }) { +function classifyOutOfScope({ spec, ruleId, leg, classify, divergentCases }) { const found = []; + const unusable = []; + const observations = new Map(); + + for (const [queryName] of Object.entries(spec.queries || {})) { + const rowKey = `${ruleId}::${queryName}`; + const backendEntry = leg.backend.get(rowKey); + const detectorResult = leg.detector.resultsByKey.get(rowKey); + const { observed, usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + continue; + } + observations.set(queryName, observed); + } // What did this rule's CONTROL queries — valid uses of the same command — do on // this engine? THREE states, not two, and the difference decides whether a - // rejected trigger means anything: - // rejected the command itself is unsupported here, so the trigger's rejection - // says nothing about the rule's specific condition -> suppress - // accepted the command works, so a rejected trigger really is the rule's - // condition going unreported on this version -> report it - // unknown no control verdict arrived (errored/absent). We cannot tell the two - // apart, so we must not emit confident advice either way. - // Collapsing this to a boolean is what let the suppression fail open: an errored - // control read as "not rejected" and produced the exact "widen appliesTo" advice - // this check exists to prevent. + // rejected trigger means anything. const controlVerdicts = Object.entries(spec.queries || {}) .filter(([, def]) => (def.role || 'trigger') === 'control') - .map(([name]) => { - const entry = leg.backend.get(`${ruleId}::${name}`); - const { observed } = readBackendObservation(entry, { actual: 0, severities: [] }); - return observed.backendRejected; - }); + .map(([name]) => observations.get(name)?.backendRejected); const controlAlsoRejected = controlVerdicts.some((v) => v === true); // A rule with controls, none of which produced a verdict, cannot be judged here. const controlUnknown = @@ -276,16 +741,11 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; - const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); - if (!backendEntry) continue; // this leg never ran the query - const detectorResult = (leg.detector.results || []).find( - (r) => r.ruleId === ruleId && r.queryName === queryName - ); - // Same reason as above: an errored observation must not read as "the engine - // accepted this". On this path that coercion would turn a genuinely - // mis-scoped rule into a silent `out-of-scope` PASS, because the - // version-scope-too-narrow check requires backendRejected === true. - const { observed: outOfScopeObserved } = readBackendObservation(backendEntry, detectorResult); + const rowKey = `${ruleId}::${queryName}`; + const outOfScopeObserved = observations.get(queryName); + if (!outOfScopeObserved) continue; + const pairedDivergence = divergentCases.has(`${leg.key}::${rowKey}`); + if (pairedDivergence && leg.executionBackend === 'analytics') continue; const drift = classify({ ruleId, version: leg.version, @@ -297,6 +757,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { observed: outOfScopeObserved, wiring: spec.wiring, detectorPath: spec.detectorPath, + executionBackend: leg.executionBackend, // An unknown control verdict is treated the same as a rejected one: both // mean "we cannot claim this engine supports the command", and staying quiet // is the only honest option. @@ -306,7 +767,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { }); if (drift) found.push(drift); } - return found; + return { drifts: found, unusable }; } /** @@ -376,11 +837,14 @@ function main() { const versionMatchesRange = makeRangeMatcher(); const { specs, enforcedRules, manifest } = loadContracts(args.contracts); const legs = args.legs.map(loadLeg); + const backendPairs = pairBackendLegs(legs); + const divergentCases = indexDivergentCases(backendPairs); log(`contracts=${specs.size} enforced(default-error)=${enforcedRules.size} legs=${legs.length}`); for (const leg of legs) { log( - ` leg ${leg.label}: engine=${leg.version} grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + + ` leg ${leg.label} (${leg.executionBackend}): engine=${leg.version} ` + + `grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + `detectorResults=${(leg.detector.results || []).length} backendCases=${leg.backend.size}` ); } @@ -395,7 +859,18 @@ function main() { // compiled-simplified leg). Recorded so the report can say WHY a cell is blank, // but never a failure: the rule is inert there by design. const notApplicable = []; - const matrix = []; // one row per rule × version, for the summary table + const matrix = []; // one row per rule × backend-qualified leg, for the summary table + const addDrift = (drift, leg, extra = {}) => { + const enriched = { + ...drift, + ...legFields(leg), + ...extra, + executionBackend: drift.executionBackend || leg.executionBackend, + }; + enriched.key = findingKey(enriched); + drifts.push(enriched); + return enriched; + }; // A rule that ships enabled at error severity but has no contract file is // invisible to this whole check. Compare the manifest's declared set against @@ -425,15 +900,13 @@ function main() { if (contractSurface !== 'both' && contractSurface !== legSurface) { notApplicable.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), surface: legSurface, reason: `contract declares grammarSurface "${contractSurface}"`, }); matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), status: 'not-applicable', drifts: 0, }); @@ -453,10 +926,11 @@ function main() { requiredParserRules: spec.requiredParserRules, detectorPath: spec.detectorPath, parserRuleNames: leg.parserRuleNames, + executionBackend: leg.executionBackend, }); if (grammarDrift) { - drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'drift', drifts: 1 }); + addDrift(grammarDrift, leg, { enforced: isEnforced, contractFile: file }); + matrix.push({ ruleId, ...legFields(leg), status: 'drift', drifts: 1 }); continue; } } @@ -467,27 +941,47 @@ function main() { // Deliberately out of scope on this engine. Still run the classifier // for the one case that matters — an engine that rejects a trigger the // rule has been scoped away from (a missed diagnostic). - const outOfScopeDrifts = classifyOutOfScope({ + const outOfScope = classifyOutOfScope({ spec, ruleId, leg, classify: classifyDrift, + divergentCases, }); - for (const drift of outOfScopeDrifts) { - drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + for (const drift of outOfScope.drifts) { + addDrift(drift, leg, { enforced: isEnforced, contractFile: file }); + } + if (outOfScope.unusable.length > 0) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: outOfScope.unusable, + }); } matrix.push({ ruleId, - version: leg.version, - leg: leg.label, - status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', - drifts: outOfScopeDrifts.length, + ...legFields(leg), + status: + outOfScope.unusable.length > 0 + ? 'inconclusive' + : outOfScope.drifts.length > 0 + ? 'drift' + : 'out-of-scope', + drifts: outOfScope.drifts.length, }); continue; } // In scope on this engine but nothing pins its behavior there. - coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'uncovered', drifts: 0 }); + coverageHoles.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reason: 'no version expectation matches this engine', + }); + matrix.push({ ruleId, ...legFields(leg), status: 'uncovered', drifts: 0 }); continue; } @@ -505,6 +999,7 @@ function main() { // because the two need opposite advice: not-applicable is expected and needs // no action, unusable means something did not answer and needs a re-run. let ruleNotApplicable = 0; + let ruleCoverageHoles = 0; const unusable = []; // Per-trigger engine verdicts for this rule on this leg, so a relaxation can // be judged across the WHOLE rule rather than one query at a time. A single @@ -534,9 +1029,99 @@ function main() { triggersExpected++; } - const detectorResult = (leg.detector.results || []).find( - (r) => r.ruleId === ruleId && r.queryName === queryName - ); + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, leg.executionBackend); + } catch (error) { + artifactFatal(`${file} query "${queryName}"`, error); + } + const rowKey = `${ruleId}::${queryName}`; + const detectorResult = leg.detector.resultsByKey.get(rowKey); + const backendEntry = leg.backend.get(rowKey); + if ( + detectorResult && + !detectorResult.notApplicable && + detectorResult.outcome !== 'not-applicable' + ) { + if (detectorResult.expected !== oracleSelection.detector.count) { + fatal( + `detector report row ${rowKey} expected=${JSON.stringify(detectorResult.expected)} ` + + `does not match contract detectorCount=${oracleSelection.detector.count}` + ); + } + if ((detectorResult.role || 'trigger') !== role) { + fatal( + `detector report row ${rowKey} role=${JSON.stringify(detectorResult.role)} ` + + `does not match contract role=${JSON.stringify(role)}` + ); + } + } + + if (oracleSelection.status === 'coverage-missing') { + const { usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: isEnforced, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleCoverageHoles++; + continue; + } + if (oracleSelection.status === 'not-applicable') { + const backendState = backendEntry + ? classifyBackendReportRow(backendEntry) + : { status: 'error' }; + if (!detectorResult || backendState.status !== 'not-applicable') { + unusable.push( + `${queryName} (${ + !detectorResult + ? 'no detector result' + : 'backend did not report not-applicable' + })` + ); + continue; + } + notApplicable.push({ + ruleId, + queryName, + ...legFields(leg), + surface: leg.surface, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleNotApplicable++; + if (isEnforced) { + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: true, + reason: + `default-error rule is not applicable on ${leg.executionBackend}: ` + + `${oracleSelection.reason}`, + kind: 'backend-oracle', + issue: oracleSelection.oracle.issue, + owner: oracleSelection.oracle.owner, + }); + ruleCoverageHoles++; + } + continue; + } + // A case the surface cannot express at all (a `runtimeOnly` rule on a // compiled-simplified leg) is excluded rather than compared. Its zero // diagnostics are `lint_runner` deliberately skipping the rule, so @@ -546,15 +1131,14 @@ function main() { if (detectorResult && detectorResult.notApplicable) { notApplicable.push({ ruleId, - version: leg.version, queryName, + ...legFields(leg), surface: leg.surface, reason: detectorResult.notApplicable, }); ruleNotApplicable++; continue; } - const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { // No comparable pair, so there is nothing to classify. Attempting it @@ -571,6 +1155,7 @@ function main() { continue; } compared++; + const pairedDivergence = divergentCases.get(`${leg.key}::${rowKey}`); if (role === 'trigger') { triggersCompared++; // Bucket this trigger by what the ENGINE did, but only where the contract @@ -578,8 +1163,11 @@ function main() { // head-without-sort, whose queries are all valid PPL) never "relaxes", and // counting it as relaxed would fabricate a full-fix verdict for a rule the // engine was never rejecting in the first place. - const pinnedRejection = (expected.backend && expected.backend.kind) === 'rejection'; - if (pinnedRejection) { + const pinnedRejection = oracleSelection.oracle.kind === 'rejection'; + if ( + pinnedRejection && + (!pairedDivergence || leg.executionBackend === 'standard') + ) { if (observed.backendRejected === false) { relaxedTriggers.push(queryName); if ((observed.detectorCount || 0) > 0) relaxedDetectorFlagged = true; @@ -589,24 +1177,29 @@ function main() { } } - const drift = classifyDrift({ - ruleId, - version: leg.version, - queryName, - role, - query, - expected: { - detectorCount: expected.detectorCount, - severity: expected.severity, - backendKind: expected.backend && expected.backend.kind, - }, - observed, - wiring: spec.wiring, - detectorPath: spec.detectorPath, - parserRuleNames: leg.parserRuleNames, - requiredParserRules: spec.requiredParserRules, - expectedBackend: expected.backend, - }); + const drift = + pairedDivergence && leg.executionBackend === 'analytics' + ? null + : classifyDrift({ + ruleId, + version: leg.version, + queryName, + role, + query, + expected: { + detectorCount: oracleSelection.detector.count, + severity: oracleSelection.detector.severity, + matchMessage: oracleSelection.detector.matchMessage, + backendKind: oracleSelection.oracle.kind, + }, + observed, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + requiredParserRules: spec.requiredParserRules, + expectedBackend: oracleSelection.oracle, + executionBackend: leg.executionBackend, + }); if (drift) { // `expectationRange` is what the annotation anchors to: the version @@ -630,26 +1223,29 @@ function main() { // rule as a whole. This supersedes the per-query `engine-relaxed` findings — // they each said "scope this rule away from this version", which is the wrong // action whenever another trigger still rejects. - const relaxationScope = classifyRelaxationScope({ - ruleId, - version: leg.version, - relaxedTriggers, - holdingTriggers, - unobservedTriggers, - detectorFlagged: relaxedDetectorFlagged, - wiring: spec.wiring, - detectorPath: spec.detectorPath, - }); + const relaxationScope = + leg.executionBackend === 'standard' + ? classifyRelaxationScope({ + ruleId, + version: leg.version, + relaxedTriggers, + holdingTriggers, + unobservedTriggers, + detectorFlagged: relaxedDetectorFlagged, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + executionBackend: leg.executionBackend, + }) + : null; const kept = relaxationScope ? perQueryDrifts.filter((d) => d.supersededBy !== DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED) : perQueryDrifts; for (const drift of kept) { - drifts.push(drift); + addDrift(drift, leg); ruleDrifts++; } if (relaxationScope) { - drifts.push({ - ...relaxationScope, + addDrift(relaxationScope, leg, { enforced: isEnforced, contractFile: file, expectationRange: expectation.version, @@ -667,11 +1263,31 @@ function main() { // nothing failed and there is nothing to re-run, so it must not fail the run. // Checked BEFORE the inconclusive test, which would otherwise catch it // (compared === 0) and demand a re-run that could never change the outcome. - if (compared === 0 && ruleNotApplicable > 0) { + if (unusable.length > 0) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: unusable, + }); matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); + } else if (ruleCoverageHoles > 0) { + matrix.push({ + ruleId, + ...legFields(leg), + status: ruleDrifts > 0 ? 'drift' : 'uncovered', + drifts: ruleDrifts, + }); + } else if (compared === 0 && ruleNotApplicable > 0) { + matrix.push({ + ruleId, + ...legFields(leg), status: 'not-applicable', drifts: 0, }); @@ -679,40 +1295,125 @@ function main() { inconclusive.push({ ruleId, file, - version: leg.version, + ...legFields(leg), enforced: isEnforced, reasons: unusable, }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'inconclusive', drifts: ruleDrifts }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); } else { - if (unusable.length > 0) { - log( - `WARN: ${ruleId} @ ${leg.version} compared ${compared} case(s); ` + - `${unusable.length} not compared: ${unusable.join(', ')}` - ); - } matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), status: ruleDrifts === 0 ? 'agree' : 'drift', drifts: ruleDrifts, }); } } + + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface === 'runtime-bundle' || contractSurface === 'both') { + for (const pair of backendPairs) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const rowKey = `${ruleId}::${queryName}`; + const divergent = divergentCases.get(`${pair.analytics.key}::${rowKey}`); + if (!divergent || divergent.pair.key !== pair.key) continue; + + const query = queryDef.query.split('{{index}}').join(spec.index); + const drift = classifyExecutionBackendDivergence({ + ruleId, + version: pair.engineVersion, + queryName, + role: queryDef.role || 'trigger', + query, + standardObserved: divergent.standardObserved, + analyticsObserved: divergent.analyticsObserved, + standardLeg: pair.standard.label, + analyticsLeg: pair.analytics.label, + grammarHash: pair.grammarHash, + detectorPath: spec.detectorPath, + }); + if (!drift) continue; + + const expectation = selectExpectation(spec, pair.engineVersion, versionMatchesRange); + addDrift(drift, pair.analytics, { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation && expectation.version, + expectationEngine: expectation && expectation.engine, + pairKey: pair.key, + standardLeg: pair.standard.label, + standardLegKey: pair.standard.key, + analyticsLeg: pair.analytics.label, + analyticsLegKey: pair.analytics.key, + }); + + for (const row of matrix) { + if ( + row.ruleId === ruleId && + (row.legKey === pair.standard.key || row.legKey === pair.analytics.key) + ) { + row.status = 'drift'; + row.drifts += 1; + } + } + } + } + } } - const enforcedDrifts = drifts.filter((d) => d.enforced); - const enforcedHoles = coverageHoles.filter((h) => h.enforced); + const isObservedAnalyticsFinding = (entry) => + args.observeAnalytics && + (entry.executionBackend === 'analytics' || + (Array.isArray(entry.executionBackends) && + entry.executionBackends.includes('analytics'))) && + (entry.driftClass === DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE || + entry.driftClass === DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH || + entry.kind === 'backend-oracle'); + for (const drift of drifts) { + drift.blocking = !!drift.enforced && !isObservedAnalyticsFinding(drift); + } + for (const hole of coverageHoles) { + hole.blocking = !!hole.enforced && !isObservedAnalyticsFinding(hole); + } + const enforcedDrifts = drifts.filter((d) => d.blocking); + const enforcedHoles = coverageHoles.filter((h) => h.blocking); const enforcedInconclusive = inconclusive.filter((i) => i.enforced); + for (const row of matrix) { + row.key = reportItemKey(row, 'matrix'); + } + for (const hole of coverageHoles) { + hole.key = reportItemKey(hole, 'coverage-hole'); + } + for (const entry of inconclusive) { + entry.key = reportItemKey(entry, 'inconclusive'); + } + for (const entry of notApplicable) { + entry.key = reportItemKey(entry, 'not-applicable'); + } const report = { - schemaVersion: 1, + schemaVersion: 2, + keyDimensions: ['leg', 'engineVersion', 'grammarSurface', 'executionBackend'], legs: legs.map((l) => ({ + key: l.key, label: l.label, engineVersion: l.version, grammarHash: l.grammarHash, + sqlSha: l.sqlSha, surface: l.surface, + executionBackend: l.executionBackend, + })), + backendPairs: backendPairs.map((pair) => ({ + key: pair.key, + engineVersion: pair.engineVersion, + grammarHash: pair.grammarHash, + standardLegKey: pair.standard.key, + analyticsLegKey: pair.analytics.key, })), enforcedRules: [...enforcedRules].sort(), missingContracts, @@ -726,6 +1427,9 @@ function main() { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, enforcedCoverageHoles: enforcedHoles.length, + observedAnalyticsFindings: + drifts.filter((d) => d.enforced && !d.blocking).length + + coverageHoles.filter((h) => h.enforced && !h.blocking).length, missingContractCount: missingContracts.length, enforcedInconclusive: enforcedInconclusive.length, // An inconclusive default-error rule fails too: "we could not check" must @@ -792,6 +1496,9 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { `${report.result.enforcedDriftCount} enforced drift(s)`, `${report.result.enforcedCoverageHoles} coverage hole(s)`, ]; + if (report.result.observedAnalyticsFindings) { + reasons.push(`${report.result.observedAnalyticsFindings} analytics observation(s)`); + } if (report.result.enforcedInconclusive) { reasons.push(`${report.result.enforcedInconclusive} inconclusive`); } @@ -803,23 +1510,28 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { // reader knows a column speaks for OSD's compiled grammar rather than the // engine's exported one — the two do not run the same set of rules. `Engine versions: ${legs - .map((l) => - l.surface && l.surface !== 'runtime-bundle' ? `\`${l.version}\` (${l.surface})` : `\`${l.version}\`` - ) + .map((l) => { + const identity = + l.label === l.version ? `\`${l.version}\`` : `\`${l.label}\` → \`${l.version}\``; + return l.surface && l.surface !== 'runtime-bundle' + ? `${identity} (${l.executionBackend}, ${l.surface})` + : `${identity} (${l.executionBackend})`; + }) .join(', ')} — ` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); - // Columns are keyed on the LEG LABEL, not the engine version: two legs can share - // a version while validating different surfaces (a 3.7 runtime-bundle leg and a - // 3.7 compiled leg), and keying on version alone made them collide so one leg's - // results silently rendered in place of the other's. + // Columns use the full leg key, including execution backend and grammar surface. + // A label or engine version alone is not unique once the same candidate runs + // through both standard and analytics. const columns = legs.map((l) => ({ - label: l.label, + key: l.key, heading: l.surface && l.surface !== 'runtime-bundle' - ? `\`${l.version}\`
      ${l.surface}` - : `\`${l.version}\``, + ? `\`${l.version}\`
      ${l.executionBackend}
      ${l.surface}` + + (l.label === l.version ? '' : `
      ${l.label}`) + : `\`${l.version}\`
      ${l.executionBackend}` + + (l.label === l.version ? '' : `
      ${l.label}`), })); const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); lines.push(`| Rule | ${columns.map((c) => c.heading).join(' | ')} |`); @@ -834,7 +1546,7 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { }; for (const ruleId of rules) { const cells = columns.map((column) => { - const row = report.matrix.find((m) => m.ruleId === ruleId && m.leg === column.label); + const row = report.matrix.find((m) => m.ruleId === ruleId && m.legKey === column.key); if (!row) return '—'; if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; // An unmapped status must still render as something visible. A blank cell @@ -851,7 +1563,8 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { lines.push(''); for (const entry of report.inconclusive) { lines.push( - `- \`${entry.ruleId}\` on engine \`${entry.version}\`: no case could be compared — ` + + `- \`${entry.ruleId}\` on engine \`${entry.version}\` (${entry.executionBackend}): ` + + `no case could be compared — ` + `${entry.reasons.join('; ')}. This is NOT a lint finding: the engine or the detector run ` + `did not answer, so nothing was validated. Check that leg's job logs (an unreachable ` + `cluster, an index that failed to seed, or a detector runner that died mid-corpus) and ` + @@ -880,11 +1593,18 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { lines.push('### Coverage holes'); lines.push(''); for (const hole of coverageHoles) { + const query = hole.queryName ? ` query \`${hole.queryName}\`` : ''; + const fix = + hole.kind === 'backend-oracle' + ? `add a reviewed \`${hole.executionBackend}\` backend oracle for this query` + : `add an \`expectations[]\` entry whose \`version\` range covers \`${hole.version}\`, ` + + `or narrow the rule's \`appliesTo\` so it does not apply there`; lines.push( - `- \`${hole.ruleId}\` has no expectation matching engine \`${hole.version}\`` + + `- \`${hole.ruleId}\`${query} has no ${hole.executionBackend} coverage for engine ` + + `\`${hole.version}\`` + `${hole.enforced ? ' (ENFORCED — this rule ships to users on that engine unpinned)' : ''}. ` + - `FIX (${hole.file}): add an \`expectations[]\` entry whose \`version\` range covers ` + - `\`${hole.version}\`, or narrow the rule's \`appliesTo\` so it does not apply there.` + `${hole.reason ? `${hole.reason}. ` : ''}` + + `FIX (${hole.file}): ${fix}.` ); } lines.push(''); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs index 06b1a384db7..422177ce204 100644 --- a/scripts/ppl-lint/annotate.mjs +++ b/scripts/ppl-lint/annotate.mjs @@ -47,6 +47,12 @@ function escapeData(value) { return String(value).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); } +function backendLabel(entry) { + return Array.isArray(entry.executionBackends) && entry.executionBackends.length > 1 + ? entry.executionBackends.join(' vs ') + : entry.executionBackend || 'standard'; +} + /** * Line of the `expectations[]` entry whose `version` is `range`, 1-indexed. * @@ -124,10 +130,12 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r findRuleIdLine(text); annotations.push({ - level: drift.enforced ? 'error' : 'warning', + level: (drift.blocking ?? drift.enforced) ? 'error' : 'warning', file: file ? contractRepoPath(contractsDir, file, workspace) : undefined, line, - title: `PPL lint drift: ${drift.driftClass} (${drift.ruleId} @ ${drift.version})`, + title: + `PPL lint drift: ${drift.driftClass} ` + + `(${drift.ruleId} @ ${drift.version}, ${backendLabel(drift)})`, // Message order matters: the UI truncates, so lead with what moved, then the // action, then where. The summary carries the full rationale. message: [ @@ -143,15 +151,17 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r for (const hole of report.coverageHoles || []) { const text = contractText(hole.file); annotations.push({ - level: hole.enforced ? 'error' : 'warning', + level: (hole.blocking ?? hole.enforced) ? 'error' : 'warning', file: hole.file ? contractRepoPath(contractsDir, hole.file, workspace) : undefined, line: findRuleIdLine(text), - title: `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}`, + title: + `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}, ${backendLabel(hole)}`, message: - `No expectation in this contract matches engine ${hole.version}, so nothing pins ` + - `"${hole.ruleId}" there. Add a reviewed expectation whose version range covers ` + - `${hole.version}; do not widen an existing range to absorb it unless the behavior is ` + - `genuinely identical.`, + (hole.reason + ? `${hole.reason}. ` + : `No expectation in this contract matches engine ${hole.version}. `) + + `Nothing pins "${hole.ruleId}" for the ${backendLabel(hole)} route there. Add a reviewed ` + + `${backendLabel(hole)} oracle or expectation; never use another route's oracle as fallback.`, }); } @@ -163,9 +173,12 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r level: 'warning', file: entry.file ? contractRepoPath(contractsDir, entry.file, workspace) : undefined, line: findRuleIdLine(text), - title: `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version} (leg problem)`, + title: + `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version}, ` + + `${backendLabel(entry)} (leg problem)`, message: - `No case could be compared for "${entry.ruleId}" on engine ${entry.version}` + + `No case could be compared for "${entry.ruleId}" on engine ${entry.version} ` + + `(${backendLabel(entry)})` + (entry.reasons && entry.reasons.length > 0 ? ` — ${entry.reasons.join('; ')}` : '') + `. This is NOT a lint finding: the engine or the detector run did not answer, so ` + `nothing was validated. Check that leg's job logs and re-run. Do not edit the rule or ` + diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index 840e631ef88..faf21f078b1 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -7,7 +7,7 @@ * Assemble the PPL lint validation run manifest and the compact per-rule PR * summary in the result job (design §3.3, §4.4, T10). * - * Inputs (env, all optional so a partial run still produces a manifest): + * Inputs (env; identity fields may be empty on a partial run): * SQL_SHA, OSD_REF, OSD_SHA, EVENT_NAME, SCHEDULE, * BACKEND_RESULT, DETECTOR_RESULT, GITHUB_STEP_SUMMARY. * Artifact files under ./artifacts (downloaded from both jobs): @@ -22,24 +22,143 @@ import fs from 'fs'; import path from 'path'; +import { + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, +} from './contract-schema.mjs'; + const ARTIFACTS = 'artifacts'; -function readJson(file) { +function readJson(file, errors) { try { if (fs.existsSync(file)) { return JSON.parse(fs.readFileSync(file, 'utf8')); } + errors.push(`required artifact is missing: ${file}`); } catch (error) { // eslint-disable-next-line no-console console.error(`[ppl-lint-manifest] could not parse ${file}: ${error.message}`); + errors.push(`required artifact is malformed: ${file}: ${error.message}`); } return undefined; } function main() { - const target = readJson(path.join(ARTIFACTS, 'target.json')) || {}; - const detector = readJson(path.join(ARTIFACTS, 'detector-report.json')) || {}; - const backend = readJson(path.join(ARTIFACTS, 'backend-report.json')) || []; + const artifactErrors = []; + const targetRaw = readJson(path.join(ARTIFACTS, 'target.json'), artifactErrors) || {}; + const detector = + readJson(path.join(ARTIFACTS, 'detector-report.json'), artifactErrors) || {}; + const backend = + readJson(path.join(ARTIFACTS, 'backend-report.json'), artifactErrors) || []; + + let target = {}; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactErrors.push(`invalid target.json: ${error.message}`); + } + const executionBackend = target.executionBackend || ''; + if (target.schemaVersion !== 2 || target.legacy) { + artifactErrors.push( + `required workflow target schemaVersion must be 2, got ${JSON.stringify(target.schemaVersion)}` + ); + } + if (executionBackend !== 'standard') { + artifactErrors.push( + `required workflow target executionBackend must be "standard", got ${JSON.stringify(executionBackend)}` + ); + } + if (!target.sqlSha) { + artifactErrors.push('target sqlSha must be non-empty'); + } else if (process.env.SQL_SHA && target.sqlSha !== process.env.SQL_SHA) { + artifactErrors.push( + `target sqlSha ${JSON.stringify(target.sqlSha)} does not match workflow SQL_SHA ${JSON.stringify(process.env.SQL_SHA)}` + ); + } + if (detector.schemaVersion !== 2) { + artifactErrors.push( + `detector schemaVersion must be 2, got ${JSON.stringify(detector.schemaVersion)}` + ); + } + if (detector.executionBackend !== executionBackend) { + artifactErrors.push( + `detector executionBackend ${JSON.stringify(detector.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + if (detector[field] !== target[field]) { + artifactErrors.push( + `detector ${field} ${JSON.stringify(detector[field])} does not match target ${JSON.stringify(target[field])}` + ); + } + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + artifactErrors.push( + `detector surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + artifactErrors.push('detector defaultErrorRules must be an array'); + } + + let backendByKey = new Map(); + try { + backendByKey = indexBackendReport(backend, target); + } catch (error) { + artifactErrors.push(`invalid backend-report.json: ${error.message}`); + } + if (backendByKey.size === 0) { + artifactErrors.push('backend-report.json must be a non-empty array'); + } + if (!Array.isArray(detector.results) || detector.results.length === 0) { + artifactErrors.push('detector-report.json must contain a non-empty results array'); + } + const detectorKeys = new Set(); + for (const entry of Array.isArray(detector.results) ? detector.results : []) { + const key = `${entry.ruleId}::${entry.queryName}`; + if (!entry.ruleId || !entry.queryName) { + artifactErrors.push(`detector-report.json contains an invalid row ${JSON.stringify(entry)}`); + continue; + } + if (detectorKeys.has(key)) { + artifactErrors.push(`detector-report.json contains duplicate row ${key}`); + } + detectorKeys.add(key); + if (entry.executionBackend !== executionBackend) { + artifactErrors.push( + `detector row ${key} executionBackend ${JSON.stringify(entry.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + if (!backendByKey.has(key)) { + artifactErrors.push(`detector row ${key} has no matching backend row`); + } + if (!Number.isInteger(entry.expected) || !Number.isInteger(entry.actual)) { + artifactErrors.push(`detector row ${key} must contain integer expected/actual counts`); + } else if (entry.actual !== entry.expected) { + artifactErrors.push( + `detector row ${key} count mismatch: expected ${entry.expected}, got ${entry.actual}` + ); + } + if (entry.severityMatched !== true) { + artifactErrors.push(`detector row ${key} did not match its severity assertion`); + } + if (entry.messageMatched !== true) { + artifactErrors.push(`detector row ${key} did not match its message assertion`); + } + } + for (const [key, entry] of backendByKey) { + if (!detectorKeys.has(key)) { + artifactErrors.push(`backend row ${key} has no matching detector row`); + } + const state = classifyBackendReportRow(entry); + if (state.status !== 'observed' || entry.outcome !== 'pass') { + artifactErrors.push( + `backend row ${key} did not pass its oracle (outcome=${JSON.stringify(entry.outcome)})` + ); + } + } const eventName = process.env.EVENT_NAME || ''; const osdRef = process.env.OSD_REF || 'main'; @@ -56,7 +175,8 @@ function main() { const backendResult = process.env.BACKEND_RESULT || 'unknown'; const detectorResult = process.env.DETECTOR_RESULT || 'unknown'; - const passed = backendResult === 'success' && detectorResult === 'success'; + const passed = + backendResult === 'success' && detectorResult === 'success' && artifactErrors.length === 0; // The selected validation set is the set of rules the detector run actually // evaluated (post schedule filtering). @@ -65,6 +185,7 @@ function main() { ).sort(); const manifest = { + schemaVersion: 2, mode, // A workflow_dispatch osd_ref run is pre-merge evidence, never a // branch-protection result (design §4.1.1, T11). @@ -77,11 +198,13 @@ function main() { osdSha: process.env.OSD_SHA || '', engineVersion: target.engineVersion || detector.engineVersion || '', grammarHash: target.grammarHash || detector.grammarHash || '', + executionBackend, differential: !!detector.differential, validationSet, result: { backend: backendResult, detector: detectorResult, + artifactErrors, passed, }, }; @@ -89,6 +212,10 @@ function main() { fs.writeFileSync('run-manifest.json', JSON.stringify(manifest, null, 2)); writeSummary(manifest, detector, backend); + + if (artifactErrors.length > 0) { + throw new Error(`invalid PPL lint artifacts:\n- ${artifactErrors.join('\n- ')}`); + } } /** Compact per-rule PR summary: Rule | Version | Grammar | Detector | Backend | Result. */ @@ -112,6 +239,7 @@ function writeSummary(manifest, detector, backend) { lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (${manifest.osdRepo} @ \`${manifest.osdRef}\`)`); lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); + lines.push(`- Execution backend: \`${manifest.executionBackend || '—'}\``); lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); lines.push( `- Result: backend **${manifest.result.backend}**, detector **${manifest.result.detector}** → ` + @@ -124,13 +252,19 @@ function writeSummary(manifest, detector, backend) { for (const r of detector.results || []) { const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); const detectorCell = `${r.actual}/${r.expected}${r.severities && r.severities.length ? ` (${r.severities.join(',')})` : ''}`; - const backendCell = be - ? be.rejected - ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` - : 'accepted' - : '—'; + const backendCell = !be + ? '—' + : typeof be.rejected !== 'boolean' + ? be.outcome || 'no verdict' + : be.rejected + ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` + : 'accepted'; const ok = - r.actual === r.expected && (!be || (r.role === 'trigger' ? be.rejected : !be.rejected)); + r.actual === r.expected && + r.severityMatched === true && + r.messageMatched === true && + !!be && + be.outcome === 'pass'; lines.push( `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ok ? 'Pass' : 'Fail'} |` diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs new file mode 100644 index 00000000000..5df505f07de --- /dev/null +++ b/scripts/ppl-lint/contract-schema.mjs @@ -0,0 +1,397 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +const EXECUTION_BACKENDS = new Set(['standard', 'analytics']); +const CONTRACT_SCHEMA_VERSIONS = new Set([3, 4]); +const APPLICABLE_BACKEND_KINDS = new Set(['rejection', 'result-shape', 'advisory']); + +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function describe(value) { + return typeof value === 'string' ? `"${value}"` : JSON.stringify(value); +} + +function requireObject(value, label) { + if (!isObject(value)) { + throw new TypeError(`${label} must be a JSON object.`); + } + return value; +} + +function requireNonEmptyString(value, label) { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`${label} must be a non-empty string.`); + } + return value; +} + +function requireNonNegativeInteger(value, label) { + if (!Number.isInteger(value) || value < 0) { + throw new TypeError(`${label} must be a non-negative integer.`); + } + return value; +} + +function assertOptionalString(value, label) { + if (value !== undefined && (typeof value !== 'string' || value.length === 0)) { + throw new TypeError(`${label} must be a non-empty string when present.`); + } +} + +function assertBackendOracle(oracle, ruleId, executionBackend) { + const label = `[${ruleId}] ${executionBackend} backend oracle`; + requireObject(oracle, label); + const kind = requireNonEmptyString(oracle.kind, `${label}.kind`); + + if (kind === 'not-applicable') { + const reason = requireNonEmptyString(oracle.reason, `${label}.reason`); + if (reason.trim().length === 0) { + throw new TypeError(`${label}.reason must not be blank.`); + } + requireNonEmptyString(oracle.owner, `${label}.owner`); + requireNonEmptyString(oracle.issue, `${label}.issue`); + return kind; + } + if (!APPLICABLE_BACKEND_KINDS.has(kind)) { + throw new Error(`[${ruleId}] unknown ${executionBackend} backend oracle.kind "${kind}".`); + } + + if (!Number.isInteger(oracle.httpStatus) || oracle.httpStatus < 100 || oracle.httpStatus > 599) { + throw new TypeError(`${label}.httpStatus must be an integer from 100 through 599.`); + } + if (kind === 'rejection') { + const body = requireObject(oracle.body, `${label}.body`); + if (!Number.isInteger(body.status)) { + throw new TypeError(`${label}.body.status must be an integer.`); + } + if (body.error !== undefined) { + const error = requireObject(body.error, `${label}.body.error`); + assertOptionalString(error.type, `${label}.body.error.type`); + assertOptionalString(error.reason, `${label}.body.error.reason`); + } + } + if (kind === 'result-shape' && oracle.expect !== undefined) { + const expect = requireObject(oracle.expect, `${label}.expect`); + if ( + expect.datarowsNonEmpty !== undefined && + typeof expect.datarowsNonEmpty !== 'boolean' + ) { + throw new TypeError(`${label}.expect.datarowsNonEmpty must be a boolean.`); + } + if (expect.datarowsCount !== undefined) { + requireNonNegativeInteger(expect.datarowsCount, `${label}.expect.datarowsCount`); + } + assertOptionalString(expect.columnAllNull, `${label}.expect.columnAllNull`); + } + return kind; +} + +export function assertExecutionBackend(value, label = 'executionBackend') { + if (!EXECUTION_BACKENDS.has(value)) { + throw new Error( + `${label} must be "standard" or "analytics", got ${describe(value)}.` + ); + } + return value; +} + +/** + * Validate target.json and return the identity consumed by report readers. + * + * Execution identity is never inferred. Every producer in the current + * workflows writes schemaVersion 2, so an unversioned target is an incomplete + * artifact rather than a compatibility mode. + */ +export function normalizeTarget(target) { + requireObject(target, 'target'); + + const hasSchemaVersion = Object.prototype.hasOwnProperty.call(target, 'schemaVersion'); + if (!hasSchemaVersion) { + throw new Error('target.schemaVersion is required; expected 2.'); + } + + if (target.schemaVersion !== 2) { + throw new Error( + `Unsupported target schemaVersion ${describe(target.schemaVersion)}; expected 2.` + ); + } + const executionBackend = assertExecutionBackend( + target.executionBackend, + 'target.executionBackend' + ); + requireNonEmptyString(target.engineVersion, 'target.engineVersion'); + if (typeof target.grammarHash !== 'string') { + throw new TypeError('target.grammarHash must be a string.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'grammarBundle') && + typeof target.grammarBundle !== 'string' + ) { + throw new TypeError('target.grammarBundle must be a string when present.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'sqlSha') && + typeof target.sqlSha !== 'string' + ) { + throw new TypeError('target.sqlSha must be a string when present.'); + } + if (!Number.isInteger(target.shardCount) || target.shardCount < 1) { + throw new Error('target.shardCount must be a positive integer.'); + } + if (executionBackend === 'analytics') { + if (target.storage !== 'composite-parquet') { + throw new Error( + `analytics target.storage must be "composite-parquet", got ${describe(target.storage)}.` + ); + } + const analyticsStack = requireObject( + target.analyticsStack, + 'analytics target.analyticsStack' + ); + requireNonEmptyString( + analyticsStack.source, + 'analytics target.analyticsStack.source' + ); + const attestation = requireObject( + target.routeAttestation, + 'analytics target.routeAttestation' + ); + for (const check of [ + 'pluginsVerified', + 'clusterSettingsVerified', + 'fixtureIndicesVerified', + 'explainVerified', + 'profiledExecutionVerified', + ]) { + if (attestation[check] !== true) { + throw new Error(`analytics target.routeAttestation.${check} must be true.`); + } + } + } else if (target.storage !== 'lucene') { + throw new Error( + `standard target.storage must be "lucene", got ${describe(target.storage)}.` + ); + } + + return { + schemaVersion: 2, + executionBackend, + engineVersion: target.engineVersion, + grammarHash: target.grammarHash, + grammarBundle: target.grammarBundle || '', + sqlSha: target.sqlSha || '', + storage: target.storage || (executionBackend === 'standard' ? 'lucene' : ''), + shardCount: target.shardCount, + analyticsStack: target.analyticsStack, + routeAttestation: target.routeAttestation, + legacy: false, + }; +} + +export function assertContractSchema(spec) { + requireObject(spec, 'contract'); + if (!CONTRACT_SCHEMA_VERSIONS.has(spec.schemaVersion)) { + throw new Error( + `Unsupported contract schemaVersion ${describe(spec.schemaVersion)}; expected 3 or 4.` + ); + } + requireNonEmptyString(spec.ruleId, 'contract.ruleId'); + return spec.schemaVersion; +} + +/** + * Require a selected expectation to cover every top-level query exactly once. + * JSON object keys are unique after parsing, so set equality establishes the + * one-to-one query identity needed by both backend and detector readers. + */ +export function assertExactQueryCoverage(spec, expectation) { + assertContractSchema(spec); + requireObject(spec.queries, `[${spec.ruleId}] contract.queries`); + requireObject(expectation, `[${spec.ruleId}] selected expectation`); + requireObject(expectation.queries, `[${spec.ruleId}] selected expectation.queries`); + + const contractKeys = Object.keys(spec.queries).sort(); + const expectationKeys = Object.keys(expectation.queries).sort(); + if (contractKeys.length === 0) { + throw new Error(`[${spec.ruleId}] contract.queries must not be empty.`); + } + const contractSet = new Set(contractKeys); + const expectationSet = new Set(expectationKeys); + const missing = contractKeys.filter((key) => !expectationSet.has(key)); + const extra = expectationKeys.filter((key) => !contractSet.has(key)); + + if (missing.length > 0 || extra.length > 0) { + const details = []; + if (missing.length > 0) { + details.push(`missing from expectation: ${missing.join(', ')}`); + } + if (extra.length > 0) { + details.push(`not present in contract.queries: ${extra.join(', ')}`); + } + throw new Error(`[${spec.ruleId}] query coverage must be exact (${details.join('; ')}).`); + } + return contractKeys; +} + +/** + * Resolve only the route-specific backend oracle. Detector count, severity, and + * message assertions remain on the shared query expectation and are returned + * unchanged for either execution backend. + */ +export function resolveBackendOracle(spec, queryExpectation, executionBackend) { + const schemaVersion = assertContractSchema(spec); + assertExecutionBackend(executionBackend); + requireObject(queryExpectation, `[${spec.ruleId}] query expectation`); + + requireNonNegativeInteger( + queryExpectation.detectorCount, + `[${spec.ruleId}] detectorCount` + ); + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'severity') && + (typeof queryExpectation.severity !== 'string' || + queryExpectation.severity.length === 0) + ) { + throw new TypeError( + `[${spec.ruleId}] severity must be a non-empty string when present.` + ); + } + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'matchMessage') && + typeof queryExpectation.matchMessage !== 'string' + ) { + throw new TypeError(`[${spec.ruleId}] matchMessage must be a string when present.`); + } + + const detector = { + count: queryExpectation.detectorCount, + severity: queryExpectation.severity, + matchMessage: queryExpectation.matchMessage, + }; + + let oracle; + let missingReason; + if (schemaVersion === 3) { + if (executionBackend === 'standard') { + oracle = queryExpectation.backend; + missingReason = 'schema-v3 query has no backend oracle'; + } else { + missingReason = + 'schema-v3 backend oracles are standard-only; no analytics oracle is defined'; + } + } else { + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'backends') && + !isObject(queryExpectation.backends) + ) { + throw new TypeError(`[${spec.ruleId}] backends must be a JSON object.`); + } + for (const backend of Object.keys(queryExpectation.backends || {})) { + assertExecutionBackend(backend, `[${spec.ruleId}] backends key`); + } + oracle = queryExpectation.backends && queryExpectation.backends[executionBackend]; + missingReason = `schema-v4 query has no ${executionBackend} backend oracle`; + } + + if (oracle === undefined) { + return { + status: 'coverage-missing', + executionBackend, + detector, + oracle: undefined, + reason: missingReason, + }; + } + + const kind = assertBackendOracle(oracle, spec.ruleId, executionBackend); + if (kind === 'not-applicable') { + return { + status: 'not-applicable', + executionBackend, + detector, + oracle, + reason: oracle.reason, + }; + } + + return { + status: 'applicable', + executionBackend, + detector, + oracle, + reason: undefined, + }; +} + +export function backendReportKey(entry) { + requireObject(entry, 'backend report row'); + const ruleId = requireNonEmptyString(entry.ruleId, 'backend report row.ruleId'); + const queryName = requireNonEmptyString(entry.queryName, 'backend report row.queryName'); + return `${ruleId}::${queryName}`; +} + +/** + * Read the backend observation state without coercing a missing verdict to + * acceptance. Explicit infrastructure/coverage states take precedence even if + * a malformed row also happens to contain `rejected`. + */ +export function classifyBackendReportRow(entry) { + requireObject(entry, 'backend report row'); + if (entry.outcome === 'not-applicable' || entry.kind === 'not-applicable') { + return { status: 'not-applicable', rejected: undefined }; + } + if (entry.outcome === 'coverage-missing' || entry.kind === 'coverage-missing') { + return { status: 'coverage-missing', rejected: undefined }; + } + if (entry.outcome === 'error' || typeof entry.rejected !== 'boolean') { + return { status: 'error', rejected: undefined }; + } + return { status: 'observed', rejected: entry.rejected }; +} + +/** + * Validate and index the historical bare-array backend report. + * + * Every row carries the same explicit backend identity as its schema-v2 target. + */ +export function indexBackendReport(entries, targetIdentity) { + if (!Array.isArray(entries)) { + throw new TypeError('backend report must be a JSON array.'); + } + requireObject(targetIdentity, 'normalized target identity'); + assertExecutionBackend( + targetIdentity.executionBackend, + 'normalized target identity.executionBackend' + ); + + const byKey = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + const key = backendReportKey(entry); + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity) { + throw new Error( + `backend report row ${key} is missing executionBackend for a schema-v2 target.` + ); + } + const rowBackend = assertExecutionBackend( + entry.executionBackend, + `backend report row ${key}.executionBackend` + ); + if (rowBackend !== targetIdentity.executionBackend) { + throw new Error( + `backend report row ${key} executionBackend "${rowBackend}" does not match ` + + `target "${targetIdentity.executionBackend}".` + ); + } + if (byKey.has(key)) { + throw new Error(`duplicate backend report key "${key}".`); + } + byKey.set(key, entry); + } + return byKey; +} diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 3fe9c6fa5dd..560e683975c 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -31,22 +31,28 @@ /** Every drift class this module can emit, with a stable one-line meaning. */ export const DRIFT_CLASSES = { GRAMMAR_RULE_MISSING: 'grammar-rule-missing', + EXECUTION_BACKEND_DIVERGENCE: 'execution-backend-divergence', + BACKEND_ORACLE_MISMATCH: 'backend-oracle-mismatch', ENGINE_RELAXED: 'engine-relaxed', ENGINE_PARTIALLY_RELAXED: 'engine-partially-relaxed', ENGINE_TIGHTENED: 'engine-tightened', ENGINE_MESSAGE_CHANGED: 'engine-message-changed', DETECTOR_SILENT: 'detector-silent', DETECTOR_NOISY: 'detector-noisy', + DETECTOR_COUNT_MISMATCH: 'detector-count-mismatch', + DETECTOR_MESSAGE_MISMATCH: 'detector-message-mismatch', VERSION_SCOPE_TOO_NARROW: 'version-scope-too-narrow', SEVERITY_MISMATCH: 'severity-mismatch', }; /** Remediation actions, phrased as what the linter engineer changes. */ export const REMEDIATIONS = { + ALIGN_EXECUTION_BACKENDS: 'align-execution-backends', DISABLE_RULE: 'disable-rule', VERSION_SCOPE_RULE: 'version-scope-rule', UPDATE_DETECTOR: 'update-detector', UPDATE_CONTRACT: 'update-contract', + REVIEW_BACKEND_ORACLE: 'review-backend-oracle', }; /** OSD paths an engineer edits, kept in one place so a move is a one-line fix. */ @@ -186,6 +192,119 @@ function describeObservation(observed) { return `detector ${detector}, engine ${backend}`; } +function describeBackendVerdict(observed) { + if (!observed || typeof observed.backendRejected !== 'boolean') { + return 'did not produce a verdict'; + } + if (!observed.backendRejected) { + return 'ACCEPTED'; + } + const type = observed.backendType ? ` (${observed.backendType})` : ''; + return `REJECTED${type}`; +} + +function executionBackendRemediation(ruleId, detectorPath) { + return { + action: REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, + target: `analytics backend and ${detectorFile(ruleId, detectorPath)}`, + detail: + `Keep the OpenSearch version bounds unchanged. Review the route-specific backend oracles, ` + + `then either align analytics behavior with standard, narrow the detector to behavior common ` + + `to both routes, disable the rule for every route, or add a reliable execution-backend signal ` + + `to the lint context before emitting route-specific diagnostics.`, + }; +} + +/** + * Compare the two execution routes for one query on the same engine candidate. + * This is deliberately separate from product-version drift: a route difference + * cannot justify changing an OpenSearch version range. + */ +export function classifyExecutionBackendDivergence({ + ruleId, + version, + queryName, + role = 'trigger', + query, + standardObserved, + analyticsObserved, + standardLeg, + analyticsLeg, + grammarHash, + detectorPath, +}) { + if ( + !standardObserved || + !analyticsObserved || + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + return null; + } + + const where = `${ruleId} @ ${version} [${queryName}]`; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + baselineExecutionBackend: 'standard', + executionBackends: ['standard', 'analytics'], + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + evidence: + `${where}: standard ${describeBackendVerdict(standardObserved)} while analytics ` + + `${describeBackendVerdict(analyticsObserved)} on the same engine candidate and runtime ` + + `grammar${grammarHash ? ` (${grammarHash})` : ''}` + + `${standardLeg || analyticsLeg ? `; legs ${standardLeg || 'standard'} / ${analyticsLeg || 'analytics'}` : ''}.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; +} + +function backendOracleRemediation(executionBackend) { + return { + action: REMEDIATIONS.REVIEW_BACKEND_ORACLE, + target: `${executionBackend} backend and this contract file`, + detail: + `Review the captured raw response and determine whether the backend regressed or the reviewed ` + + `${executionBackend} oracle is stale. Restore the backend behavior when the status/result change ` + + `is unintended; update the oracle only after confirming the new behavior is intentional. Keep ` + + `the OpenSearch version bounds unchanged.`, + }; +} + +function classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected, + detectorPath, + reason, +}) { + const expected = expectedRejected ? 'REJECTION' : 'ACCEPTANCE'; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + executionBackends: ['analytics'], + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${ruleId} @ ${version} [${queryName}]: analytics ${describeBackendVerdict(observed)}, ` + + `but ${reason || `its reviewed oracle requires ${expected}`}.`, + remediation: backendOracleRemediation('analytics'), + }; +} + /** * Report a parser rule the detector walks that the candidate grammar no longer * defines. Exported so a caller can raise it ONCE per rule/version — the fact is @@ -203,6 +322,7 @@ export function classifyGrammarDrift({ role = 'trigger', query, detectorPath, + executionBackend = 'standard', }) { if (!Array.isArray(requiredParserRules) || !Array.isArray(parserRuleNames)) { return null; @@ -222,9 +342,10 @@ export function classifyGrammarDrift({ queryName, role, query, + executionBackend, driftClass: DRIFT_CLASSES.GRAMMAR_RULE_MISSING, evidence: - `${ruleId} @ ${version}${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + + `${ruleId} @ ${version} (${executionBackend})${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + `which this rule's detector walks.` + (observed && observed.detectorCount !== undefined ? ` ${describeObservation(observed)}.` : ''), remediation: { @@ -281,6 +402,7 @@ export function classifyRelaxationScope({ detectorFlagged = false, wiring, detectorPath, + executionBackend = 'standard', }) { if (relaxedTriggers.length === 0) { return null; @@ -292,6 +414,7 @@ export function classifyRelaxationScope({ version, driftVersion: version, role: 'trigger', + executionBackend, scope: { relaxed: [...relaxedTriggers], holding: [...holdingTriggers], @@ -306,6 +429,17 @@ export function classifyRelaxationScope({ const observed = relaxedTriggers.length + holdingTriggers.length; const basis = `${relaxedTriggers.length} of ${observed} observed trigger(s) relaxed`; + if (executionBackend === 'analytics') { + return { + ...base, + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + executionBackends: ['analytics'], + evidence: + `${where}: analytics accepted ${basis}; this is route-specific behavior, not product-version drift.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; + } + // --- Partial: some triggers relaxed, others still rejected ------------------ // The engine fixed part of the condition. Scoping the rule out of this version // would ship a false negative on everything in `holding`, so the action is to @@ -413,13 +547,14 @@ export function classifyDrift(input) { expectedBackend, detectorPath, controlAlsoRejected, + executionBackend = 'standard', } = input; const detectorFlagged = (observed.detectorCount || 0) > 0; const expectFlagged = (expected.detectorCount || 0) > 0; const backendRejected = observed.backendRejected; - const where = `${ruleId} @ ${version} [${queryName}]`; - const base = { ruleId, version, queryName, role, query, driftVersion: version }; + const where = `${ruleId} @ ${version} (${executionBackend}) [${queryName}]`; + const base = { ruleId, version, queryName, role, query, driftVersion: version, executionBackend }; // --- 1. Did the grammar move out from under the detector? ------------------- // A detector that walks a parser rule the candidate grammar no longer defines @@ -435,6 +570,7 @@ export function classifyDrift(input) { role, query, detectorPath, + executionBackend, }); if (grammarDrift) { return grammarDrift; @@ -446,6 +582,7 @@ export function classifyDrift(input) { // engine rejects the trigger, the version window is too narrow and users on // this version get no diagnostic. const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); + const expectRejection = expected.backendKind === 'rejection'; if (!inScope) { // A trigger the engine rejects normally means the version window is too // narrow. But if the rule's CONTROL — a valid query using the same command — @@ -455,6 +592,19 @@ export function classifyDrift(input) { // what is really "unsupported command", so that case is correctly silent: // the version window is doing its job. if (role === 'trigger' && backendRejected === true && controlAlsoRejected !== true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + reason: 'the standard product-version rule is inactive for this engine candidate', + }); + } return { ...base, driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, @@ -492,7 +642,7 @@ export function classifyDrift(input) { `the version context rather than deciding on its own, and remember OSD's version filter runs ` + `a rule when the cluster version is unknown — so this also fires for users whose version ` + `could not be resolved.` + - (backendRejected === true + (backendRejected === true && executionBackend === 'standard' ? ` The engine does reject this query, so widening appliesTo in ${OSD_PATHS.catalog} may be` + ` the right fix instead.` : ''), @@ -504,13 +654,24 @@ export function classifyDrift(input) { } // --- 3. Behavioral flips: the engine changed its verdict -------------------- - const expectRejection = expected.backendKind === 'rejection'; // 3a. The engine now ACCEPTS what the contract pinned as a rejection. Any // diagnostic the linter still emits is a false positive shipped to users — // the single most damaging drift, so it is reported even when the detector // count happens to match the stale expectation. if (role === 'trigger' && expectRejection && backendRejected === false) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: true, + detectorPath, + }); + } return { ...base, // A single relaxed trigger cannot tell a full fix from a partial one, and the @@ -548,6 +709,18 @@ export function classifyDrift(input) { // 3b. The engine now REJECTS what the contract pinned as valid. A control that // started failing means the linter is silently missing a real error. if (!expectRejection && backendRejected === true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + }); + } return { ...base, driftClass: DRIFT_CLASSES.ENGINE_TIGHTENED, @@ -590,6 +763,10 @@ export function classifyDrift(input) { const detectorMatches = (observed.detectorCount || 0) === (expected.detectorCount || 0); if (backendRejected === true && expectRejection && expectedBackend && detectorMatches) { const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; + const statusChanged = + expectedBackend.httpStatus !== undefined && + observed.backendStatus !== undefined && + expectedBackend.httpStatus !== observed.backendStatus; const typeChanged = expectedError.type !== undefined && observed.backendType !== undefined && @@ -598,6 +775,16 @@ export function classifyDrift(input) { expectedError.reason !== undefined && observed.backendReason !== undefined && expectedError.reason !== observed.backendReason; + if (statusChanged) { + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend still rejects the query, but its HTTP status changed from ` + + `${expectedBackend.httpStatus} to ${observed.backendStatus}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } if (typeChanged || reasonChanged) { const parts = []; if (typeChanged) parts.push(`error.type "${expectedError.type}" -> "${observed.backendType}"`); @@ -620,6 +807,36 @@ export function classifyDrift(input) { } } + // Result-shape assertions and other detailed backend oracles can change while + // the coarse accepted/rejected verdict stays the same. The Java observer + // records that assertion failure explicitly; it must not be treated as + // agreement merely because a boolean verdict is still available. + if (observed.backendOutcome === 'observed-mismatch' || observed.backendOutcome === 'fail') { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: expectRejection, + detectorPath, + reason: `its reviewed backend oracle did not match: ${ + observed.backendMismatch || 'unspecified assertion mismatch' + }`, + }); + } + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend kept the same coarse verdict but failed its detailed oracle — ` + + `${observed.backendMismatch || 'unspecified assertion mismatch'}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } + // --- 5. Detector-only disagreements ---------------------------------------- // The engine behaved as pinned, so any mismatch is on the linter side. if (expectFlagged && !detectorFlagged) { @@ -670,13 +887,31 @@ export function classifyDrift(input) { }; } - // --- 6. Right verdict, wrong severity -------------------------------------- + if (observed.detectorCount !== expected.detectorCount) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH, + evidence: + `${where}: expected exactly ${expected.detectorCount} diagnostic(s), but the detector ` + + `emitted ${observed.detectorCount} while the backend behaved as pinned.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the detector to emit exactly ${expected.detectorCount} diagnostic(s) for this ` + + `query, or re-pin detectorCount only after confirming the changed multiplicity is intentional.`, + }, + }; + } + + // --- 6. Right verdict, wrong severity/message ------------------------------ if ( expected.severity && detectorFlagged && - Array.isArray(observed.severities) && - observed.severities.length > 0 && - !observed.severities.every((s) => s === expected.severity) + (observed.severityMatched === false || + (Array.isArray(observed.severities) && + observed.severities.length > 0 && + !observed.severities.every((s) => s === expected.severity))) ) { return { ...base, @@ -694,6 +929,23 @@ export function classifyDrift(input) { }; } + if (expected.matchMessage && observed.messageMatched !== true) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH, + evidence: + `${where}: the detector diagnostic no longer contains the contracted message fragment ` + + `${JSON.stringify(expected.matchMessage)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the diagnostic message asserted by this contract, or update matchMessage only ` + + `after reviewing the new user-facing wording.`, + }, + }; + } + return null; } @@ -720,9 +972,11 @@ export function formatDriftReport(drifts) { // Most urgent action first: a false positive already reaching users outranks a // stale pinned string. const order = [ + REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, REMEDIATIONS.DISABLE_RULE, REMEDIATIONS.VERSION_SCOPE_RULE, REMEDIATIONS.UPDATE_DETECTOR, + REMEDIATIONS.REVIEW_BACKEND_ORACLE, REMEDIATIONS.UPDATE_CONTRACT, ]; for (const action of order) { @@ -730,7 +984,11 @@ export function formatDriftReport(drifts) { if (!group || group.length === 0) continue; lines.push(`## ${action} (${group.length})`); for (const drift of group) { - lines.push(`- [${drift.driftClass}] ${drift.evidence}`); + const backend = + Array.isArray(drift.executionBackends) && drift.executionBackends.length > 1 + ? drift.executionBackends.join(' vs ') + : drift.executionBackend || 'standard'; + lines.push(`- [${drift.driftClass}] [${backend}] ${drift.evidence}`); lines.push(` FIX (${drift.remediation.target}): ${drift.remediation.detail}`); // A rule-level finding (e.g. a grammar rename) has no single query behind it. if (drift.query) { diff --git a/scripts/ppl-lint/probe-discovery-backend.mjs b/scripts/ppl-lint/probe-discovery-backend.mjs index 8ebb0d25d75..dc5d5282a8a 100644 --- a/scripts/ppl-lint/probe-discovery-backend.mjs +++ b/scripts/ppl-lint/probe-discovery-backend.mjs @@ -15,9 +15,9 @@ * which is what this does, in the same report shape the aggregator and the labeler * already read. * - * Emits `[{ ruleId, queryName, rejected, outcome, observed: { httpStatus, type, - * reason } }]`, matching `backend-report.json` so `label-discovery.mjs` can read - * either source without a special case. + * Emits `[{ ruleId, queryName, executionBackend, rejected, outcome, observed: + * { httpStatus, type, reason } }]`, matching `backend-report.json` so + * `label-discovery.mjs` can read either source without a special case. * * The `outcome` field carries the distinction everything downstream depends on: * @@ -172,6 +172,7 @@ async function main() { queryName: entry.name || `discovery-${i}`, role: 'discovery', query: entry.query, + executionBackend: 'standard', ...verdict, }; }); diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index dcdfe5e4a1d..3606ee6510f 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -25,7 +25,7 @@ * regardless of where the entry script lives. That is what lets this SQL-owned * `.mjs` load OSD's Node-safe headless lint API without OSD's own Jest. * - * This is the detector half of a schema-v3 cross-repository differential + * This is the detector half of a schema-v3/v4 cross-repository differential * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). * Unlike the earlier PoC — which linted with the compiled analyzer or a * hand-rolled reparse against OSD `main`'s checked-in grammar — it lints against @@ -78,6 +78,15 @@ import fs from 'fs'; import path from 'path'; import { createRequire } from 'module'; +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + // OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved // against the OSD checkout root, not this script's SQL-repo location. const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; @@ -125,6 +134,37 @@ function fatal(message) { process.exit(2); } +function loadContractFile(file) { + try { + const spec = JSON.parse(fs.readFileSync(file, 'utf8')); + assertContractSchema(spec); + const grammarSurface = spec.grammarSurface || 'runtime-bundle'; + if (!['runtime-bundle', 'compiled-simplified', 'both'].includes(grammarSurface)) { + throw new Error( + `[${spec.ruleId}] grammarSurface must be "runtime-bundle", ` + + `"compiled-simplified", or "both", got ${JSON.stringify(grammarSurface)}.` + ); + } + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array.`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + // Validate every declared oracle, including ranges not selected by this + // target. Missing route coverage is a supported state; malformed route + // names and oracle kinds are not. + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + return { file, spec }; + } catch (error) { + fatal(`Invalid contract ${file}: ${error.message}`); + } + return undefined; // unreachable +} + /** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ function loadContracts() { const dir = process.env.PPL_LINT_CONTRACT_DIR; @@ -134,7 +174,7 @@ function loadContracts() { if (!fs.existsSync(single)) { fatal(`Contract file not found: ${single}`); } - return [{ file: single, spec: JSON.parse(fs.readFileSync(single, 'utf8')) }]; + return [loadContractFile(single)]; } if (!dir) { @@ -147,7 +187,12 @@ function loadContracts() { const manifestPath = path.join(dir, 'manifest.json'); let files; if (fs.existsSync(manifestPath)) { - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + fatal(`Invalid contract manifest ${manifestPath}: ${error.message}`); + } if (!Array.isArray(manifest.contracts)) { fatal(`manifest.json must have a "contracts" array of file names.`); } @@ -164,7 +209,7 @@ function loadContracts() { if (!fs.existsSync(file)) { fatal(`Contract referenced by manifest not found: ${file}`); } - return { file, spec: JSON.parse(fs.readFileSync(file, 'utf8')) }; + return loadContractFile(file); }); } @@ -253,7 +298,7 @@ function loadOsd() { } /** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ -function loadCandidateGrammar(osd) { +function loadCandidateGrammar(osd, target) { const bundlePath = process.env.PPL_LINT_GRAMMAR_BUNDLE; if (!bundlePath) { fatal( @@ -270,6 +315,18 @@ function loadCandidateGrammar(osd) { } catch (error) { fatal(`Could not parse grammar bundle ${bundlePath}: ${error.message}`); } + if (bundle.grammarHash !== target.grammarHash) { + fatal( + `Candidate grammar hash ${JSON.stringify(bundle.grammarHash)} does not match target ` + + `${JSON.stringify(target.grammarHash)}.` + ); + } + if (target.grammarBundle && path.basename(bundlePath) !== target.grammarBundle) { + fatal( + `Candidate grammar filename "${path.basename(bundlePath)}" does not match target ` + + `"${target.grammarBundle}".` + ); + } try { return osd.deserializeBundleOrThrow(bundle); } catch (error) { @@ -278,38 +335,44 @@ function loadCandidateGrammar(osd) { return undefined; // unreachable } -/** Read the target manifest (engineVersion + grammarHash) written beside the bundle. */ +/** Read and validate the target identity written beside the grammar bundle. */ function loadTarget() { const targetPath = process.env.PPL_LINT_TARGET_MANIFEST; - if (targetPath && fs.existsSync(targetPath)) { - try { - return JSON.parse(fs.readFileSync(targetPath, 'utf8')); - } catch (error) { - log(`WARN: could not parse target manifest ${targetPath}: ${error.message}`); - } + if (!targetPath) { + fatal('PPL_LINT_TARGET_MANIFEST is required.'); + } + if (!fs.existsSync(targetPath)) { + fatal(`Target manifest not found: ${targetPath}`); + } + try { + return normalizeTarget(JSON.parse(fs.readFileSync(targetPath, 'utf8'))); + } catch (error) { + fatal(`Invalid target manifest ${targetPath}: ${error.message}`); } - // Back-compat / local runs without a target manifest. - return { engineVersion: process.env.PPL_SQL_VERSION || '', grammarHash: '' }; + return undefined; // unreachable } /** Index the backend report by `${ruleId}::${queryName}` for the differential. */ -function loadBackendReport() { +function loadBackendReport(target) { const reportPath = process.env.PPL_LINT_BACKEND_REPORT; - if (!reportPath || !fs.existsSync(reportPath)) { + if (!reportPath) { return undefined; } + if (!fs.existsSync(reportPath)) { + fatal(`Backend report not found: ${reportPath}`); + } let entries; try { entries = JSON.parse(fs.readFileSync(reportPath, 'utf8')); } catch (error) { - log(`WARN: could not parse backend report ${reportPath}: ${error.message}`); - return undefined; + fatal(`Could not parse backend report ${reportPath}: ${error.message}`); } - const byKey = new Map(); - for (const entry of Array.isArray(entries) ? entries : []) { - byKey.set(`${entry.ruleId}::${entry.queryName}`, entry); + try { + return indexBackendReport(entries, target); + } catch (error) { + fatal(`Invalid backend report ${reportPath}: ${error.message}`); } - return byKey; + return undefined; // unreachable } /** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ @@ -372,7 +435,7 @@ function versionMatchesRange(range, version) { * Exactly one must match (design §5.3): zero means the rule test does not cover * this version; more than one means overlapping ranges. Both fail. */ -function selectExpectation(spec, version, isCalcite, failures) { +function selectExpectation(spec, version, isCalcite, failures, { allowMissing = false } = {}) { const expectations = spec.expectations || []; const matches = expectations.filter((exp) => { if (!versionMatchesRange(exp.version, version)) return false; @@ -384,10 +447,13 @@ function selectExpectation(spec, version, isCalcite, failures) { } const label = version || 'unknown'; if (matches.length === 0) { - failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + if (!allowMissing) { + failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + } } else { - failures.push( - `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} (exactly one required).` + fatal( + `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} ` + + '(exactly one required).' ); } return undefined; @@ -490,6 +556,9 @@ function buildContext(spec, engineVersion) { function main() { const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; const reportPath = process.env.PPL_LINT_REPORT; + const target = loadTarget(); + const backendReport = loadBackendReport(target); + const contracts = loadContracts(); const osd = loadOsd(); const { getBundledCatalog, getDetector, lintQuery, osdRoot, surface } = osd; @@ -498,17 +567,24 @@ function main() { // The compiled surface lints with OSD's own checked-in grammar, so there is no // candidate bundle to load. On the runtime surface a missing bundle stays a hard // failure — never a quiet downgrade to the compiled grammar. - const grammar = surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd); - const target = loadTarget(); - const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; - const backendReport = loadBackendReport(); + const grammar = + surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd, target); + const engineVersion = target.engineVersion; + const executionBackend = target.executionBackend; + const observeAnalytics = process.env.PPL_LINT_OBSERVE_ANALYTICS === '1'; + const observeOnly = + process.env.PPL_LINT_OBSERVE_ONLY === '1' || observeAnalytics; + if (observeAnalytics && executionBackend !== 'analytics') { + fatal('PPL_LINT_OBSERVE_ANALYTICS=1 requires an analytics target.'); + } - const contracts = loadContracts(); const failures = []; // Contracts this surface did not score, recorded so the report says a rule was // skipped for surface rather than leaving its absence unexplained. const skippedForSurface = []; const report = { + schemaVersion: 2, + executionBackend, osdRoot, schedule, engineVersion, @@ -521,6 +597,8 @@ function main() { // see WHY a rule has no scored cases here. skippedForSurface, grammarHash: target.grammarHash || '', + observeAnalytics, + observeOnly, differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD // catalog this run linted with. The multi-version aggregator enforces its @@ -537,7 +615,7 @@ function main() { log(`OSD root: ${osdRoot}`); log( - `schedule=${schedule} engineVersion=${engineVersion || '(unset)'} ` + + `schedule=${schedule} engineVersion=${engineVersion} executionBackend=${executionBackend} ` + `grammarHash=${target.grammarHash || '(unset)'} differential=${!!backendReport} ` + `contracts=${contracts.length}` ); @@ -576,6 +654,8 @@ function main() { role: queryDef.role || 'trigger', query: (queryDef.query || '').split('{{index}}').join(index), surface, + executionBackend, + outcome: 'not-applicable', notApplicable: `contract declares grammarSurface "${contractSurface}"`, }); } @@ -588,23 +668,71 @@ function main() { } const context = buildContext(spec, engineVersion); - const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures); + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures, { + allowMissing: observeOnly, + }); if (!expectation) { + if (!observeOnly) { + continue; + } + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + if (surface === 'compiled-simplified' && entry.runtimeOnly) { + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + executionBackend, + outcome: 'not-applicable', + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + const result = lintQuery(query, grammar, context); + const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + executionBackend, + expected: 0, + actual: matches.length, + severities: matches.map((m) => m.severity), + severityMatched: true, + messageMatched: true, + backendOracleStatus: 'coverage-missing', + expectationStatus: 'coverage-missing', + }); + } continue; } const queries = spec.queries || {}; const expectedQueries = expectation.queries || {}; - for (const queryName of Object.keys(expectedQueries)) { + try { + assertExactQueryCoverage(spec, expectation); + } catch (error) { + fatal(`Invalid contract ${file}: ${error.message}`); + } + for (const queryName of Object.keys(queries)) { const queryDef = queries[queryName]; - if (!queryDef) { - failures.push(`[${ruleId}] expectation references unknown query "${queryName}".`); - continue; - } const role = queryDef.role || 'trigger'; const query = queryDef.query.split('{{index}}').join(index); const expected = expectedQueries[queryName]; - const expectedCount = expected.detectorCount; + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, executionBackend); + } catch (error) { + fatal(`Invalid contract ${file} query "${queryName}": ${error.message}`); + } + const expectedCount = oracleSelection.detector.count; + const expectedSeverity = oracleSelection.detector.severity; + const expectedMessage = oracleSelection.detector.matchMessage; // A `runtimeOnly` rule walks grammar productions that exist only in the // runtime bundle, so `lint_runner` skips it on the compiled surface. Its @@ -623,6 +751,8 @@ function main() { role, query, surface, + executionBackend, + outcome: 'not-applicable', notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', }); continue; @@ -639,9 +769,12 @@ function main() { ); const severityOk = - !expected.severity || actual === 0 || matches.every((m) => m.severity === expected.severity); + !expectedSeverity || + actual === 0 || + matches.every((m) => m.severity === expectedSeverity); const messageOk = - !expected.matchMessage || matches.some((m) => (m.message || '').includes(expected.matchMessage)); + !expectedMessage || + matches.some((m) => (m.message || '').includes(expectedMessage)); const resultEntry = { ruleId, @@ -651,6 +784,10 @@ function main() { expected: expectedCount, actual, severities: matches.map((m) => m.severity), + severityMatched: severityOk, + messageMatched: messageOk, + executionBackend, + backendOracleStatus: oracleSelection.status, }; if (!ok) { @@ -659,10 +796,30 @@ function main() { ); } if (!severityOk) { - failures.push(`[${ruleId}/${queryName}] expected severity "${expected.severity}" for: ${query}`); + failures.push( + `[${ruleId}/${queryName}] expected severity "${expectedSeverity}" for: ${query}` + ); } if (!messageOk) { - failures.push(`[${ruleId}/${queryName}] expected message to contain "${expected.matchMessage}" for: ${query}`); + failures.push( + `[${ruleId}/${queryName}] expected message to contain "${expectedMessage}" for: ${query}` + ); + } + + if (oracleSelection.status === 'not-applicable') { + resultEntry.outcome = 'not-applicable'; + resultEntry.reason = oracleSelection.reason; + resultEntry.notApplicable = oracleSelection.reason; + } else if (oracleSelection.status === 'coverage-missing') { + resultEntry.outcome = 'coverage-missing'; + resultEntry.coverage = 'missing'; + resultEntry.reason = oracleSelection.reason; + resultEntry.coverageMissing = oracleSelection.reason; + if (!observeAnalytics) { + failures.push( + `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` + ); + } } // Differential: the observed backend behavior must agree with the observed @@ -672,56 +829,71 @@ function main() { // passes. This catches drift the two halves would otherwise hide by both // pinning to the same JSON. if (backendReport) { - const backendKind = expected.backend && expected.backend.kind; - const expectRejected = backendKind === 'rejection'; const be = backendReport.get(`${ruleId}::${queryName}`); if (!be) { failures.push(`[${ruleId}/${queryName}] no backend report entry (backend did not run this query).`); } else { - resultEntry.backendRejected = !!be.rejected; - if (!!be.rejected !== expectRejected) { - failures.push( - `[${ruleId}/${queryName}] differential: backend ${be.rejected ? 'rejected' : 'accepted'} ` + - `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` - ); - } - // Trigger cross-check: a trigger the detector flags must be one the engine - // ALSO objects to — but only where the contract claims the engine objects - // at all. - // - // For a `rejection` rule the two coincide: detector flags <-> engine - // rejects, and a disagreement means one side drifted. That is the original - // check and it is unchanged. - // - // An ADVISORY rule is different by design. It flags a query the engine - // runs happily: `head-without-sort` marks non-determinism, - // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via - // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that - // rule working, not drift — so pairing the detector against `be.rejected` - // failed every advisory trigger unconditionally. That, not runtime cost, - // is the structural reason those contracts could only run nightly. - // - // The contracts already carry the distinction in `backend.kind`, so this - // reads data that exists rather than adding a flag. Advisory triggers keep - // full coverage from the other two assertions: the backend-kind check above - // fires if the engine starts REJECTING a query pinned as accepted, and the - // `detectorCount` assertion fires if the detector stops flagging it. Only - // the pairing rule is scoped to the rules it makes sense for. - const detectorFlagged = actual > 0; - if (role === 'trigger' && expectRejected && detectorFlagged !== !!be.rejected) { - failures.push( - `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` - ); - } - // A control must pass on both sides regardless of kind: it is a valid - // query the rule has to stay quiet on. Unlike a trigger, that claim does - // not vary with `backend.kind`. - if (role === 'control' && (detectorFlagged || be.rejected)) { + const backendObservation = classifyBackendReportRow(be); + if (oracleSelection.status !== 'applicable') { + // A missing or non-applicable oracle is never an acceptance claim. Keep + // any backend observation visible, but do not coerce a missing verdict + // through `!!be.rejected` or score a differential against another route. + resultEntry.backendOutcome = backendObservation.status; + } else if (backendObservation.status !== 'observed') { + resultEntry.backendOutcome = backendObservation.status; failures.push( - `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `and backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + + `(outcome=${JSON.stringify(backendObservation.status)}).` ); + } else { + const backendKind = oracleSelection.oracle.kind; + const expectRejected = backendKind === 'rejection'; + const backendRejected = backendObservation.rejected; + resultEntry.backendRejected = backendRejected; + if (backendRejected !== expectRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Trigger cross-check: a trigger the detector flags must be one the engine + // ALSO objects to — but only where the contract claims the engine objects + // at all. + // + // For a `rejection` rule the two coincide: detector flags <-> engine + // rejects, and a disagreement means one side drifted. That is the original + // check and it is unchanged. + // + // An ADVISORY rule is different by design. It flags a query the engine + // runs happily: `head-without-sort` marks non-determinism, + // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via + // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that + // rule working, not drift — so pairing the detector against `be.rejected` + // failed every advisory trigger unconditionally. That, not runtime cost, + // is the structural reason those contracts could only run nightly. + // + // The contracts already carry the distinction in `backend.kind`, so this + // reads data that exists rather than adding a flag. Advisory triggers keep + // full coverage from the other two assertions: the backend-kind check above + // fires if the engine starts REJECTING a query pinned as accepted, and the + // `detectorCount` assertion fires if the detector stops flagging it. Only + // the pairing rule is scoped to the rules it makes sense for. + const detectorFlagged = actual > 0; + if (role === 'trigger' && expectRejected && detectorFlagged !== backendRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + // A control must pass on both sides regardless of kind: it is a valid + // query the rule has to stay quiet on. Unlike a trigger, that claim does + // not vary with `backend.kind`. + if (role === 'control' && (detectorFlagged || backendRejected)) { + failures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } } } } @@ -746,7 +918,7 @@ function main() { fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); log(`wrote report to ${reportPath}`); } catch (error) { - log(`WARN: could not write report to ${reportPath}: ${error.message}`); + fatal(`Could not write detector report ${reportPath}: ${error.message}`); } } From 0cd55a6b5c0f2740b86c762a3eb6082cface152e Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 13:00:23 -0700 Subject: [PATCH 65/78] fix(ci): follow current analytics artifact names Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1364ecd3f6f..2ab965e3a14 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -342,7 +342,7 @@ task downloadTestPplFrontendZip(type: Download) { } task downloadAnalyticsBackendLuceneZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-lucene-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-lucene-${pluginVersion}.zip" dest analyticsBackendLuceneZipDest overwrite false onlyIfModified true @@ -350,7 +350,7 @@ task downloadAnalyticsBackendLuceneZip(type: Download) { } task downloadParquetDataFormatZip(type: Download) { - src "${featureBuildBase}/1-parquet-data-format-${pluginVersion}.zip" + src "${featureBuildBase}/parquet-data-format-${pluginVersion}.zip" dest parquetDataFormatZipDest overwrite false onlyIfModified true @@ -358,7 +358,7 @@ task downloadParquetDataFormatZip(type: Download) { } task downloadCompositeEngineZip(type: Download) { - src "${featureBuildBase}/1-composite-engine-${pluginVersion}.zip" + src "${featureBuildBase}/2-composite-engine-${pluginVersion}.zip" dest compositeEngineZipDest overwrite false onlyIfModified true @@ -366,7 +366,7 @@ task downloadCompositeEngineZip(type: Download) { } task downloadAnalyticsBackendDatafusionZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-datafusion-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-datafusion-${pluginVersion}.zip" dest analyticsBackendDatafusionZipDest overwrite false onlyIfModified true From 0c8ae23c6294ea137e661f7ef8ab84f4bd886b73 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 13:51:22 -0700 Subject: [PATCH 66/78] fix(ci): harden analytics lint observation Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 144 ++++++++- integ-test/build.gradle | 65 +++- .../remote/PplLintRuleValidationIT.java | 19 +- scripts/ppl-lint/README.md | 2 + .../__tests__/aggregate-versions.test.mjs | 23 ++ .../__tests__/contract-schema.test.mjs | 22 ++ .../validate-pr-build-targets.test.mjs | 271 +++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 6 +- scripts/ppl-lint/contract-schema.mjs | 6 + .../ppl-lint/validate-pr-build-targets.mjs | 285 ++++++++++++++++++ 10 files changed, 825 insertions(+), 18 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs create mode 100644 scripts/ppl-lint/validate-pr-build-targets.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index b02a2a5ecf7..de047b07569 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -557,6 +557,8 @@ jobs: needs: Get-CI-Image-Tag runs-on: ubuntu-latest timeout-minutes: 30 + env: + ANALYTICS_FEATURE_BUILD_LATEST: https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -573,12 +575,64 @@ jobs: distribution: 'temurin' java-version: 25 - - name: Run analytics contract observation against the PR build + - name: Resolve analytics feature build + id: analytics-build run: | set -euo pipefail mkdir -p leg + requested_manifest="${ANALYTICS_FEATURE_BUILD_LATEST}/manifest.yml" + resolved_manifest=$(curl --fail --silent --show-error --location \ + --retry 3 --retry-all-errors \ + --output leg/analytics-feature-manifest.yml \ + --write-out '%{url_effective}' \ + "$requested_manifest") + artifact_root="${resolved_manifest%/manifest.yml}" + plugin_base="${artifact_root}/plugins" + native_url="${artifact_root}/dist/libopensearch_native.so" + { + echo "artifact_root=$artifact_root" + echo "plugin_base=$plugin_base" + echo "native_url=$native_url" + } >> "$GITHUB_OUTPUT" + ANALYTICS_ARTIFACT_ROOT="$artifact_root" \ + ANALYTICS_PLUGIN_BASE="$plugin_base" \ + ANALYTICS_NATIVE_URL="$native_url" \ + ANALYTICS_RESOLVED_MANIFEST="$resolved_manifest" \ + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + manifest = Path("leg/analytics-feature-manifest.yml") + context = { + "schemaVersion": 1, + "stage": "feature-build-resolved", + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ["ANALYTICS_ARTIFACT_ROOT"], + "pluginBase": os.environ["ANALYTICS_PLUGIN_BASE"], + "nativeLibraryUrl": os.environ["ANALYTICS_NATIVE_URL"], + "resolvedManifestUrl": os.environ["ANALYTICS_RESOLVED_MANIFEST"], + "manifestSha256": "sha256:" + hashlib.sha256(manifest.read_bytes()).hexdigest(), + } + Path("leg/analytics-bootstrap.json").write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + + - name: Run analytics contract observation against the PR build + id: analytics-observation + env: + ANALYTICS_FEATURE_BUILD_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_LIB_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + set -euo pipefail chown -R 1000:1000 "$(pwd)" su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ + -PanalyticsFeatureBuildBase=${ANALYTICS_FEATURE_BUILD_BASE} \ + -PanalyticsNativeLibUrl=${ANALYTICS_NATIVE_LIB_URL} \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ -Dppl.lint.sql_sha=${GITHUB_SHA} \ @@ -586,6 +640,69 @@ jobs: -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/leg/target.json" + - name: Record analytics bootstrap provenance + if: ${{ always() }} + env: + OBSERVATION_OUTCOME: ${{ steps.analytics-observation.outcome }} + ANALYTICS_ARTIFACT_ROOT: ${{ steps.analytics-build.outputs.artifact_root }} + ANALYTICS_PLUGIN_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + mkdir -p leg + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + context_file = Path("leg/analytics-bootstrap.json") + if context_file.exists(): + context = json.loads(context_file.read_text(encoding="utf-8")) + else: + context = { + "schemaVersion": 1, + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ.get("ANALYTICS_ARTIFACT_ROOT") or None, + "pluginBase": os.environ.get("ANALYTICS_PLUGIN_BASE") or None, + "nativeLibraryUrl": os.environ.get("ANALYTICS_NATIVE_URL") or None, + } + + def describe(file): + digest = hashlib.sha256() + with file.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return { + "path": str(file), + "size": file.stat().st_size, + "sha256": "sha256:" + digest.hexdigest(), + } + + distributions = Path("integ-test/build/distributions") + native_libraries = sorted( + Path("integ-test/build/native").glob( + "*/release/libopensearch_native.so" + ) + ) + artifacts = ( + [describe(file) for file in sorted(distributions.glob("*.zip"))] + if distributions.is_dir() + else [] + ) + artifacts.extend(describe(file) for file in native_libraries) + context["stage"] = "observation-finished" + context["outcome"] = os.environ.get("OBSERVATION_OUTCOME") or "not-run" + context["effectiveJavaLibraryPaths"] = [ + str(file.parent) for file in native_libraries + ] + context["downloadedArtifacts"] = artifacts + context_file.write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + - name: Upload analytics leg artifacts if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -629,6 +746,22 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Download all leg artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-leg-* + path: legs + + - name: Validate PR-build target identity + run: | + node scripts/ppl-lint/validate-pr-build-targets.mjs \ + --standard legs/ppl-lint-leg-pr-build/target.json \ + --analytics legs/ppl-lint-leg-pr-build-analytics/target.json \ + --standard-report legs/ppl-lint-leg-pr-build/backend-report.json \ + --analytics-report legs/ppl-lint-leg-pr-build-analytics/backend-report.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --schedule nightly + - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -674,12 +807,6 @@ jobs: done exit 1 - - name: Download all leg artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 - with: - pattern: ppl-lint-leg-* - path: legs - # One detector pass per leg, each against THAT engine's grammar bundle. The # runner is the same SQL-owned script the single-version workflow uses, so # the detector half cannot drift between the two checks. @@ -783,6 +910,7 @@ jobs: --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ --summary "$GITHUB_STEP_SUMMARY" \ + --all-rules \ --observe-analytics \ "${args[@]}" @@ -797,6 +925,8 @@ jobs: legs/**/detector-report.json legs/**/detector.log legs/**/target.json + legs/**/analytics-bootstrap.json + legs/**/analytics-feature-manifest.yml # Discovery: harvest queries from OSD's own lint tests, run both halves over them, # and report detector/engine disagreements as LEADS. diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 2ab965e3a14..39c3b5e512d 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -300,6 +300,7 @@ def getGeoSpatialPlugin() { ext.pluginVersion = opensearch_version.tokenize('-')[0] ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildArtifactRoot = featureBuildBase.replaceFirst('/plugins/?$', '') ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -308,6 +309,16 @@ ext.analyticsBackendLuceneZipDest = "${buildDir}/distributions/analytics-backend ext.parquetDataFormatZipDest = "${buildDir}/distributions/parquet-data-format-${pluginVersion}-SNAPSHOT.zip" ext.compositeEngineZipDest = "${buildDir}/distributions/composite-engine-${pluginVersion}-SNAPSHOT.zip" ext.analyticsBackendDatafusionZipDest = "${buildDir}/distributions/analytics-backend-datafusion-${pluginVersion}-SNAPSHOT.zip" +ext.analyticsNativeLibUrl = project.findProperty('analyticsNativeLibUrl') ?: + "${featureBuildArtifactRoot}/dist/libopensearch_native.so" +ext.analyticsNativeLibDest = "${buildDir}/native/${pluginVersion}/release/libopensearch_native.so" +ext.analyticsNativeLibDir = project.findProperty('nativeLibPath') ? + rootProject.file(project.findProperty('nativeLibPath')).canonicalFile : + file(analyticsNativeLibDest).parentFile.canonicalFile +ext.analyticsJavaLibraryPath = [ + analyticsNativeLibDir.absolutePath, + System.getProperty('java.library.path') +].findAll { it != null && !it.isEmpty() }.join(File.pathSeparator) task downloadAnalyticsEngineZip(type: Download) { src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip" @@ -373,6 +384,50 @@ task downloadAnalyticsBackendDatafusionZip(type: Download) { onlyIf { !project.findProperty('analyticsBackendDatafusionZip') } } +task downloadAnalyticsNativeLib(type: Download) { + src analyticsNativeLibUrl + dest analyticsNativeLibDest + // The mutable observation URL can publish another build under the same + // product version. Revalidate an existing file and never expose a partial + // download to the test cluster. + overwrite true + onlyIfModified true + tempAndMove true + retries 3 + onlyIf { !project.findProperty('nativeLibPath') } + doFirst { + def osName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT) + def osArch = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT) + if (!osName.contains('linux') || !(osArch in ['amd64', 'x86_64'])) { + throw new GradleException( + "The default analytics native artifact is Linux/x64 only " + + "(detected ${osName}/${osArch}); pass -PnativeLibPath=.") + } + } +} + +task validateAnalyticsNativeLib { + dependsOn downloadAnalyticsNativeLib + doLast { + File nativeLib = new File(analyticsNativeLibDir, 'libopensearch_native.so') + if (!nativeLib.isFile() || !nativeLib.canRead() || nativeLib.length() == 0) { + throw new GradleException( + "Expected a readable non-empty native library at ${nativeLib}. " + + "Pass -PnativeLibPath= " + + "or -PanalyticsNativeLibUrl=.") + } + byte[] magic = new byte[4] + int bytesRead + nativeLib.withInputStream { stream -> bytesRead = stream.read(magic) } + if (bytesRead != magic.length || + (magic[0] & 0xff) != 0x7f || (magic[1] & 0xff) != 0x45 || + (magic[2] & 0xff) != 0x4c || (magic[3] & 0xff) != 0x46) { + throw new GradleException( + "Analytics native library ${nativeLib} is not an ELF shared object.") + } + } +} + def getAnalyticsEnginePlugin() { provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) } } @@ -475,10 +530,9 @@ testClusters { systemProperty 'io.netty.tryReflectionSetAccessible', 'true' systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' - // Native library path for DataFusion/parquet -- pass via -PnativeLibPath=/path/to/release/ - if (project.findProperty('nativeLibPath')) { - systemProperty 'java.library.path', project.findProperty('nativeLibPath') - } + // DataFusion/parquet loads libopensearch_native.so at cluster startup. Use the + // matching feature-build artifact unless a local release directory is supplied. + systemProperty 'java.library.path', analyticsJavaLibraryPath } } @@ -543,7 +597,8 @@ task analyticsEnginePplLintIT(type: RestIntegTestTask) { useCluster testClusters.analyticsEnginePplLintIT dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, downloadCompositeEngineZip, downloadParquetDataFormatZip, - downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip, + validateAnalyticsNativeLib dependsOn ':opensearch-sql-plugin:bundlePlugin' systemProperty 'tests.analytics.parquet_indices', 'true' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 01012b7406d..f873e4c0021 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -1205,7 +1205,9 @@ private boolean pluginComponentMatches(String component, String required) { private void verifyAnalyticsClusterSettings() throws IOException { Response nodesResponse = client().performRequest(new Request("GET", "/_nodes/settings?flat_settings=true")); - JSONObject nodes = new JSONObject(getResponseBody(nodesResponse, true)).getJSONObject("nodes"); + JSONObject nodesBody = new JSONObject(getResponseBody(nodesResponse, true)); + analyticsRouteAttestation.put("nodeSettings", nodesBody); + JSONObject nodes = nodesBody.getJSONObject("nodes"); requireAttestation(nodes.length() > 0, "node settings response contained no nodes"); for (String nodeId : nodes.keySet()) { String startupDataFormat = @@ -1239,12 +1241,12 @@ private void verifyAnalyticsClusterSettings() throws IOException { .performRequest( new Request("GET", "/_cluster/settings?flat_settings=true&include_defaults=true")); JSONObject settings = new JSONObject(getResponseBody(response, true)); + analyticsRouteAttestation.put("clusterSettings", settings); requireEffectiveSetting(settings, "cluster.pluggable.dataformat", "composite"); requireEffectiveSetting(settings, "cluster.pluggable.dataformat.enabled", "true"); requireEffectiveSetting(settings, "cluster.composite.primary_data_format", "parquet"); requireEffectiveSettingContains(settings, "cluster.composite.secondary_data_formats", "lucene"); - analyticsRouteAttestation.put("clusterSettings", settings); } private void verifyAnalyticsFixtureIndices() throws IOException { @@ -1284,11 +1286,11 @@ private void verifyAnalyticsFixtureIndices() throws IOException { Response countResponse = client().performRequest(new Request("GET", "/" + indexName + "/_count")); long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + documentCounts.put(indexName, count); + fixtureEvidence.put("documentCount", count); requireAttestation( count > 0, "fixture " + indexName + " contains no documents; fixture ingestion did not complete"); - documentCounts.put(indexName, count); - fixtureEvidence.put("documentCount", count); } } @@ -1346,9 +1348,12 @@ private String sha256(String value) { } private void verifyAnalyticsExplainCanaries() throws IOException { + JSONObject explainPlans = new JSONObject(); + analyticsRouteAttestation.put("explainPlans", explainPlans); for (String indexEnum : requiredIndexEnums()) { String query = analyticsCanaryQuery(indexEnum); String explained = explainQueryToString(query); + explainPlans.put(indexEnum, explained); requireAttestation( explained.contains("LogicalTableScan(table=[[opensearch,"), "fixture " + indexEnum + " did not use LogicalTableScan(opensearch): " + explained); @@ -1360,8 +1365,13 @@ private void verifyAnalyticsExplainCanaries() throws IOException { private void verifyAnalyticsProfileCanaries() throws IOException { JSONArray executionTypes = new JSONArray(); + JSONObject profiles = new JSONObject(); + analyticsRouteAttestation + .put("profileExecutionTypes", executionTypes) + .put("profiles", profiles); for (String indexEnum : requiredIndexEnums()) { JSONObject response = runProfiledPplQuery(analyticsCanaryQuery(indexEnum)); + profiles.put(indexEnum, response); JSONObject profile = response.getJSONObject("profile"); JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); requireAttestation( @@ -1377,7 +1387,6 @@ private void verifyAnalyticsProfileCanaries() throws IOException { executionTypes.put(stage.getString("execution_type")); } } - analyticsRouteAttestation.put("profileExecutionTypes", executionTypes); } private String analyticsCanaryQuery(String indexEnum) { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index db60cbe23ca..79bd5d8f1dc 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -89,6 +89,8 @@ OSD_REF= ./scripts/ppl-lint-rule-validation.sh PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh # Run the corpus through the full composite/Parquet + DataFusion stack. +# The published default stack is Linux/x64; other platforms need compatible +# local plugin artifacts and -PnativeLibPath. RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh # Use locally built analytics plugins (all trailing arguments pass to Gradle). diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 76f64e31348..b498f594745 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -562,6 +562,29 @@ test('analytics coverage gaps cannot hide missing raw observations', () => { assert.match(report.inconclusive[0].reasons.join(' '), /no engine verdict/); }); +test('--all-rules makes incomplete non-default observations fail as infrastructure', () => { + const contracts = writeContracts(); + const manifestFile = path.join(contracts, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + manifest.defaultError = []; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const leg = writeLeg({ + version: '3.8.0', + defaultErrorRules: [], + cases: { control: { detector: 0, rejected: false } }, + }); + + const { status, report } = run({ + contracts, + legs: [['3.8.0', leg]], + extraArgs: ['--all-rules'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); +}); + test('paired detector reports must be identical across execution backends', () => { const grammarHash = 'sha256:shared-runtime-grammar'; const standard = writeLeg({ diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs index 6dce090d45d..1f4516ca83b 100644 --- a/scripts/ppl-lint/__tests__/contract-schema.test.mjs +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -312,6 +312,28 @@ test('backend oracle payloads fail closed when required shapes are malformed', ( }, expected: /\.body\.status must be an integer/, }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: 500 }, + }, + expected: /\.httpStatus must equal .*\.body\.status/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 201, + }, + expected: /\.httpStatus must be 200/, + }, + { + oracle: { + kind: 'advisory', + httpStatus: 204, + }, + expected: /\.httpStatus must be 200/, + }, { oracle: { kind: 'result-shape', diff --git a/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs b/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs new file mode 100644 index 00000000000..430daf81797 --- /dev/null +++ b/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs @@ -0,0 +1,271 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + validatePrBuildArtifacts, + validatePrBuildTargetPair, +} from '../validate-pr-build-targets.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'validate-pr-build-targets.mjs'); +const tmpDirs = []; + +function standardTarget(overrides = {}) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:candidate-grammar', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + ...overrides, + }; +} + +function analyticsTarget(overrides = {}) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:candidate-grammar', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + ...overrides, + }; +} + +function backendReport(executionBackend, queryNames = ['trigger', 'control']) { + return queryNames.map((queryName) => ({ + ruleId: 'test-rule', + queryName, + role: queryName === 'control' ? 'control' : 'trigger', + query: + queryName === 'control' + ? 'source=test-index | head 1' + : 'source=test-index | head 0', + executionBackend, + rejected: queryName !== 'control', + observed: { + httpStatus: queryName === 'control' ? 200 : 400, + rejected: queryName !== 'control', + }, + })); +} + +function writeContractCorpus() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-contracts-')); + tmpDirs.push(dir); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ schemaVersion: 3, contracts: ['test-rule.spec.json'] }) + ); + fs.writeFileSync( + path.join(dir, 'test-rule.spec.json'), + JSON.stringify({ + schemaVersion: 3, + ruleId: 'test-rule', + schedule: 'pr', + queries: { + trigger: { role: 'trigger', query: 'source={{index}} | head 0' }, + control: { role: 'control', query: 'source={{index}} | head 1' }, + }, + }) + ); + return dir; +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('matching standard and analytics PR-build targets pass', () => { + const pair = validatePrBuildTargetPair(standardTarget(), analyticsTarget()); + assert.equal(pair.standard.executionBackend, 'standard'); + assert.equal(pair.analytics.executionBackend, 'analytics'); +}); + +test('the two PR-build target roles require their exact backend identities', () => { + assert.throws( + () => validatePrBuildTargetPair(analyticsTarget(), analyticsTarget()), + /standard PR-build target executionBackend must be "standard"/ + ); + assert.throws( + () => validatePrBuildTargetPair(standardTarget(), standardTarget()), + /analytics PR-build target executionBackend must be "analytics"/ + ); +}); + +test('both PR-build targets require the same non-empty SQL SHA', () => { + assert.throws( + () => validatePrBuildTargetPair(standardTarget({ sqlSha: '' }), analyticsTarget()), + /both report a non-empty SQL SHA/ + ); + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ sqlSha: 'different-sql-sha' }) + ), + /report different values for SQL SHA/ + ); +}); + +test('both PR-build targets require the same engine version', () => { + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ engineVersion: '3.9.0-SNAPSHOT' }) + ), + /report different values for engine version/ + ); +}); + +test('both PR-build targets require the same non-empty grammar hash', () => { + assert.throws( + () => validatePrBuildTargetPair(standardTarget(), analyticsTarget({ grammarHash: ' ' })), + /both report a non-empty grammar hash/ + ); + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ grammarHash: 'sha256:different-grammar' }) + ), + /report different values for grammar hash/ + ); +}); + +test('target schema validation runs before pair identity comparison', () => { + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget({ schemaVersion: 1 }), + analyticsTarget() + ), + /standard PR-build target is invalid: Unsupported target schemaVersion/ + ); +}); + +test('paired backend reports require exact, usable query coverage', () => { + const base = { + standardTarget: standardTarget(), + analyticsTarget: analyticsTarget(), + standardReport: backendReport('standard'), + analyticsReport: backendReport('analytics'), + contractsDir: writeContractCorpus(), + }; + const result = validatePrBuildArtifacts(base); + assert.equal(result.expectedRows, 2); + + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + analyticsReport: backendReport('analytics', ['trigger']), + }), + /analytics PR-build backend report query coverage is incomplete.*test-rule::control/ + ); + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + standardReport: [ + ...backendReport('standard'), + { ...backendReport('standard')[0] }, + ], + }), + /duplicate backend report key/ + ); + + const errored = backendReport('analytics'); + errored[0] = { ...errored[0], outcome: 'error' }; + assert.throws( + () => validatePrBuildArtifacts({ ...base, analyticsReport: errored }), + /contains rows without an engine verdict: test-rule::trigger/ + ); + + const unobservedCoverageGap = backendReport('analytics'); + unobservedCoverageGap[0] = { + ...unobservedCoverageGap[0], + outcome: 'coverage-missing', + }; + delete unobservedCoverageGap[0].rejected; + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + analyticsReport: unobservedCoverageGap, + }), + /contains rows without an engine verdict: test-rule::trigger/ + ); + + const changedQuery = backendReport('analytics'); + changedQuery[0] = { ...changedQuery[0], query: 'source=different-index | head 0' }; + assert.throws( + () => validatePrBuildArtifacts({ ...base, analyticsReport: changedQuery }), + /executed different query text for test-rule::trigger/ + ); +}); + +test('the CLI reads and validates both PR-build artifact sets', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-pair-')); + tmpDirs.push(dir); + const standardFile = path.join(dir, 'standard.json'); + const analyticsFile = path.join(dir, 'analytics.json'); + const standardReportFile = path.join(dir, 'standard-report.json'); + const analyticsReportFile = path.join(dir, 'analytics-report.json'); + fs.writeFileSync(standardFile, JSON.stringify(standardTarget())); + fs.writeFileSync(analyticsFile, JSON.stringify(analyticsTarget())); + fs.writeFileSync(standardReportFile, JSON.stringify(backendReport('standard'))); + fs.writeFileSync(analyticsReportFile, JSON.stringify(backendReport('analytics'))); + + const result = spawnSync( + process.execPath, + [ + SCRIPT, + '--standard', + standardFile, + '--analytics', + analyticsFile, + '--standard-report', + standardReportFile, + '--analytics-report', + analyticsReportFile, + '--contracts', + writeContractCorpus(), + '--schedule', + 'nightly', + ], + { encoding: 'utf8' } + ); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /verified engine=3\.8\.0-SNAPSHOT/); + assert.match(result.stdout, /backends=standard,analytics/); + assert.match(result.stdout, /backendRows=2/); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index e5dd0510ca6..973b1f8728f 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -1382,7 +1382,11 @@ function main() { } const enforcedDrifts = drifts.filter((d) => d.blocking); const enforcedHoles = coverageHoles.filter((h) => h.blocking); - const enforcedInconclusive = inconclusive.filter((i) => i.enforced); + // `--all-rules` widens observation to the whole corpus. Semantic drift remains + // enforced only for default-error rules, but a missing detector row or backend + // verdict is an infrastructure failure for every rule we asked the run to + // observe. + const enforcedInconclusive = inconclusive.filter((i) => i.enforced || args.allRules); for (const row of matrix) { row.key = reportItemKey(row, 'matrix'); } diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs index 5df505f07de..291f2f9c405 100644 --- a/scripts/ppl-lint/contract-schema.mjs +++ b/scripts/ppl-lint/contract-schema.mjs @@ -63,11 +63,17 @@ function assertBackendOracle(oracle, ruleId, executionBackend) { if (!Number.isInteger(oracle.httpStatus) || oracle.httpStatus < 100 || oracle.httpStatus > 599) { throw new TypeError(`${label}.httpStatus must be an integer from 100 through 599.`); } + if ((kind === 'result-shape' || kind === 'advisory') && oracle.httpStatus !== 200) { + throw new TypeError(`${label}.httpStatus must be 200.`); + } if (kind === 'rejection') { const body = requireObject(oracle.body, `${label}.body`); if (!Number.isInteger(body.status)) { throw new TypeError(`${label}.body.status must be an integer.`); } + if (body.status !== oracle.httpStatus) { + throw new TypeError(`${label}.httpStatus must equal ${label}.body.status.`); + } if (body.error !== undefined) { const error = requireObject(body.error, `${label}.body.error`); assertOptionalString(error.type, `${label}.body.error.type`); diff --git a/scripts/ppl-lint/validate-pr-build-targets.mjs b/scripts/ppl-lint/validate-pr-build-targets.mjs new file mode 100644 index 00000000000..b7944ed76d7 --- /dev/null +++ b/scripts/ppl-lint/validate-pr-build-targets.mjs @@ -0,0 +1,285 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { + assertContractSchema, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, +} from './contract-schema.mjs'; + +function normalizeLabeledTarget(target, label) { + try { + return normalizeTarget(target); + } catch (error) { + throw new Error(`${label} target is invalid: ${error.message}`); + } +} + +function requireMatchingNonEmptyField(standard, analytics, field, label) { + if ( + typeof standard[field] !== 'string' || + standard[field].trim().length === 0 || + typeof analytics[field] !== 'string' || + analytics[field].trim().length === 0 + ) { + throw new Error( + `standard and analytics targets must both report a non-empty ${label}: ` + + `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` + ); + } + if (standard[field] !== analytics[field]) { + throw new Error( + `standard and analytics targets report different values for ${label}: ` + + `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` + ); + } +} + +export function validatePrBuildTargetPair(standardRaw, analyticsRaw) { + const standard = normalizeLabeledTarget(standardRaw, 'standard PR-build'); + const analytics = normalizeLabeledTarget(analyticsRaw, 'analytics PR-build'); + + if (standard.executionBackend !== 'standard') { + throw new Error( + `standard PR-build target executionBackend must be "standard", got ` + + `${JSON.stringify(standard.executionBackend)}` + ); + } + if (analytics.executionBackend !== 'analytics') { + throw new Error( + `analytics PR-build target executionBackend must be "analytics", got ` + + `${JSON.stringify(analytics.executionBackend)}` + ); + } + + requireMatchingNonEmptyField(standard, analytics, 'engineVersion', 'engine version'); + requireMatchingNonEmptyField(standard, analytics, 'sqlSha', 'SQL SHA'); + requireMatchingNonEmptyField(standard, analytics, 'grammarHash', 'grammar hash'); + + return { standard, analytics }; +} + +function expectedBackendReportKeys(contractsDir, schedule) { + if (schedule !== 'pr' && schedule !== 'nightly') { + throw new Error(`schedule must be "pr" or "nightly", got ${JSON.stringify(schedule)}`); + } + const manifest = readJson(path.join(contractsDir, 'manifest.json'), 'contract manifest'); + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + !Array.isArray(manifest.contracts) + ) { + throw new TypeError('contract manifest.contracts must be a JSON array'); + } + + const files = new Set(); + const ruleIds = new Set(); + const expectedRows = new Map(); + for (const file of manifest.contracts) { + if (typeof file !== 'string' || file.length === 0) { + throw new TypeError('contract manifest entries must be non-empty strings'); + } + if (files.has(file)) { + throw new Error(`contract manifest contains duplicate file ${JSON.stringify(file)}`); + } + files.add(file); + + const spec = readJson(path.join(contractsDir, file), `contract ${file}`); + assertContractSchema(spec); + if (ruleIds.has(spec.ruleId)) { + throw new Error(`contract manifest contains duplicate ruleId ${JSON.stringify(spec.ruleId)}`); + } + ruleIds.add(spec.ruleId); + if (schedule === 'pr' && (spec.schedule || 'pr') !== 'pr') { + continue; + } + if ( + spec.queries === null || + typeof spec.queries !== 'object' || + Array.isArray(spec.queries) || + Object.keys(spec.queries).length === 0 + ) { + throw new TypeError(`[${spec.ruleId}] contract.queries must be a non-empty JSON object`); + } + for (const queryName of Object.keys(spec.queries)) { + const key = `${spec.ruleId}::${queryName}`; + if (expectedRows.has(key)) { + throw new Error(`contract corpus contains duplicate query key ${JSON.stringify(key)}`); + } + const query = spec.queries[queryName]; + if (query === null || typeof query !== 'object' || Array.isArray(query)) { + throw new TypeError(`[${spec.ruleId}] query ${JSON.stringify(queryName)} must be an object`); + } + expectedRows.set(key, { role: query.role || 'trigger' }); + } + } + if (expectedRows.size === 0) { + throw new Error(`contract corpus selected no queries for schedule ${JSON.stringify(schedule)}`); + } + return expectedRows; +} + +function validateBackendReport(raw, target, label, expectedRows) { + let rows; + try { + rows = indexBackendReport(raw, target); + } catch (error) { + throw new Error(`${label} backend report is invalid: ${error.message}`); + } + + const missing = [...expectedRows.keys()].filter((key) => !rows.has(key)).sort(); + const extra = [...rows.keys()].filter((key) => !expectedRows.has(key)).sort(); + if (missing.length > 0 || extra.length > 0) { + const details = []; + if (missing.length > 0) details.push(`missing: ${missing.join(', ')}`); + if (extra.length > 0) details.push(`unexpected: ${extra.join(', ')}`); + throw new Error(`${label} backend report query coverage is incomplete (${details.join('; ')})`); + } + + const unusable = []; + for (const [key, row] of rows) { + const expected = expectedRows.get(key); + if (row.role !== expected.role) { + throw new Error( + `${label} backend report row ${key}.role must be ${JSON.stringify(expected.role)}, ` + + `got ${JSON.stringify(row.role)}` + ); + } + if (typeof row.query !== 'string' || row.query.length === 0) { + throw new Error(`${label} backend report row ${key}.query must be a non-empty string`); + } + const status = classifyBackendReportRow(row).status; + if ( + status === 'error' || + (status === 'coverage-missing' && typeof row.rejected !== 'boolean') + ) { + unusable.push(key); + } + } + if (unusable.length > 0) { + throw new Error( + `${label} backend report contains rows without an engine verdict: ${unusable.sort().join(', ')}` + ); + } + return rows; +} + +export function validatePrBuildArtifacts({ + standardTarget, + analyticsTarget, + standardReport, + analyticsReport, + contractsDir, + schedule = 'nightly', +}) { + const pair = validatePrBuildTargetPair(standardTarget, analyticsTarget); + const expectedRows = expectedBackendReportKeys(contractsDir, schedule); + const standardRows = validateBackendReport( + standardReport, + pair.standard, + 'standard PR-build', + expectedRows + ); + const analyticsRows = validateBackendReport( + analyticsReport, + pair.analytics, + 'analytics PR-build', + expectedRows + ); + for (const key of expectedRows.keys()) { + const standard = standardRows.get(key); + const analytics = analyticsRows.get(key); + if (standard.query !== analytics.query) { + throw new Error( + `standard and analytics backend reports executed different query text for ${key}: ` + + `standard=${JSON.stringify(standard.query)}, analytics=${JSON.stringify(analytics.query)}` + ); + } + } + return { ...pair, expectedRows: expectedRows.size, standardRows, analyticsRows }; +} + +function parseArgs(argv) { + const options = new Map([ + ['--standard', 'standard'], + ['--analytics', 'analytics'], + ['--standard-report', 'standardReport'], + ['--analytics-report', 'analyticsReport'], + ['--contracts', 'contracts'], + ['--schedule', 'schedule'], + ]); + const args = { schedule: 'nightly' }; + const seen = new Set(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const key = options.get(arg); + if (!key) { + throw new Error(`unknown argument ${JSON.stringify(arg)}`); + } + const value = argv[++i]; + if (!value) { + throw new Error(`${arg} requires a value`); + } + if (seen.has(key)) { + throw new Error(`${arg} may be specified only once`); + } + seen.add(key); + args[key] = value; + } + for (const key of [ + 'standard', + 'analytics', + 'standardReport', + 'analyticsReport', + 'contracts', + ]) { + if (!args[key]) { + throw new Error(`${[...options].find(([, value]) => value === key)[0]} is required`); + } + } + return args; +} + +function readJson(file, label) { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + throw new Error(`could not read ${label} ${file}: ${error.message}`); + } +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const { standard, analytics, expectedRows } = validatePrBuildArtifacts({ + standardTarget: readJson(args.standard, 'standard PR-build target'), + analyticsTarget: readJson(args.analytics, 'analytics PR-build target'), + standardReport: readJson(args.standardReport, 'standard PR-build backend report'), + analyticsReport: readJson(args.analyticsReport, 'analytics PR-build backend report'), + contractsDir: args.contracts, + schedule: args.schedule, + }); + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-target-pair] verified engine=${standard.engineVersion} ` + + `sqlSha=${standard.sqlSha} grammarHash=${standard.grammarHash} ` + + `backends=${standard.executionBackend},${analytics.executionBackend} ` + + `backendRows=${expectedRows}` + ); +} + +if (process.argv[1] && process.argv[1].endsWith('validate-pr-build-targets.mjs')) { + try { + main(); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-target-pair] FATAL: ${error.message}`); + process.exitCode = 2; + } +} From 7126c6abdb39f45c249bb2d891b9e068c9b35437 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 14:22:55 -0700 Subject: [PATCH 67/78] fix(ci): resolve analytics lint contracts Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 39c3b5e512d..4676f594042 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -606,6 +606,7 @@ task analyticsEnginePplLintIT(type: RestIntegTestTask) { systemProperty 'ppl.lint.execution_backend', 'analytics' systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath filter { includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' From 92dc0bb6d087dfd3df07ce66829c8ac47e464702 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 14:59:30 -0700 Subject: [PATCH 68/78] fix(ci): support append-only analytics fixtures Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 122 +++++++++++- .../sql/legacy/AnalyticsFieldStripTests.java | 59 +++++- .../org/opensearch/sql/legacy/TestUtils.java | 89 +++++++-- .../contracts/flat-object-subfield.spec.json | 184 ++++++++++++------ scripts/ppl-lint/run-frontend-contract.mjs | 6 +- 5 files changed, 367 insertions(+), 93 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index f873e4c0021..e8dbafc42a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -169,6 +169,8 @@ public void init() throws Exception { // super.init() aborted partway, so redo the part that is version-independent. increaseMaxCompilationsRate(); } + clusterVersion = fetchClusterVersion(); + // Fall through to fixture seeding either way. // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { @@ -194,7 +196,6 @@ public void init() throws Exception { "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); } } - clusterVersion = fetchClusterVersion(); } @Test @@ -221,6 +222,8 @@ public void testValidatesLintRuleContracts() throws IOException { String ruleId = contract.getString("ruleId"); runContract(contract, ruleId, failures, report); } + } else { + recordUnattestedRouteContracts(contracts, report); } try { @@ -365,6 +368,52 @@ && recordEnforcementCoverageGaps( } } + /** + * Route attestation failure prevents query execution, but it must not produce a misleadingly + * empty report. Emit one non-verdict row per query; explicit, complete non-applicable rows remain + * non-applicable because they do not depend on the unavailable fixture. + */ + private void recordUnattestedRouteContracts(List contracts, JSONArray report) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + String index = contract.getString("index"); + JSONObject queries = contract.getJSONObject("queries"); + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations( + contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + JSONObject expectedQueries = + matches.size() == 1 ? matches.get(0).optJSONObject("queries") : null; + + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = + expectedQueries == null ? null : expectedQueries.optJSONObject(queryName); + JSONObject backend = null; + int schemaVersion = contract.optInt("schemaVersion"); + if (expected != null && (schemaVersion == 3 || schemaVersion == 4)) { + try { + backend = resolveBackendOracle(schemaVersion, expected); + } catch (RuntimeException ignored) { + // Malformed contracts are still failures; keep this report row as a non-verdict. + } + } + JSONObject entry = reportEntry(ruleId, queryName, role, query, "route-attestation-failed"); + if (isCompleteNotApplicableOracle(backend)) { + entry.put("kind", "not-applicable"); + recordNotApplicable(ruleId, queryName, backend, entry, report); + } else { + report.put( + entry + .put("outcome", "error") + .put("error", "analytics route attestation failed before contract execution")); + } + } + } + } + /** * The first index fixture this contract needs that failed to seed, or null when every index it * declares is present. Only ever non-null in observe-only mode, where a seeding failure is @@ -1283,9 +1332,7 @@ private void verifyAnalyticsFixtureIndices() throws IOException { requireIndexSetting( indexName, settings, "index.number_of_shards", Integer.toString(expectedShards)); - Response countResponse = - client().performRequest(new Request("GET", "/" + indexName + "/_count")); - long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + long count = analyticsDocumentCount(indexName); documentCounts.put(indexName, count); fixtureEvidence.put("documentCount", count); requireAttestation( @@ -1294,6 +1341,18 @@ private void verifyAnalyticsFixtureIndices() throws IOException { } } + private long analyticsDocumentCount(String indexName) throws IOException { + JSONObject response = + executeQuery("source=" + indexName + " | stats count() as document_count"); + JSONArray rows = response.getJSONArray("datarows"); + requireAttestation(rows.length() == 1, "fixture " + indexName + " count returned " + rows); + JSONArray row = rows.getJSONArray(0); + requireAttestation( + row.length() == 1 && row.get(0) instanceof Number, + "fixture " + indexName + " count did not return one numeric value: " + rows); + return ((Number) row.get(0)).longValue(); + } + private String canonicalJson(Object value) { if (value == null || value == JSONObject.NULL) { return "null"; @@ -1803,10 +1862,19 @@ private List manifestContractNames() throws IOException { return names; } - /** Union of index enums required by the contracts scheduled to run this session. */ + /** + * Union of index enums required by the contracts scheduled to run this session. + * + *

      A schema-v4 contract whose selected analytics oracle marks every query explicitly + * non-applicable does not need its unrepresentable fixture. Missing or malformed oracles remain + * fixture-requiring so they cannot turn into an implicit skip. + */ private Set requiredIndexEnums() throws IOException { Set indices = new LinkedHashSet<>(); for (JSONObject contract : loadScheduledContracts()) { + if (!contractRequiresFixture(contract)) { + continue; + } JSONObject fixture = contract.optJSONObject("backendFixture"); if (fixture == null) { continue; @@ -1825,6 +1893,50 @@ private Set requiredIndexEnums() throws IOException { return indices; } + private boolean contractRequiresFixture(JSONObject contract) { + if (executionBackend != ExecutionBackend.ANALYTICS || contract.optInt("schemaVersion") != 4) { + return true; + } + + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations(contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + if (matches.size() != 1) { + return true; + } + + JSONObject declaredQueries = contract.optJSONObject("queries"); + JSONObject expectedQueries = matches.get(0).optJSONObject("queries"); + if (declaredQueries == null + || declaredQueries.length() == 0 + || expectedQueries == null + || !declaredQueries.keySet().equals(expectedQueries.keySet())) { + return true; + } + + for (String queryName : declaredQueries.keySet()) { + JSONObject expected = expectedQueries.optJSONObject(queryName); + JSONObject backend = expected == null ? null : resolveBackendOracle(4, expected); + if (!isCompleteNotApplicableOracle(backend)) { + return true; + } + } + return false; + } + + private boolean isCompleteNotApplicableOracle(JSONObject backend) { + return backend != null + && "not-applicable".equals(backend.optString("kind")) + && hasNonBlankString(backend, "reason") + && hasNonBlankString(backend, "owner") + && hasNonBlankString(backend, "issue"); + } + + private boolean hasNonBlankString(JSONObject object, String key) { + Object value = object.opt(key); + return value instanceof String && !((String) value).trim().isEmpty(); + } + private JSONObject loadContractFile(String resourcePath) throws IOException { String path = TestUtils.getResourceFilePath(resourcePath); return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 36d2c4ed82a..1db2e38da9e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.util.List; @@ -168,7 +169,7 @@ public void mappingStrip_noopWhenDisabled() { public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { enable(); String bulk = - "{\"index\":{\"_id\":\"1\"}}\n" + "{\"index\":{\"_index\":\"one\",\"_id\":\"1\",\"routing\":\"r1\"}}\n" + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"geo_shape_value\":\"POINT(1" + " 2)\"}\n" + "{\"index\":{\"_id\":\"2\"}}\n" @@ -178,9 +179,12 @@ public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { bulk, Set.of(path("geo_point_value"), path("geo_shape_value"), path("nested_value"))); String[] lines = out.split("\n"); - // action lines untouched - assertTrue(lines[0].contains("\"index\"")); - assertTrue(lines[2].contains("\"index\"")); + // Append-only analytics indices require generated IDs, but all other action metadata remains. + JSONObject firstAction = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(firstAction.has("_id")); + assertEquals("one", firstAction.getString("_index")); + assertEquals("r1", firstAction.getString("routing")); + assertFalse(new JSONObject(lines[2]).getJSONObject("index").has("_id")); // source lines stripped, supported field retained JSONObject doc1 = new JSONObject(lines[1]); assertTrue(doc1.has("keep_text")); @@ -213,12 +217,51 @@ public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { } @Test - public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { - String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; + public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { + String indexSource = "{\"index\":\"source-value\",\"spacing\": 2}"; + String createSource = "{\"delete\":\"also-a-source-value\"}"; + String bulk = + "{\"index\":{\"_index\":\"fixture\",\"_id\":\"1\"}}\n" + + indexSource + + "\n" + + "{\"create\":{\"_index\":\"fixture\",\"_id\":\"2\",\"routing\":\"r2\"}}\n" + + createSource + + "\n"; // disabled -> unchanged even with a drop set assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("geo_point_value")))); - // enabled but empty drop set -> unchanged + + // enabled with no dropped fields -> generated IDs for append-only writes, source unchanged enable(); - assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of()); + String[] lines = out.split("\n", -1); + JSONObject index = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(index.has("_id")); + assertEquals("fixture", index.getString("_index")); + assertEquals(indexSource, lines[1]); + JSONObject create = new JSONObject(lines[2]).getJSONObject("create"); + assertEquals("2", create.getString("_id")); + assertEquals("fixture", create.getString("_index")); + assertEquals("r2", create.getString("routing")); + assertEquals(createSource, lines[3]); + // split(..., -1) proves the original terminal newline survived. + assertEquals("", lines[4]); + } + + @Test + public void bulkStrip_rejectsActionsThatAppendOnlyStorageCannotRepresent() { + enable(); + IllegalArgumentException update = + assertThrows( + IllegalArgumentException.class, + () -> + AnalyticsIndexConfig.stripBulkFields( + "{\"update\":{\"_id\":\"1\"}}\n{\"doc\":{\"value\":1}}\n", Set.of())); + assertTrue(update.getMessage().contains("does not support update")); + + IllegalArgumentException delete = + assertThrows( + IllegalArgumentException.class, + () -> AnalyticsIndexConfig.stripBulkFields("{\"delete\":{\"_id\":\"1\"}}\n", Set.of())); + assertTrue(delete.getMessage().contains("does not support delete")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 198527d1efc..2a5b996dea1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -248,41 +248,64 @@ private static void collectAndRemoveUnsupported( } /** - * Strip the given dropped paths from every source document of a bulk NDJSON payload. - * Bulk format alternates an action line ({@code {"index":{...}}}) with a source line; only - * source lines (those without a bulk action key) are rewritten. No-op when disabled or {@code - * droppedPaths} is empty. + * Prepare a bulk NDJSON payload for an analytics-engine append-only index. + * + *

      Custom document IDs are not supported for {@code index} operations when {@code + * index.append_only.enabled} is active, so {@code _id} is removed from that action metadata + * while preserving metadata such as {@code _index} and {@code routing}. Create actions retain + * their semantics; update/delete actions fail locally because they are incompatible with + * append-only storage. + * + *

      The given dropped paths are also removed from every source document. Bulk format + * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines + * (those without a bulk action key) have mapped fields removed. No-op when analytics mode is + * disabled. * *

      Each path is removed recursively: it descends through nested objects and arrays of * objects (so a {@code nested}/object array has the field stripped from every element), * leaving unaffected siblings intact. A source line is re-serialized only when removing a - * path actually changed it; every other line (action lines and docs that never had the - * dropped path) is appended byte-for-byte unchanged, so untouched docs match the fixture - * exactly. + * path actually changed it. Action lines are re-serialized only when removing {@code _id}; + * every other line is appended byte-for-byte unchanged. */ static String stripBulkFields(String bulkBody, Set> droppedPaths) { - if (!isEnabled() || droppedPaths.isEmpty()) { + if (!isEnabled()) { return bulkBody; } String[] lines = bulkBody.split("\n", -1); StringBuilder out = new StringBuilder(bulkBody.length()); + boolean expectSource = false; for (int i = 0; i < lines.length; i++) { String line = lines[i]; String trimmed = line.trim(); - if (!trimmed.isEmpty() && trimmed.charAt(0) == '{') { - JSONObject doc = new JSONObject(trimmed); - boolean isActionLine = - doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); - if (!isActionLine) { + boolean terminalNewline = i == lines.length - 1 && trimmed.isEmpty(); + if (!terminalNewline) { + if (trimmed.isEmpty()) { + throw new IllegalArgumentException( + "analytics bulk payload contains a blank NDJSON line"); + } + + JSONObject json = new JSONObject(trimmed); + if (expectSource) { boolean removedAny = false; for (List path : droppedPaths) { - removedAny |= removePath(doc, path, 0); + removedAny |= removePath(json, path, 0); } // Only rewrite the line if we actually removed something; otherwise leave it verbatim // so untouched docs stay byte-for-byte identical to the fixture. if (removedAny) { - line = doc.toString(); + line = json.toString(); + } + expectSource = false; + } else { + String operation = bulkOperation(json); + if ("update".equals(operation) || "delete".equals(operation)) { + throw new IllegalArgumentException( + "analytics append-only bulk payload does not support " + operation + " actions"); } + if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { + line = json.toString(); + } + expectSource = true; } } out.append(line); @@ -290,9 +313,40 @@ static String stripBulkFields(String bulkBody, Set> droppedPaths) { out.append('\n'); } } + if (expectSource) { + throw new IllegalArgumentException( + "analytics bulk payload ended before the final action's source document"); + } return out.toString(); } + private static String bulkOperation(JSONObject action) { + List operations = + List.of("index", "create", "update", "delete").stream() + .filter(action::has) + .collect(Collectors.toList()); + if (operations.size() != 1 || action.length() != 1) { + throw new IllegalArgumentException( + "analytics bulk action line must contain exactly one index/create/update/delete" + + " action"); + } + String operation = operations.get(0); + if (!(action.opt(operation) instanceof JSONObject)) { + throw new IllegalArgumentException( + "analytics bulk " + operation + " action metadata must be an object"); + } + return operation; + } + + private static boolean removeCustomDocumentId(JSONObject action, String operation) { + JSONObject metadata = action.optJSONObject(operation); + if (metadata == null || !metadata.has("_id")) { + return false; + } + metadata.remove("_id"); + return true; + } + /** * Remove {@code path[idx..]} from {@code node}, descending through objects and arrays of * objects. Returns true if anything was removed. At the last path part the key is deleted from @@ -429,8 +483,9 @@ public static void loadDataByRestClient( /** * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code * droppedPaths} (the exact field paths removed from the mapping on the analytics-engine route) - * from every bulk source doc, so the index mapping and the data agree. When AE is disabled or - * {@code droppedPaths} is empty this is byte-for-byte identical to the 3-arg form. + * from every bulk source doc, so the index mapping and the data agree. Analytics append-only + * {@code index} operations also discard custom document IDs. When analytics mode is disabled this + * is byte-for-byte identical to the 3-arg form. */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath, Set> droppedPaths) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json index 440c55e27ac..0ba5c95be35 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "flat-object-subfield", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -7,7 +7,7 @@ "qualifiedName", "wcQualifiedName" ], - "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", + "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true). The analytics feature build cannot create flat_object in composite/Parquet storage, so every analytics backend oracle is explicitly non-applicable while the frontend detector assertions still run.", "wiring": { "detector": "flat-object-subfield", "enabled": true, @@ -61,55 +61,87 @@ "flat-object-dotted-subfield": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-bare-root": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-in-where": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "non-flat-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } } @@ -122,55 +154,87 @@ "flat-object-dotted-subfield": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes.service] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-bare-root": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-in-where": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes.service] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "non-flat-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } } diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 3606ee6510f..7bd7de2ce61 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -807,9 +807,9 @@ function main() { } if (oracleSelection.status === 'not-applicable') { - resultEntry.outcome = 'not-applicable'; - resultEntry.reason = oracleSelection.reason; - resultEntry.notApplicable = oracleSelection.reason; + // Only the backend fixture is non-applicable. The detector still ran above and its + // count/severity/message assertions remain ordinary, comparable frontend evidence. + resultEntry.backendOracleReason = oracleSelection.reason; } else if (oracleSelection.status === 'coverage-missing') { resultEntry.outcome = 'coverage-missing'; resultEntry.coverage = 'missing'; From 946f19e939f62c0ddb25ae6dc54e7dbfdf4ec521 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 15:01:39 -0700 Subject: [PATCH 69/78] test(analytics): preserve bulk separators Signed-off-by: Hanyu Wei --- .../sql/legacy/AnalyticsFieldStripTests.java | 9 ++-- .../org/opensearch/sql/legacy/TestUtils.java | 50 ++++++++++--------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 1db2e38da9e..0fcedfee093 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -223,7 +223,7 @@ public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { String bulk = "{\"index\":{\"_index\":\"fixture\",\"_id\":\"1\"}}\n" + indexSource - + "\n" + + "\n\n" + "{\"create\":{\"_index\":\"fixture\",\"_id\":\"2\",\"routing\":\"r2\"}}\n" + createSource + "\n"; @@ -238,13 +238,14 @@ public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { assertFalse(index.has("_id")); assertEquals("fixture", index.getString("_index")); assertEquals(indexSource, lines[1]); - JSONObject create = new JSONObject(lines[2]).getJSONObject("create"); + assertEquals("", lines[2]); + JSONObject create = new JSONObject(lines[3]).getJSONObject("create"); assertEquals("2", create.getString("_id")); assertEquals("fixture", create.getString("_index")); assertEquals("r2", create.getString("routing")); - assertEquals(createSource, lines[3]); + assertEquals(createSource, lines[4]); // split(..., -1) proves the original terminal newline survived. - assertEquals("", lines[4]); + assertEquals("", lines[5]); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 2a5b996dea1..25385201e19 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -280,32 +280,36 @@ static String stripBulkFields(String bulkBody, Set> droppedPaths) { boolean terminalNewline = i == lines.length - 1 && trimmed.isEmpty(); if (!terminalNewline) { if (trimmed.isEmpty()) { - throw new IllegalArgumentException( - "analytics bulk payload contains a blank NDJSON line"); - } - - JSONObject json = new JSONObject(trimmed); - if (expectSource) { - boolean removedAny = false; - for (List path : droppedPaths) { - removedAny |= removePath(json, path, 0); - } - // Only rewrite the line if we actually removed something; otherwise leave it verbatim - // so untouched docs stay byte-for-byte identical to the fixture. - if (removedAny) { - line = json.toString(); - } - expectSource = false; - } else { - String operation = bulkOperation(json); - if ("update".equals(operation) || "delete".equals(operation)) { + if (expectSource) { throw new IllegalArgumentException( - "analytics append-only bulk payload does not support " + operation + " actions"); + "analytics bulk action is missing its source document"); } - if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { - line = json.toString(); + } else { + JSONObject json = new JSONObject(trimmed); + if (expectSource) { + boolean removedAny = false; + for (List path : droppedPaths) { + removedAny |= removePath(json, path, 0); + } + // Only rewrite the line if we actually removed something; otherwise leave it + // verbatim so untouched docs stay byte-for-byte identical to the fixture. + if (removedAny) { + line = json.toString(); + } + expectSource = false; + } else { + String operation = bulkOperation(json); + if ("update".equals(operation) || "delete".equals(operation)) { + throw new IllegalArgumentException( + "analytics append-only bulk payload does not support " + + operation + + " actions"); + } + if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { + line = json.toString(); + } + expectSource = true; } - expectSource = true; } } out.append(line); From d9739e8ccd8c7c31bc9a276dd27a8cf039c5efd2 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 3 Aug 2026 21:11:51 -0700 Subject: [PATCH 70/78] feat(ci): validate approved 13-rule PPL lint catalog Add channel-aware schema-v4 contracts for the approved twelve lint detectors and command-suggestion syntax feature. Keep default-off detector contracts dormant, enforce strict catalog wiring, and expand multi-version/discovery coverage across the active shipping corpus. Add the disabled-object backend fixture, normalized frontend and backend oracles, required-lane annotations, and syntax-aware aggregation. The exact OSD shipping census remains report-only until the paired OSD default-alignment change lands. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 17 +- .../workflows/ppl-lint-rule-validation.yml | 1 + .../remote/PplLintRuleValidationIT.java | 17 +- .../sql/legacy/SQLIntegTestCase.java | 5 + .../org/opensearch/sql/legacy/TestUtils.java | 5 + .../opensearch/sql/legacy/TestsConstants.java | 2 + ...pl_lint_disabled_object_index_mapping.json | 13 + .../ppl-lint/contracts/agg-on-text.spec.json | 127 +++++++ .../contracts/command-suggestion.spec.json | 142 ++++++++ .../dedup-consecutive-unsupported.spec.json | 6 +- .../contracts/disabled-join-type.spec.json | 6 +- .../contracts/division-by-zero.spec.json | 81 +++-- .../contracts/enabled-false-object.spec.json | 129 +++++++ .../contracts/field-validation.spec.json | 216 ++++++++---- .../contracts/flat-object-subfield.spec.json | 9 +- .../contracts/head-without-sort.spec.json | 6 +- .../invalid-capture-group-name.spec.json | 161 ++++++--- .../ppl-lint/contracts/manifest.json | 68 ++-- .../multisearch-min-subsearch.spec.json | 84 +++-- .../replace-wildcard-asymmetry.spec.json | 159 ++++++--- .../contracts/rex-scan-cost.spec.json | 103 ++++++ .../contracts/type-mismatch-numeric.spec.json | 125 +++++++ .../contracts/union-min-datasets.spec.json | 84 +++-- ...ed-window-function-in-eventstats.spec.json | 236 +++++++++---- .../wildcard-source-zero-match.spec.json | 94 ++++++ .../resources/ppl_lint_disabled_object.json | 4 + scripts/ppl-lint/README.md | 90 ++--- scripts/ppl-lint/__tests__/annotate.test.mjs | 80 +++++ .../__tests__/assemble-run-manifest.test.mjs | 15 + .../__tests__/contract-schema.test.mjs | 151 +++++++++ .../__tests__/harvest-queries.test.mjs | 36 ++ scripts/ppl-lint/aggregate-versions.mjs | 205 +++++++++++- scripts/ppl-lint/annotate.mjs | 140 +++++++- scripts/ppl-lint/assemble-run-manifest.mjs | 26 ++ scripts/ppl-lint/contract-schema.mjs | 233 +++++++++++-- scripts/ppl-lint/harvest-queries.mjs | 57 ++-- scripts/ppl-lint/label-discovery.mjs | 26 +- scripts/ppl-lint/run-frontend-contract.mjs | 315 +++++++++++++++--- 38 files changed, 2789 insertions(+), 485 deletions(-) create mode 100644 integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json create mode 100644 integ-test/src/test/resources/ppl_lint_disabled_object.json diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index de047b07569..8e31ab55918 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -9,18 +9,11 @@ name: PPL lint multi-version validation # happens to be. A rule that is correct on main can be a false positive on 3.6 or # a false negative on 3.7, and nothing notices. # -# This workflow validates every DEFAULT-ERROR rule (enabled: true + severity: -# error in OSD's rules_catalog.json) against SEVERAL released engine versions -# plus the PR build, and — when a rule disagrees with any of them — says what to -# change in the linter rather than only that a count was wrong. -# -# Why default-error only: an error-severity rule is one the user cannot opt out -# of and which marks their query as broken. A wrong error is the most expensive -# possible lint defect, so that set gets the multi-version treatment first. -# Warning/info rules stay on the single-version check. The set is not hand-copied: -# the detector run records the catalog's default-error census, and the aggregate -# step fails if a rule in that census has no contract file (see manifest.json's -# `defaultError` note). +# This workflow validates every active shipping contract — 12 detector rules and +# the command-suggestion syntax feature where its runtime grammar surface is +# available — against released engine versions plus the PR build. The aggregate +# still records the exact default-error census separately, but --all-rules makes +# warning/info omissions and syntax regressions visible too. # # Shape — a per-version matrix of observation legs, then one aggregation: # diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 078fd285494..6f83c60a60b 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -370,6 +370,7 @@ jobs: # green (design §4.4). A workflow_dispatch run is pre-merge evidence and is # intentionally not what repo admins pin to branch protection. - name: Require both validation jobs to have succeeded + if: ${{ always() }} env: BACKEND_RESULT: ${{ needs.backend-validation.result }} DETECTOR_RESULT: ${{ needs.detector-validation.result }} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index e8dbafc42a4..c6a503fddaf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -32,7 +32,7 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the schema-v3/schema-v4 PPL lint rule validation contract. + * Backend half of the schema-v3/schema-v4 PPL frontend validation contract. * *

      This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from * the current checkout. For every contract (see {@code @@ -50,9 +50,9 @@ * confirmed by a single run, e.g. head nondeterminism, fallback warnings). *

    * - *

    The contract files are shared verbatim with the SQL-owned OSD detector runner ({@code + *

    The contract files are shared verbatim with the SQL-owned OSD frontend runner ({@code * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer - * diagnostic count and the SQL backend behavior; neither side can drift without a red build. The + * output and the SQL backend behavior; neither side can drift without a red build. The * rejection-body parsing mirrors {@link * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT}; the Calcite setup follows {@link * org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. @@ -60,15 +60,14 @@ *

    While the ephemeral cluster is alive, the test also exports the candidate runtime grammar * bundle it built ({@code GET /_plugins/_ppl/_grammar}) and a small target manifest pairing the * bundle with the backend version and grammar hash. These become workflow artifacts that the - * detector-validation job injects into OSD's headless lint API, so both halves validate against the - * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code + * detector-validation job injects into OSD's headless lint and syntax APIs, so both halves validate + * against the SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. * *

    The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips - * contracts declaring {@code schedule: "nightly"}, while nightly runs the full corpus. Every - * contract in the corpus currently declares {@code schedule: "pr"}, so the two are equivalent - * today; the filter stays because it is the only mechanism for holding a new contract back from PR - * runs while its oracle is still settling. + * contracts declaring {@code schedule: "nightly"}, while nightly runs all 13 active contracts. The + * filter holds new detector and syntax contracts back from PR runs while their standard and + * analytics oracles are still settling. * *

    Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's * {@code enforced} list — that list records oracle quality and review status, not blocking diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 38e37c41d31..97db1640a5e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -877,6 +877,11 @@ public enum Index { "flat_object", getFlatObjectIndexMapping(), "src/test/resources/flat_object.json"), + PPL_LINT_DISABLED_OBJECT( + TestsConstants.TEST_INDEX_PPL_LINT_DISABLED_OBJECT, + "ppl_lint_disabled_object", + getPplLintDisabledObjectIndexMapping(), + "src/test/resources/ppl_lint_disabled_object.json"), DUPLICATION_NULLABLE( TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE, "duplication_nullable", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 25385201e19..1d92f807249 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -587,6 +587,11 @@ public static String getFlatObjectIndexMapping() { return getMappingFile(mappingFile); } + public static String getPplLintDisabledObjectIndexMapping() { + String mappingFile = "ppl_lint_disabled_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 957ff0108d6..357b5d37ef4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -77,6 +77,8 @@ public class TestsConstants { public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias"; public static final String TEST_INDEX_FLATTENED_VALUE = TEST_INDEX + "_flattened_value"; public static final String TEST_INDEX_FLAT_OBJECT = TEST_INDEX + "_flat_object"; + public static final String TEST_INDEX_PPL_LINT_DISABLED_OBJECT = + TEST_INDEX + "_ppl_lint_disabled_object"; public static final String TEST_INDEX_GEOIP = TEST_INDEX + "_geoip"; public static final String DATASOURCES = ".ql-datasources"; public static final String TEST_INDEX_STATE_COUNTRY = TEST_INDEX + "_state_country"; diff --git a/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json new file mode 100644 index 00000000000..6893eceb163 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/ppl_lint_disabled_object_index_mapping.json @@ -0,0 +1,13 @@ +{ + "mappings": { + "properties": { + "session": { + "type": "object", + "enabled": false + }, + "status": { + "type": "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json new file mode 100644 index 00000000000..4c574aaa86c --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json @@ -0,0 +1,127 @@ +{ + "schemaVersion": 4, + "ruleId": "agg-on-text", + "channel": "lint", + "note": "The standard engine accepts both text aggregations: avg(text) returns null while sum(text) returns a non-empty numeric result (observed as 0.0). The warning catches this misleading coercion rather than predicting backend rejection.", + "grammarSurface": "both", + "schedule": "nightly", + "wiring": { + "detector": "agg-on-text", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "firstname": "text", + "balance": "long" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "avg-text-field": { + "role": "trigger", + "query": "source={{index}} | stats avg(firstname) as avg_firstname" + }, + "sum-text-field": { + "role": "trigger", + "query": "source={{index}} | stats sum(firstname) as sum_firstname" + }, + "avg-numeric-control": { + "role": "control", + "query": "source={{index}} | stats avg(balance) as avg_balance" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "avg-text-field": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "text field" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "avg_firstname" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "avg_firstname" + } + } + } + }, + "sum-text-field": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "text field" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "avg-numeric-control": { + "frontend": { + "count": 0 + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json new file mode 100644 index 00000000000..e1bb53ffb3b --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json @@ -0,0 +1,142 @@ +{ + "schemaVersion": 4, + "ruleId": "command-suggestion", + "channel": "syntax", + "grammarSurface": "runtime-bundle", + "schedule": "nightly", + "wiring": { + "code": "UNKNOWN_COMMAND" + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "misspelled-command": { + "role": "trigger", + "query": "source={{index}} | wherre age > 1" + }, + "valid-command-control": { + "role": "control", + "query": "source={{index}} | where age > 1" + }, + "unrecognizable-command": { + "role": "suppression-control", + "query": "source={{index}} | zzzzzzzz" + }, + "incomplete-expression": { + "role": "suppression-control", + "query": "source={{index}} | where age >" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "misspelled-command": { + "frontend": { + "count": 1, + "code": "UNKNOWN_COMMAND", + "fixText": "where", + "matchMessage": "where", + "rawMessage": true + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + } + }, + "valid-command-control": { + "frontend": { + "count": 0, + "code": "UNKNOWN_COMMAND", + "totalErrors": 0 + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "unrecognizable-command": { + "frontend": { + "count": 0, + "code": "UNKNOWN_COMMAND", + "totalErrors": 1 + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + } + }, + "incomplete-expression": { + "frontend": { + "count": 0, + "code": "UNKNOWN_COMMAND", + "totalErrors": 1 + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index fffdce393dd..b1dbcda115a 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -5,11 +5,12 @@ "schedule": "pr", "wiring": { "detector": "dedup-consecutive-unsupported", - "enabled": true, + "enabled": false, "severity": "warning", "runtimeOnly": false, "needsContext": false, "needsExplain": false, + "sourceScoped": false, "appliesTo": { "minVersion": "3.3.0", "engine": "calcite" @@ -25,7 +26,8 @@ } }, "frontendContext": { - "isCalcite": true + "isCalcite": true, + "forceEnable": true }, "index": "opensearch-sql_test_index_account", "queries": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index d2d5df24260..ebfe44389ac 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -5,11 +5,12 @@ "schedule": "pr", "wiring": { "detector": "disabled-join-type", - "enabled": true, + "enabled": false, "severity": "warning", "runtimeOnly": false, "needsContext": false, "needsExplain": false, + "sourceScoped": false, "appliesTo": {} }, "backendFixture": { @@ -23,7 +24,8 @@ } }, "frontendContext": { - "isCalcite": true + "isCalcite": true, + "forceEnable": true }, "index": "opensearch-sql_test_index_account", "queries": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index bc30f5ecb77..88c4226fd8e 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "division-by-zero", "note": "The detector flags both division and modulo by a literal zero because both operations return null silently. The backend result-shape oracle verifies that behavior independently for each operator.", "grammarSurface": "both", @@ -11,7 +11,8 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": {} + "appliesTo": {}, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -51,43 +52,79 @@ "divide-by-zero-literal": { "detectorCount": 1, "severity": "warning", - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "columnAllNull": "ratio" + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } } } }, "divide-by-decimal-zero-literal": { "detectorCount": 1, "severity": "warning", - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "columnAllNull": "ratio" + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } } } }, "divide-by-nonzero-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } }, "modulo-by-zero-literal": { "detectorCount": 1, "severity": "warning", - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "columnAllNull": "m" + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json new file mode 100644 index 00000000000..b9ebc0eef91 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json @@ -0,0 +1,129 @@ +{ + "schemaVersion": 4, + "ruleId": "enabled-false-object", + "channel": "lint", + "note": "The standard Calcite route can still project and filter enabled:false object values from _source. These result-shape oracles pin that observed acceptance; the warning communicates that the object is not indexed/searchable through ordinary OpenSearch field semantics.", + "grammarSurface": "both", + "schedule": "nightly", + "wiring": { + "detector": "enabled-false-object", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "PPL_LINT_DISABLED_OBJECT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "status": "keyword" + }, + "disabledObjectFields": [ + "session" + ] + }, + "index": "opensearch-sql_test_index_ppl_lint_disabled_object", + "queries": { + "disabled-object-field": { + "role": "trigger", + "query": "source={{index}} | fields session.id | head 1" + }, + "disabled-object-filter": { + "role": "trigger", + "query": "source={{index}} | where session.id = 'abc' | fields status" + }, + "indexed-field-control": { + "role": "control", + "query": "source={{index}} | where status = 'ok' | fields status | head 1" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "disabled-object-field": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "not searchable" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + }, + "disabled-object-filter": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "not searchable" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 1 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 1 + } + } + } + }, + "indexed-field-control": { + "frontend": { + "count": 0 + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 5814d837249..6225eaed87b 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "field-validation", "grammarSurface": "both", "schedule": "pr", @@ -10,7 +10,8 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": {} + "appliesTo": {}, + "sourceScoped": true }, "backendFixture": { "indices": [ @@ -63,32 +64,59 @@ "unknown-field-existence": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } } } }, "grok-field-slot-shape-typo": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } } } }, "known-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -100,14 +128,27 @@ "unknown-field-existence": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } @@ -115,25 +156,47 @@ "grok-field-slot-shape-typo": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } }, "known-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -146,14 +209,27 @@ "detectorCount": 1, "severity": "error", "matchMessage": "nonexistent_field", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [nonexistent_field] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } } } } @@ -161,25 +237,47 @@ "grok-field-slot-shape-typo": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [field] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } } } } }, "known-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json index 0ba5c95be35..70006fafbdf 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -10,12 +10,16 @@ "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true). The analytics feature build cannot create flat_object in composite/Parquet storage, so every analytics backend oracle is explicitly non-applicable while the frontend detector assertions still run.", "wiring": { "detector": "flat-object-subfield", - "enabled": true, + "enabled": false, "severity": "error", "runtimeOnly": false, "needsContext": true, "needsExplain": false, - "appliesTo": {} + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.8.0", + "engine": "calcite" + } }, "backendFixture": { "indices": [ @@ -28,6 +32,7 @@ }, "frontendContext": { "isCalcite": true, + "forceEnable": true, "deriveFromMapping": { "name": "keyword", "status": "integer", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 56c272abc9c..cb1d65825d0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -5,11 +5,12 @@ "schedule": "pr", "wiring": { "detector": "head-without-sort", - "enabled": true, + "enabled": false, "severity": "info", "runtimeOnly": false, "needsContext": false, "needsExplain": false, + "sourceScoped": false, "appliesTo": {} }, "backendFixture": { @@ -22,7 +23,8 @@ } }, "frontendContext": { - "isCalcite": true + "isCalcite": true, + "forceEnable": true }, "index": "opensearch-sql_test_index_account", "queries": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index d96142d62e0..4c2cfc36c6f 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "invalid-capture-group-name", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -14,7 +14,10 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": {} + "appliesTo": { + "minVersion": "3.4.0" + }, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -54,14 +57,27 @@ "rex-capture-name-underscore": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } @@ -69,25 +85,47 @@ "rex-capture-name-hyphen": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -100,14 +138,27 @@ "rex-capture-name-underscore": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid capture group name 'user_name'." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } } } } @@ -115,25 +166,47 @@ "rex-capture-name-hyphen": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid capture group name 'user-name'." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } } } } }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index d316c1e8de1..3e83f493e0c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,46 +1,68 @@ { - "schemaVersion": 3, - "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus. EVERY contract now declares `schedule: \"pr\"`, so every contract runs \u2014 and asserts \u2014 on every pull request: neither reader consults `enforced`, so any contract that runs is a hard assertion. The `enforced` / `nonEnforcing` lists below therefore describe oracle QUALITY and review status, not whether a mismatch blocks (design \u00a75.1, \u00a75.2). They are what a reviewer should read when judging how much to trust a red result.", + "schemaVersion": 4, + "description": "Active PPL frontend/backend compatibility corpus for the approved 12 detector rules plus the command-suggestion syntax feature. New contracts remain nightly until their runtime and analytics oracles are reviewed; census drift is report-only until the paired OSD default-alignment change lands.", "contracts": [ - "invalid-capture-group-name.spec.json", - "unsupported-window-function-in-eventstats.spec.json", + "agg-on-text.spec.json", + "command-suggestion.spec.json", "division-by-zero.spec.json", - "head-without-sort.spec.json", - "disabled-join-type.spec.json", + "enabled-false-object.spec.json", "field-validation.spec.json", - "flat-object-subfield.spec.json", - "dedup-consecutive-unsupported.spec.json", + "invalid-capture-group-name.spec.json", "multisearch-min-subsearch.spec.json", + "replace-wildcard-asymmetry.spec.json", + "rex-scan-cost.spec.json", + "type-mismatch-numeric.spec.json", "union-min-datasets.spec.json", - "replace-wildcard-asymmetry.spec.json" + "unsupported-window-function-in-eventstats.spec.json", + "wildcard-source-zero-match.spec.json" + ], + "dormantContracts": [ + "dedup-consecutive-unsupported.spec.json", + "disabled-join-type.spec.json", + "flat-object-subfield.spec.json", + "head-without-sort.spec.json" ], "enforced": [ + "field-validation.spec.json", "invalid-capture-group-name.spec.json", - "unsupported-window-function-in-eventstats.spec.json", "multisearch-min-subsearch.spec.json", + "replace-wildcard-asymmetry.spec.json", "union-min-datasets.spec.json", - "replace-wildcard-asymmetry.spec.json" + "unsupported-window-function-in-eventstats.spec.json" ], "defaultError": [ + "field-validation.spec.json", "invalid-capture-group-name.spec.json", - "unsupported-window-function-in-eventstats.spec.json", "multisearch-min-subsearch.spec.json", - "union-min-datasets.spec.json", "replace-wildcard-asymmetry.spec.json", - "field-validation.spec.json", - "flat-object-subfield.spec.json" + "union-min-datasets.spec.json", + "unsupported-window-function-in-eventstats.spec.json" + ], + "requiredSyntaxFeatures": [ + "command-suggestion.spec.json" + ], + "pendingReview": [ + "agg-on-text.spec.json", + "command-suggestion.spec.json", + "enabled-false-object.spec.json", + "rex-scan-cost.spec.json", + "type-mismatch-numeric.spec.json", + "wildcard-source-zero-match.spec.json" ], - "pendingReview": [], "nonEnforcing": [ + "agg-on-text.spec.json", "division-by-zero.spec.json", - "head-without-sort.spec.json", - "disabled-join-type.spec.json", - "dedup-consecutive-unsupported.spec.json" + "enabled-false-object.spec.json", + "rex-scan-cost.spec.json", + "type-mismatch-numeric.spec.json", + "wildcard-source-zero-match.spec.json" ], "notes": { - "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. The most trustworthy oracles in the corpus \u2014 a mismatch here is almost certainly a real drift.", - "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog \u2014 the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", - "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design \u00a75.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", - "nonEnforcing": "Warning / info / advisory / result-shape rules. Their oracle is weaker than a clean rejection (an advisory rule's query SUCCEEDS, so the contract asserts a result shape or mere acceptance), which makes them likelier to move for reasons unrelated to the lint rule \u2014 dedup-consecutive, for instance, depends on the Calcite-to-v2 fallback staying enabled. They ran nightly-only until every contract moved to the PR schedule so the multi-version rollup sees a full trigger census on each PR; they now block like any other contract, and a red result here warrants checking the oracle before editing a rule." + "enforced": "Reviewed lint error contracts with deterministic backend behavior.", + "defaultError": "Exact approved six-rule detector error census. command-suggestion is an error-channel feature but is intentionally excluded because it is not a detector.", + "requiredSyntaxFeatures": "Syntax-channel features validated through the production runtime grammar listener.", + "pendingReview": "Nightly contracts whose standard and analytics observations must be reviewed before promotion to the required PR schedule.", + "nonEnforcing": "Oracle-quality classification for warning, info, advisory, and result-shape contracts; scheduling determines execution, not this list.", + "dormantContracts": "Preserved default-off detector regression contracts. They do not count toward active shipping coverage and must force-enable their detector when run." } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 345742aa9d8..21cdae63845 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "multisearch-min-subsearch", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -7,7 +7,7 @@ "multisearchCommand", "subSearch" ], - "notes": "Query-initial (no leading pipe) on purpose \u2014 see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", + "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", "wiring": { "detector": "multisearch-min-subsearch", "enabled": true, @@ -17,7 +17,8 @@ "needsExplain": false, "appliesTo": { "minVersion": "3.4.0" - } + }, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -53,14 +54,27 @@ "multisearch-single-subsearch": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "SyntaxCheckException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } } } } @@ -68,25 +82,47 @@ "multisearch-single-subsearch-with-where": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "SyntaxCheckException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } } } } }, "multisearch-two-subsearches-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index c6fa5b162b5..29def9f3b8a 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "replace-wildcard-asymmetry", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -17,7 +17,8 @@ "appliesTo": { "minVersion": "3.4.0", "engine": "calcite" - } + }, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -54,14 +55,27 @@ "replace-wildcard-count-mismatch": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } @@ -69,25 +83,47 @@ "replace-wildcard-count-mismatch-reverse": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } } } }, "replace-symmetric-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -100,14 +136,27 @@ "replace-wildcard-count-mismatch": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } } } } @@ -115,25 +164,47 @@ "replace-wildcard-count-mismatch-reverse": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } } } } }, "replace-symmetric-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json new file mode 100644 index 00000000000..918eaacb0cd --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json @@ -0,0 +1,103 @@ +{ + "schemaVersion": 4, + "ruleId": "rex-scan-cost", + "channel": "lint", + "grammarSurface": "both", + "schedule": "nightly", + "wiring": { + "detector": "rex-scan-cost", + "enabled": true, + "severity": "info", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "email": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "parse-text-field": { + "role": "trigger", + "query": "source={{index}} | parse email '.+@(?.+)' | fields email, host | head 1" + }, + "grok-text-field": { + "role": "trigger", + "query": "source={{index}} | grok email '.+@%{HOSTNAME:grok_host}' | fields email, grok_host | head 1" + }, + "plain-field-control": { + "role": "control", + "query": "source={{index}} | fields email | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "parse-text-field": { + "frontend": { + "count": 1, + "severity": "info", + "matchMessage": "every input row" + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + }, + "grok-text-field": { + "frontend": { + "count": 1, + "severity": "info", + "matchMessage": "every input row" + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + }, + "plain-field-control": { + "frontend": { + "count": 0 + }, + "backends": { + "standard": { + "kind": "advisory", + "httpStatus": 200 + }, + "analytics": { + "kind": "advisory", + "httpStatus": 200 + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json new file mode 100644 index 00000000000..0f3e14791f5 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json @@ -0,0 +1,125 @@ +{ + "schemaVersion": 4, + "ruleId": "type-mismatch-numeric", + "channel": "lint", + "grammarSurface": "both", + "schedule": "nightly", + "wiring": { + "detector": "type-mismatch-numeric", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": true, + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "age": "long" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "numeric-field-string-value": { + "role": "trigger", + "query": "source={{index}} | where age = \"thirty\" | fields age" + }, + "string-value-numeric-field": { + "role": "trigger", + "query": "source={{index}} | where \"thirty\" = age | fields age" + }, + "numeric-string-control": { + "role": "control", + "query": "source={{index}} | where age = \"32\" | fields age | head 1" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "numeric-field-string-value": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "not a number" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + } + } + }, + "string-value-numeric-field": { + "frontend": { + "count": 1, + "severity": "warning", + "matchMessage": "not a number" + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsCount": 0 + } + } + } + }, + "numeric-string-control": { + "frontend": { + "count": 0 + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index d5eadcc25f5..6de9db9a7d6 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "union-min-datasets", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -8,7 +8,7 @@ "unionDataset", "pplCommands" ], - "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' \u2014 a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", "wiring": { "detector": "union-min-datasets", "enabled": true, @@ -19,7 +19,8 @@ "appliesTo": { "minVersion": "3.7.0", "engine": "calcite" - } + }, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -56,14 +57,27 @@ "union-single-dataset": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Union command requires at least two datasets. Provided: 1" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } } } } @@ -71,25 +85,47 @@ "union-single-dataset-with-fields": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Union command requires at least two datasets. Provided: 1" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } } } } }, "union-two-datasets-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 75e45299c0f..fda4d273fb9 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "unsupported-window-function-in-eventstats", "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", "grammarSurface": "both", @@ -13,7 +13,8 @@ "needsExplain": false, "appliesTo": { "minVersion": "3.4.0" - } + }, + "sourceScoped": false }, "backendFixture": { "indices": [ @@ -50,14 +51,27 @@ "eventstats-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 500, - "body": { - "status": 500, - "error": { - "type": "UnsupportedOperationException", - "reason": "There was internal problem at backend" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } } } } @@ -65,25 +79,47 @@ "eventstats-dense-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 500, - "body": { - "status": 500, - "error": { - "type": "UnsupportedOperationException", - "reason": "There was internal problem at backend" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } } } } }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -95,14 +131,27 @@ "eventstats-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 500, - "body": { - "status": 500, - "error": { - "type": "UnsupportedOperationException", - "reason": "Unexpected window function: rank" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } } } } @@ -110,25 +159,47 @@ "eventstats-dense-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 500, - "body": { - "status": 500, - "error": { - "type": "UnsupportedOperationException", - "reason": "Unexpected window function: dense_rank" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: dense_rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: dense_rank" + } } } } }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } @@ -140,14 +211,27 @@ "eventstats-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "CalciteUnsupportedException", - "reason": "Unexpected window function: rank" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } } } } @@ -155,25 +239,47 @@ "eventstats-dense-rank": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "CalciteUnsupportedException", - "reason": "Unexpected window function: dense_rank" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } } } } }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json new file mode 100644 index 00000000000..df01303aefb --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json @@ -0,0 +1,94 @@ +{ + "schemaVersion": 4, + "ruleId": "wildcard-source-zero-match", + "channel": "lint", + "grammarSurface": "both", + "schedule": "nightly", + "wiring": { + "detector": "wildcard-source-zero-match", + "enabled": true, + "severity": "info", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "sourceScoped": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true, + "visibleIndices": [ + "{{index}}" + ] + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "missing-wildcard-source": { + "role": "trigger", + "query": "source={{index}}-definitely-missing-* | head 1" + }, + "matching-wildcard-control": { + "role": "control", + "query": "source={{index}}* | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "missing-wildcard-source": { + "frontend": { + "count": 1, + "severity": "info", + "matchMessage": "matches no known index" + }, + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 404, + "body": { + "status": 404 + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 404, + "body": { + "status": 404 + } + } + } + }, + "matching-wildcard-control": { + "frontend": { + "count": 0 + }, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl_lint_disabled_object.json b/integ-test/src/test/resources/ppl_lint_disabled_object.json new file mode 100644 index 00000000000..30759d7a585 --- /dev/null +++ b/integ-test/src/test/resources/ppl_lint_disabled_object.json @@ -0,0 +1,4 @@ +{"index":{"_id":"1"}} +{"session":{"id":"abc","raw":"not-indexed"},"status":"ok"} +{"index":{"_id":"2"}} +{"session":{"id":"def","raw":"not-indexed"},"status":"error"} diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 79bd5d8f1dc..f79a3052513 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -1,8 +1,8 @@ # PPL lint rule validation -A required, cross-repository GitHub Actions check that proves the OpenSearch -Dashboards (OSD) PPL lint detectors and the SQL backend still agree — on the -**same candidate runtime grammar** built by a SQL pull request. +A cross-repository GitHub Actions check that proves the OpenSearch Dashboards +(OSD) PPL lint detectors and runtime syntax validation still agree with the SQL +backend on the **same candidate runtime grammar** built by a SQL pull request. PPL language behavior lives in SQL; PPL lint detectors live in OSD. A SQL change can silently invalidate an OSD rule (a parser refactor stops a detector matching, @@ -32,10 +32,10 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j 2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a Node code dependency (no OSD server, no Monaco, no browser), then runs [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner - deserializes the candidate bundle through OSD's production headless lint API - (`src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint`) and lints - each query with the **real** detectors on the **candidate** grammar. It then - asserts the detector-vs-backend differential. + deserializes the candidate bundle through OSD's production headless APIs and + runs each query with either the real lint detectors or the shared runtime + syntax listener on the **candidate** grammar. It then asserts the + frontend-vs-backend differential. 3. **validation-result**. `if: always()`, `needs: [backend-validation, detector-validation]`. Fails unless both succeeded — so a skipped detector (because the backend failed first) still reds the check instead of looking @@ -50,11 +50,12 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j | `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | | `schedule` (nightly) | full corpus + coverage | `main` | No | -Every contract declares `schedule: "pr"`, so a PR run exercises the **whole corpus** -— 11 rules, 35 queries. A contract that runs also asserts: neither the IT nor the -detector runner consults the manifest's `enforced` list, so any contract on the PR -schedule can fail the required check. Keep that in mind when adding one; a new -contract whose oracle has not settled should say `schedule: "nightly"` until it has. +The active corpus contains 12 detector contracts plus the +`command-suggestion` syntax contract. Seven reviewed contracts currently declare +`schedule: "pr"`; the six new contracts remain `nightly` until their standard and +analytics observations are reviewed. A contract that runs also asserts: neither +the IT nor the frontend runner consults the manifest's `enforced` list, so any +contract on the PR schedule can fail the required check. `workflow_dispatch` inputs: @@ -65,7 +66,7 @@ contract whose oracle has not settled should say `schedule: "nightly"` until it an immutable commit SHA and recorded in the run manifest. A manual run **cannot** satisfy branch protection; merge the OSD change first, then rerun the required `pull_request` check against OSD `main`. -- `schedule` — `pr` (fast blocking subset) or `nightly` (full corpus). +- `schedule` — `pr` (reviewed blocking contracts) or `nightly` (all active contracts). To validate an OSD change that is not yet merged, push it to a branch on your OSD fork and dispatch with `osd_repo=/OpenSearch-Dashboards` and @@ -124,15 +125,17 @@ writes `detector-report.json`. ## Contract format (schema v3 and v4) -One JSON file per rule under `contracts/`, listed in `manifest.json`. Each file -has a top-level `queries` map (each `{ role: "trigger"|"control", query }`) and a -version-scoped `expectations[]`. Exactly one expectation must match the candidate -backend version (zero or more than one fails before any query runs). +One JSON file per rule or syntax feature under `contracts/`, listed in +`manifest.json`. Each file has `channel: "lint"|"syntax"` (missing defaults to +`lint`), a top-level `queries` map, and version-scoped `expectations[]`. +`suppression-control` is syntax-only: the frontend must retain a raw syntax error +without producing the contracted friendly rewrite. ```jsonc { "schemaVersion": 4, "ruleId": "union-min-datasets", + "channel": "lint", "grammarSurface": "runtime-bundle", "schedule": "pr", "wiring": { "detector": "union-min-datasets", "enabled": true, "severity": "error", ... }, @@ -149,14 +152,14 @@ backend version (zero or more than one fails before any query runs). "engine": "calcite", "queries": { "union-single-dataset": { - "detectorCount": 1, "severity": "error", + "frontend": { "count": 1, "severity": "error" }, "backends": { "standard": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } }, "analytics": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } } }, "union-two-datasets-control": { - "detectorCount": 0, + "frontend": { "count": 0 }, "backends": { "standard": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } }, "analytics": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } @@ -168,6 +171,10 @@ backend version (zero or more than one fails before any query runs). } ``` +Legacy lint expectations using `detectorCount`, `severity`, and `matchMessage` +normalize to the same internal frontend oracle. Syntax expectations use +`frontend.code`, `fixText`, `rawMessage`, and `totalErrors`. + Schema v3's `backend` is read only as `backends.standard`; it is never an implicit analytics oracle. Schema v4's `backends` selects the configured `standard` or `analytics` execution backend. Missing analytics oracles are @@ -201,32 +208,21 @@ rule cannot be validated end to end. `manifest.json` partitions the corpus: -- `enforced` — reviewed error rules with a deterministic backend rejection and a - valid negative control. These block `validation-result` on the single-version - check: `invalid-capture-group-name`, - `unsupported-window-function-in-eventstats`, `multisearch-min-subsearch`, - `union-min-datasets`, `replace-wildcard-asymmetry`. +- `enforced` — the six reviewed detector error contracts with deterministic + backend behavior. - `defaultError` — every rule that ships **enabled at error severity** in OSD's - `rules_catalog.json`. This is the set the **multi-version** check enforces (see - below). It is a superset of `enforced`, adding `field-validation` and - `flat-object-subfield`. -- `pendingReview` — error rules awaiting Peng/Chen usefulness review before - joining `enforced`. Empty: `field-validation` and `flat-object-subfield` are now - pinned across versions by the multi-version check, but stay out of the - single-version `enforced` set because their backend oracle is a semantic - `Field [...] not found.` rejection they share with each other rather than a - rule-unique grammar rejection. -- `nonEnforcing` — warning/info/advisory/result-shape rules. Their oracle is weaker - than a clean rejection: an advisory rule's query *succeeds*, so the contract can - only assert a result shape or plain acceptance, which is likelier to move for - reasons unrelated to the lint rule (`dedup-consecutive` depends on the - Calcite-to-v2 fallback staying on). These ran nightly-only until every contract - moved to the PR schedule, so they now block like any other. A red result here is - worth checking against the oracle before editing a rule. - -The `enforced` / `nonEnforcing` split therefore describes **oracle quality and review -status, not blocking behavior** — it tells a reviewer how much to trust a red result, -not whether one can occur. + `rules_catalog.json`; it contains exactly six detector rules. +- `requiredSyntaxFeatures` — `command-suggestion` only. Syntax features never + appear in `defaultError` or the detector catalog. +- `pendingReview` — the six nightly contracts awaiting oracle review and PR + promotion. +- `nonEnforcing` — oracle-quality classification for warning, info, advisory, + and result-shape contracts. Scheduling determines whether a contract runs. +- `dormantContracts` — four preserved default-off detector contracts. They do + not count as active shipping coverage and explicitly force-enable their rule. + +The `enforced` / `nonEnforcing` split describes **oracle quality and review +status, not blocking behavior**. ## Multi-version validation @@ -412,6 +408,12 @@ different places a developer looks: 2. **The job summary** — the rule × version table plus the full grouped remediation report, which stays the authoritative account. +The required single-version lane follows the same rule: frontend and backend +failures with a `[rule/query]` identity anchor on that contract's `ruleId`. +Shipping-census findings anchor on `manifest.json` (as warnings while census +enforcement is report-only). Artifact and job failures without a trustworthy +repository location remain file-less rather than pointing at a guessed line. + Without the annotations the only thing above the summary is `Process completed with exit code 1`, so the natural next click lands in raw job logs rather than the remediation. Severity is not cosmetic: diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs index 357b046d47c..1209c1b0637 100644 --- a/scripts/ppl-lint/__tests__/annotate.test.mjs +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -8,6 +8,7 @@ import test from 'node:test'; import { buildAnnotations, + buildRequiredAnnotations, contractRepoPath, findExpectationLine, findRuleIdLine, @@ -37,6 +38,22 @@ const CONTRACT = `{ `; const readStub = (text) => () => text; +const REQUIRED_MANIFEST = `{ + "schemaVersion": 4, + "contracts": [ + "invalid-capture-group-name.spec.json" + ], + "defaultError": [ + "invalid-capture-group-name.spec.json" + ] +} +`; +const readRequired = (_dir, file) => + file === 'manifest.json' + ? REQUIRED_MANIFEST + : file === 'invalid-capture-group-name.spec.json' + ? CONTRACT + : undefined; test('anchors on the expectation entry that drifted, not the first one', () => { assert.equal(findExpectationLine(CONTRACT, '>=3.7.0'), 14); @@ -211,6 +228,69 @@ test('an unreadable contract still produces a file-less annotation', () => { assert.equal(annotations[0].file, 'c/gone.spec.json'); }); +test('required-lane failures anchor to their contract and census findings to the manifest', () => { + const annotations = buildRequiredAnnotations( + { + detectorFailures: [ + '[invalid-capture-group-name/trigger] expected 1 diagnostic, got 0', + ], + censusProblems: ['active lint contracts do not equal enabled catalog rules'], + censusEnforced: false, + }, + { + contractsDir: '/w/integ-test/resources/contracts', + workspace: '/w', + readFile: readRequired, + } + ); + + assert.equal(annotations.length, 2); + assert.deepEqual( + { + level: annotations[0].level, + file: annotations[0].file, + line: annotations[0].line, + }, + { + level: 'error', + file: 'integ-test/resources/contracts/invalid-capture-group-name.spec.json', + line: 3, + } + ); + assert.equal(annotations[1].level, 'warning'); + assert.equal(annotations[1].file, 'integ-test/resources/contracts/manifest.json'); + assert.match(annotations[1].message, /REPORT ONLY/); +}); + +test('required artifact row failures recover the rule identity for an inline annotation', () => { + const annotations = buildRequiredAnnotations( + { + artifactErrors: [ + 'backend row invalid-capture-group-name::trigger did not pass its oracle (outcome="fail")', + ], + }, + { + contractsDir: '/w/contracts', + workspace: '/w', + readFile: readRequired, + } + ); + + assert.equal(annotations.length, 1); + assert.equal(annotations[0].file, 'contracts/invalid-capture-group-name.spec.json'); + assert.match(annotations[0].title, /invalid-capture-group-name\/trigger/); +}); + +test('required job failures without a contract identity remain file-less', () => { + const annotations = buildRequiredAnnotations( + { backendResult: 'failure', detectorResult: 'skipped' }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readRequired } + ); + assert.equal(annotations.length, 2); + assert.ok(annotations.every((annotation) => annotation.file === undefined)); +}); + test('a clean report emits nothing', () => { assert.deepEqual(buildAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); + assert.deepEqual(buildRequiredAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); }); diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs index 1c8e1f4b21d..a3ee6f10acd 100644 --- a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -172,3 +172,18 @@ test('detector severity and message mismatches fail the manifest and summary', ( assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); } }); + +test('syntax-specific frontend mismatches fail artifact validation', () => { + for (const field of ['fixMatched', 'rawMessageMatched', 'totalErrorsMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + fs.writeFileSync(file, JSON.stringify(detector)); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not match its (fix|raw-message|total-error) assertion/); + } +}); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs index 1f4516ca83b..914dd31569f 100644 --- a/scripts/ppl-lint/__tests__/contract-schema.test.mjs +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -10,7 +10,10 @@ import { assertContractSchema, assertExactQueryCoverage, classifyBackendReportRow, + contractChannel, indexBackendReport, + normalizeFrontendOracle, + normalizeLintWiring, normalizeTarget, resolveBackendOracle, } from '../contract-schema.mjs'; @@ -489,3 +492,151 @@ test('schema-v2 standard backend rows cannot omit identity', () => { /does not match target "standard"/ ); }); + +test('missing channel remains a backwards-compatible lint contract', () => { + const contract = spec(4, { + detectorCount: 1, + severity: 'warning', + backends: { + standard: { kind: 'advisory', httpStatus: 200 }, + }, + }); + assert.equal(contractChannel(contract), 'lint'); + assert.deepEqual( + normalizeFrontendOracle(contract, contract.expectations[0].queries.trigger), + { + channel: 'lint', + count: 1, + severity: 'warning', + matchMessage: undefined, + } + ); +}); + +test('syntax frontend assertions normalize stable code, fix, raw message, and error census', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + const frontend = normalizeFrontendOracle(contract, { + frontend: { + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'where', + rawMessage: true, + totalErrors: 1, + }, + }); + assert.deepEqual(frontend, { + channel: 'syntax', + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'where', + rawMessage: true, + totalErrors: 1, + }); + assert.equal(assertContractSchema(contract), 4); +}); + +test('lint and syntax frontend fields cannot cross channels', () => { + assert.throws( + () => + normalizeFrontendOracle(spec(4), { + frontend: { count: 1, code: 'UNKNOWN_COMMAND' }, + }), + /code is not valid/ + ); + const syntax = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + assert.throws( + () => normalizeFrontendOracle(syntax, { detectorCount: 1 }), + /must use frontend/ + ); + assert.throws( + () => + assertContractSchema({ + ...syntax, + wiring: { code: 'UNKNOWN_COMMAND', detector: 'command-suggestion' }, + }), + /must contain only/ + ); +}); + +test('suppression-control is syntax-only', () => { + assert.throws( + () => + assertContractSchema({ + ...spec(4), + queries: { + suppressed: { + role: 'suppression-control', + query: 'source=t | zzzzzzzz', + }, + }, + }), + /valid only for syntax/ + ); + assert.doesNotThrow(() => + assertContractSchema({ + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + suppressed: { + role: 'suppression-control', + query: 'source=t | zzzzzzzz', + }, + }, + }) + ); +}); + +test('normalized wiring exposes omitted version, engine, and source scope gates', () => { + const catalog = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + sourceScoped: true, + }); + const omittedVersion = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { engine: 'calcite' }, + sourceScoped: true, + }); + const omittedEngine = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0' }, + sourceScoped: true, + }); + const omittedSourceScope = normalizeLintWiring('example-rule', { + detector: 'example-rule', + enabled: true, + severity: 'warning', + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + }); + + assert.notDeepEqual(omittedVersion, catalog); + assert.notDeepEqual(omittedEngine, catalog); + assert.notDeepEqual(omittedSourceScope, catalog); + assert.equal(omittedSourceScope.sourceScoped, false); +}); diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs index 42539f4529e..4d50c56155d 100644 --- a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -17,9 +17,13 @@ */ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { test } from 'node:test'; import { + findTestFiles, harvestContext, harvestFile, referencedIdentifiers, @@ -36,6 +40,38 @@ const RULES = [ 'division-by-zero', ]; +test('test discovery recurses through current rule locations and excludes generated data', () => { + const osd = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-harvest-')); + const lintRoot = path.join(osd, 'packages/osd-monaco/src/ppl/lint'); + const files = [ + 'rules/inline_rule.test.ts', + 'rules/__tests__/nested_rule.test.tsx', + '__tests__/catalog.test.ts', + 'rules/__fixtures__/fixture.test.ts', + 'rules/__snapshots__/snapshot.test.ts', + 'generated/generated.test.ts', + 'rules/slow.bench.test.ts', + 'rules/manual.verify.test.ts', + ]; + try { + for (const file of files) { + const absolute = path.join(lintRoot, file); + fs.mkdirSync(path.dirname(absolute), { recursive: true }); + fs.writeFileSync(absolute, ''); + } + assert.deepEqual( + findTestFiles(osd).map((file) => path.relative(lintRoot, file)), + [ + '__tests__/catalog.test.ts', + 'rules/__tests__/nested_rule.test.tsx', + 'rules/inline_rule.test.ts', + ] + ); + } finally { + fs.rmSync(osd, { recursive: true, force: true }); + } +}); + // --- attribution ------------------------------------------------------------- test('an exact describe title names its rule', () => { diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 973b1f8728f..a8c29560fb1 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -39,6 +39,7 @@ import { assertExactQueryCoverage, assertExecutionBackend, classifyBackendReportRow, + contractChannel, indexBackendReport, normalizeTarget, resolveBackendOracle, @@ -214,6 +215,22 @@ function normalizeDetectorReport(detector, target) { if (!Array.isArray(detector.defaultErrorRules)) { throw new TypeError('detector report.defaultErrorRules must be a JSON array'); } + for (const field of ['enabledRules', 'requiredSyntaxFeatures', 'activeContractRules']) { + if (detector[field] === undefined) continue; + if (!Array.isArray(detector[field])) { + throw new TypeError(`detector report.${field} must be a JSON array`); + } + const values = new Set(); + for (const value of detector[field]) { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`detector report.${field} entries must be non-empty strings`); + } + if (values.has(value)) { + throw new Error(`detector report.${field} contains duplicate rule "${value}"`); + } + values.add(value); + } + } const census = new Set(); for (const ruleId of detector.defaultErrorRules) { if (typeof ruleId !== 'string' || ruleId.length === 0) { @@ -301,7 +318,14 @@ function reportItemKey(item, kind) { function loadContracts(dir) { const manifest = readJson(path.join(dir, 'manifest.json')); const specs = new Map(); - for (const name of manifest.contracts || []) { + const contractNames = manifest.contracts || []; + if (!Array.isArray(contractNames)) { + fatal(`${path.join(dir, 'manifest.json')} contracts must be an array`); + } + if (new Set(contractNames).size !== contractNames.length) { + fatal(`${path.join(dir, 'manifest.json')} contracts contains duplicate file names`); + } + for (const name of contractNames) { const spec = readJson(path.join(dir, name)); try { assertContractSchema(spec); @@ -318,12 +342,36 @@ function loadContracts(dir) { } catch (error) { artifactFatal(path.join(dir, name), error); } + if (specs.has(spec.ruleId)) { + fatal(`contract manifest contains duplicate ruleId "${spec.ruleId}"`); + } specs.set(spec.ruleId, { spec, file: name }); } // `defaultError` is the multi-version enforced set: every rule that ships // enabled at error severity. Fall back to `enforced` for older manifests so // this script still runs against an un-migrated corpus. const enforcedFiles = new Set(manifest.defaultError || manifest.enforced || []); + for (const file of enforcedFiles) { + if (!contractNames.includes(file)) { + fatal(`manifest.defaultError references inactive or missing contract "${file}"`); + } + } + const requiredSyntaxFiles = manifest.requiredSyntaxFeatures || []; + if (!Array.isArray(requiredSyntaxFiles)) { + fatal('manifest.requiredSyntaxFeatures must be an array'); + } + if (new Set(requiredSyntaxFiles).size !== requiredSyntaxFiles.length) { + fatal('manifest.requiredSyntaxFeatures contains duplicate file names'); + } + for (const file of requiredSyntaxFiles) { + const entry = [...specs.values()].find((candidate) => candidate.file === file); + if (!entry) { + fatal(`manifest.requiredSyntaxFeatures references inactive or missing contract "${file}"`); + } + if (contractChannel(entry.spec) !== 'syntax') { + fatal(`manifest.requiredSyntaxFeatures entry "${file}" is not a syntax contract`); + } + } const enforcedRules = new Set(); for (const [ruleId, { file }] of specs) { if (enforcedFiles.has(file)) enforcedRules.add(ruleId); @@ -515,6 +563,7 @@ function pairBackendLegs(legs) { function detectorParityValue(entry) { return { + channel: entry.channel || 'lint', role: entry.role || 'trigger', query: entry.query || '', expected: entry.expected, @@ -524,6 +573,14 @@ function detectorParityValue(entry) { typeof entry.severityMatched === 'boolean' ? entry.severityMatched : undefined, messageMatched: typeof entry.messageMatched === 'boolean' ? entry.messageMatched : undefined, + fixMatched: typeof entry.fixMatched === 'boolean' ? entry.fixMatched : undefined, + rawMessageMatched: + typeof entry.rawMessageMatched === 'boolean' ? entry.rawMessageMatched : undefined, + totalErrorsMatched: + typeof entry.totalErrorsMatched === 'boolean' ? entry.totalErrorsMatched : undefined, + code: entry.code, + codes: entry.codes, + totalErrors: entry.totalErrors, }; } @@ -660,9 +717,109 @@ function auditDefaultErrorCensus(legs, specs, enforcedRules) { }); } } + for (const ruleId of [...enforcedRules].sort()) { + if (!census.has(ruleId)) { + missing.push({ + ruleId, + reason: 'listed under manifest.defaultError but not enabled at error severity in OSD', + }); + } + } return missing; } +function setsEqual(left, right) { + return left.size === right.size && [...left].every((value) => right.has(value)); +} + +function auditShippingCensus(legs, specs, manifest) { + const reports = legs + .map((leg) => leg.detector) + .filter( + (detector) => + Array.isArray(detector.enabledRules) && + Array.isArray(detector.activeContractRules) && + Array.isArray(detector.requiredSyntaxFeatures) + ); + if (reports.length === 0) { + return { + available: false, + enforced: false, + passed: false, + problems: [ + 'detector reports predate the active shipping census; rerun with the channel-aware frontend runner', + ], + }; + } + + const activeRules = new Set(specs.keys()); + const activeLintRules = new Set( + [...specs.entries()] + .filter(([, { spec }]) => contractChannel(spec) === 'lint') + .map(([ruleId]) => ruleId) + ); + const activeSyntaxRules = new Set( + [...specs.entries()] + .filter(([, { spec }]) => contractChannel(spec) === 'syntax') + .map(([ruleId]) => ruleId) + ); + const requiredSyntaxRules = new Set( + (manifest.requiredSyntaxFeatures || []) + .map((file) => [...specs.entries()].find(([, entry]) => entry.file === file)) + .filter(Boolean) + .map(([ruleId]) => ruleId) + ); + const enabledRules = new Set(reports.flatMap((report) => report.enabledRules)); + const reportedActiveRules = new Set( + reports.flatMap((report) => report.activeContractRules) + ); + const reportedSyntaxRules = new Set( + reports.flatMap((report) => report.requiredSyntaxFeatures) + ); + const problems = []; + + if (activeLintRules.size !== 12) { + problems.push(`expected 12 active lint contracts, found ${activeLintRules.size}`); + } + if (requiredSyntaxRules.size !== 1) { + problems.push(`expected one required syntax feature, found ${requiredSyntaxRules.size}`); + } + if (activeRules.size !== 13) { + problems.push(`expected 13 active contracts, found ${activeRules.size}`); + } + if (!setsEqual(activeLintRules, enabledRules)) { + problems.push( + `active lint rules ${JSON.stringify([...activeLintRules].sort())} do not equal enabled OSD ` + + `rules ${JSON.stringify([...enabledRules].sort())}` + ); + } + if (!setsEqual(activeSyntaxRules, requiredSyntaxRules)) { + problems.push( + `active syntax rules ${JSON.stringify([...activeSyntaxRules].sort())} do not equal manifest ` + + `required syntax features ${JSON.stringify([...requiredSyntaxRules].sort())}` + ); + } + if (!setsEqual(activeRules, reportedActiveRules)) { + problems.push('detector report activeContractRules does not match this manifest'); + } + if (!setsEqual(requiredSyntaxRules, reportedSyntaxRules)) { + problems.push('detector report requiredSyntaxFeatures does not match this manifest'); + } + + return { + available: true, + enforced: reports.some( + (report) => report.census && report.census.enforced === true + ), + enabledRules: [...enabledRules].sort(), + activeContractRules: [...activeRules].sort(), + activeLintRules: [...activeLintRules].sort(), + requiredSyntaxFeatures: [...requiredSyntaxRules].sort(), + passed: problems.length === 0, + problems, + }; +} + /** * Read one backend report entry into an observation, distinguishing "the engine * accepted this" from "we never got an answer". @@ -877,6 +1034,16 @@ function main() { // the census each detector leg recorded from the OSD catalog it linted with, so // a new default-error rule cannot land unvalidated. const missingContracts = auditDefaultErrorCensus(legs, specs, enforcedRules); + const shippingCensus = auditShippingCensus(legs, specs, manifest); + const blockCensusDrift = !shippingCensus.available || shippingCensus.enforced; + for (const entry of missingContracts) { + entry.blocking = blockCensusDrift; + } + if (!shippingCensus.passed) { + for (const problem of shippingCensus.problems) { + log(`CENSUS REPORT-ONLY: ${problem}`); + } + } for (const [ruleId, { spec, file }] of specs) { const isEnforced = enforcedRules.has(ruleId); @@ -1366,6 +1533,26 @@ function main() { } } + for (const collection of [drifts, coverageHoles, inconclusive, notApplicable, matrix]) { + for (const entry of collection) { + const contract = specs.get(entry.ruleId); + entry.channel = contract ? contractChannel(contract.spec) : 'lint'; + } + } + for (const drift of drifts) { + if (drift.channel !== 'syntax') continue; + drift.remediation = { + action: 'review-syntax-validation', + target: + 'OSD runtime_validation_core and the syntax contract expectation', + detail: + `Reproduce "${drift.ruleId}" with the candidate runtime grammar. Update the shared OSD ` + + `parser/listener core if UNKNOWN_COMMAND identity, suppression, or quick-fix behavior ` + + `regressed; update this contract only after confirming an intentional syntax UX change. ` + + `Do not change detector catalog appliesTo metadata for a syntax-channel failure.`, + }; + } + const isObservedAnalyticsFinding = (entry) => args.observeAnalytics && (entry.executionBackend === 'analytics' || @@ -1375,10 +1562,12 @@ function main() { entry.driftClass === DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH || entry.kind === 'backend-oracle'); for (const drift of drifts) { - drift.blocking = !!drift.enforced && !isObservedAnalyticsFinding(drift); + drift.blocking = + (!!drift.enforced || args.allRules) && !isObservedAnalyticsFinding(drift); } for (const hole of coverageHoles) { - hole.blocking = !!hole.enforced && !isObservedAnalyticsFinding(hole); + hole.blocking = + (!!hole.enforced || args.allRules) && !isObservedAnalyticsFinding(hole); } const enforcedDrifts = drifts.filter((d) => d.blocking); const enforcedHoles = coverageHoles.filter((h) => h.blocking); @@ -1387,6 +1576,7 @@ function main() { // verdict is an infrastructure failure for every rule we asked the run to // observe. const enforcedInconclusive = inconclusive.filter((i) => i.enforced || args.allRules); + const blockingMissingContracts = missingContracts.filter((entry) => entry.blocking); for (const row of matrix) { row.key = reportItemKey(row, 'matrix'); } @@ -1421,6 +1611,7 @@ function main() { })), enforcedRules: [...enforcedRules].sort(), missingContracts, + shippingCensus, manifestDescription: manifest.description || '', matrix, drifts, @@ -1435,13 +1626,14 @@ function main() { drifts.filter((d) => d.enforced && !d.blocking).length + coverageHoles.filter((h) => h.enforced && !h.blocking).length, missingContractCount: missingContracts.length, + blockingMissingContractCount: blockingMissingContracts.length, enforcedInconclusive: enforcedInconclusive.length, // An inconclusive default-error rule fails too: "we could not check" must // never render as "it is fine". passed: enforcedDrifts.length === 0 && enforcedHoles.length === 0 && - missingContracts.length === 0 && + blockingMissingContracts.length === 0 && enforcedInconclusive.length === 0, }, }; @@ -1475,13 +1667,14 @@ function main() { // eslint-disable-next-line no-console console.error( `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + - `${enforcedHoles.length} coverage hole(s), ${missingContracts.length} unvalidated ` + + `${enforcedHoles.length} coverage hole(s), ${blockingMissingContracts.length} unvalidated ` + `default-error rule(s) and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` ); process.exit(1); } log( - `PASS: every default-error rule agrees with all ${legs.length} engine version(s)` + + `PASS: every ${args.allRules ? 'active shipping' : 'default-error'} rule agrees with all ` + + `${legs.length} engine version(s)` + (drifts.length > 0 ? ` (${drifts.length} non-enforced finding(s) reported)` : '') + '.' ); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs index 422177ce204..9c9d6bf2c16 100644 --- a/scripts/ppl-lint/annotate.mjs +++ b/scripts/ppl-lint/annotate.mjs @@ -4,7 +4,7 @@ */ /** - * GitHub Actions annotations for the PPL lint multi-version check. + * GitHub Actions annotations for the PPL lint required and multi-version checks. * * The drift report and the job summary already say exactly what to change. The * problem is WHERE a developer looks first: GitHub renders workflow-command @@ -88,6 +88,16 @@ export function findRuleIdLine(contractText) { return undefined; } +function findJsonKeyLine(jsonText, key) { + if (!jsonText || !key) return undefined; + const lines = jsonText.split('\n'); + const pattern = new RegExp(`^\\s*${JSON.stringify(key)}\\s*:`); + for (let i = 0; i < lines.length; i++) { + if (pattern.test(lines[i])) return i + 1; + } + return undefined; +} + /** * Repo-relative path of a contract file, for `file=`. * @@ -189,7 +199,7 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r for (const missing of report.missingContracts || []) { const ruleId = missing.ruleId || missing; annotations.push({ - level: 'error', + level: missing.blocking === false ? 'warning' : 'error', // A rule with no contract has no file to point at; the manifest is where the // reader's edit goes. file: undefined, @@ -198,13 +208,125 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r `"${ruleId}" ships enabled at error severity in OSD's rules_catalog.json but ` + `${missing.reason || 'has no contract in this corpus'}. A default-error rule with no ` + `contract is invisible to this check. Add a contract file and list it under ` + - `manifest.defaultError, or lower the rule's severity in OSD.`, + `manifest.defaultError, or lower the rule's severity in OSD.` + + (missing.blocking === false + ? ' This compatibility phase reports the census mismatch without blocking until the paired OSD default-alignment change lands.' + : ''), }); } return annotations; } +function ruleIdentity(message) { + const bracketed = String(message).match(/^\[([A-Za-z0-9._-]+)(?:\/([A-Za-z0-9._-]+))?\]/); + if (bracketed && !['census', 'contracts', 'grammar-export', 'report'].includes(bracketed[1])) { + return { ruleId: bracketed[1], queryName: bracketed[2] }; + } + const row = String(message).match( + /\b(?:backend|detector) row ([A-Za-z0-9._-]+)::([A-Za-z0-9._-]+)\b/ + ); + return row ? { ruleId: row[1], queryName: row[2] } : {}; +} + +function loadContractFiles(contractsDir, readFile) { + const files = new Map(); + const manifestText = readFile(contractsDir, 'manifest.json'); + if (!manifestText) return { files, manifestText }; + try { + const manifest = JSON.parse(manifestText); + const names = [...(manifest.contracts || []), ...(manifest.dormantContracts || [])]; + for (const name of names) { + const text = readFile(contractsDir, name); + if (!text) continue; + try { + const spec = JSON.parse(text); + if (spec.ruleId) files.set(spec.ruleId, { name, text }); + } catch { + // Malformed contracts are reported by the schema/runner. Keep this helper + // best-effort so annotation generation never hides the original failure. + } + } + } catch { + // The manifest parse failure is itself annotated below without a line anchor. + } + return { files, manifestText }; +} + +function manifestKeyFor(message) { + if (/defaultError/.test(message)) return 'defaultError'; + if (/requiredSyntaxFeatures/.test(message)) return 'requiredSyntaxFeatures'; + if (/dormantContracts/.test(message)) return 'dormantContracts'; + return 'contracts'; +} + +/** + * Build annotations for the required single-version lane. + * + * Detector failures use their `[rule/query]` prefix to land on the owning + * contract. Census findings land on manifest.json. Job and artifact failures + * without a trustworthy repository location remain file-less. + */ +export function buildRequiredAnnotations( + report, + { contractsDir, workspace, readFile = readContract } = {} +) { + const annotations = []; + const { files, manifestText } = loadContractFiles(contractsDir, readFile); + const seen = new Set(); + + const addFailure = (message, source) => { + const text = String(message); + const { ruleId, queryName } = ruleIdentity(text); + const contract = ruleId ? files.get(ruleId) : undefined; + const census = source === 'census' || /^\[census\]/.test(text); + const file = census + ? contractRepoPath(contractsDir, 'manifest.json', workspace) + : contract + ? contractRepoPath(contractsDir, contract.name, workspace) + : undefined; + const line = census + ? findJsonKeyLine(manifestText, manifestKeyFor(text)) + : findRuleIdLine(contract?.text); + const level = census && report.censusEnforced !== true ? 'warning' : 'error'; + const title = census + ? 'PPL lint shipping census mismatch' + : ruleId + ? `PPL lint required validation: ${ruleId}${queryName ? `/${queryName}` : ''}` + : `PPL lint required validation: ${source}`; + const key = `${level}\0${file || ''}\0${line || ''}\0${title}\0${text}`; + if (seen.has(key)) return; + seen.add(key); + annotations.push({ + level, + file, + line, + title, + message: + census && report.censusEnforced !== true + ? `${text}\nREPORT ONLY: align the active SQL manifest with the approved OSD shipping catalog before enabling census enforcement.` + : text, + }); + }; + + for (const message of report.detectorFailures || []) addFailure(message, 'frontend'); + for (const message of report.artifactErrors || []) addFailure(message, 'artifact'); + for (const message of report.censusProblems || []) addFailure(message, 'census'); + + for (const [job, result] of [ + ['backend-validation', report.backendResult], + ['detector-validation', report.detectorResult], + ]) { + if (result && result !== 'success') { + addFailure( + `${job} finished with result "${result}". See that job's logs and uploaded artifacts for the underlying failure.`, + job + ); + } + } + return annotations; +} + function readContract(contractsDir, fileName) { try { return fs.readFileSync(path.join(contractsDir, fileName), 'utf8'); @@ -238,3 +360,15 @@ export function emitAnnotations(report, options = {}) { } return annotations; } + +/** Emit required-lane annotations under the same Actions-only policy. */ +export function emitRequiredAnnotations(report, options = {}) { + const enabled = options.force || process.env.GITHUB_ACTIONS === 'true'; + if (!enabled) return []; + const annotations = buildRequiredAnnotations(report, options); + for (const annotation of annotations) { + // eslint-disable-next-line no-console + console.log(formatAnnotation(annotation)); + } + return annotations; +} diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index faf21f078b1..4d0cd183503 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -27,8 +27,10 @@ import { indexBackendReport, normalizeTarget, } from './contract-schema.mjs'; +import { emitRequiredAnnotations } from './annotate.mjs'; const ARTIFACTS = 'artifacts'; +const CONTRACTS = path.resolve('integ-test/src/test/resources/ppl-lint/contracts'); function readJson(file, errors) { try { @@ -147,6 +149,15 @@ function main() { if (entry.messageMatched !== true) { artifactErrors.push(`detector row ${key} did not match its message assertion`); } + for (const [field, label] of [ + ['fixMatched', 'fix'], + ['rawMessageMatched', 'raw-message'], + ['totalErrorsMatched', 'total-error'], + ]) { + if (entry[field] === false) { + artifactErrors.push(`detector row ${key} did not match its ${label} assertion`); + } + } } for (const [key, entry] of backendByKey) { if (!detectorKeys.has(key)) { @@ -213,6 +224,21 @@ function main() { writeSummary(manifest, detector, backend); + emitRequiredAnnotations( + { + artifactErrors, + detectorFailures: Array.isArray(detector.failures) ? detector.failures : [], + censusProblems: Array.isArray(detector.census?.problems) ? detector.census.problems : [], + censusEnforced: detector.census?.enforced === true, + backendResult, + detectorResult, + }, + { + contractsDir: CONTRACTS, + workspace: process.env.GITHUB_WORKSPACE || process.cwd(), + } + ); + if (artifactErrors.length > 0) { throw new Error(`invalid PPL lint artifacts:\n- ${artifactErrors.join('\n- ')}`); } diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs index 291f2f9c405..ca3c5a809c2 100644 --- a/scripts/ppl-lint/contract-schema.mjs +++ b/scripts/ppl-lint/contract-schema.mjs @@ -6,6 +6,17 @@ const EXECUTION_BACKENDS = new Set(['standard', 'analytics']); const CONTRACT_SCHEMA_VERSIONS = new Set([3, 4]); const APPLICABLE_BACKEND_KINDS = new Set(['rejection', 'result-shape', 'advisory']); +const CONTRACT_CHANNELS = new Set(['lint', 'syntax']); +const QUERY_ROLES = new Set(['trigger', 'control', 'suppression-control']); +const LINT_FRONTEND_FIELDS = new Set(['count', 'severity', 'matchMessage']); +const SYNTAX_FRONTEND_FIELDS = new Set([ + 'count', + 'code', + 'fixText', + 'matchMessage', + 'rawMessage', + 'totalErrors', +]); function isObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); @@ -42,6 +53,164 @@ function assertOptionalString(value, label) { } } +function assertKnownKeys(value, allowed, label) { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) { + throw new Error(`${label}.${key} is not valid for this contract channel.`); + } + } +} + +export function contractChannel(spec) { + requireObject(spec, 'contract'); + const channel = spec.channel === undefined ? 'lint' : spec.channel; + if (!CONTRACT_CHANNELS.has(channel)) { + throw new Error( + `contract.channel must be "lint" or "syntax", got ${describe(channel)}.` + ); + } + return channel; +} + +/** + * Normalize legacy detector fields and channel-specific frontend assertions. + * + * Callers continue to receive `count`, `severity`, and `matchMessage` for lint + * contracts while syntax contracts can assert stable parser error identity, + * quick-fix text, raw-message preservation, and the total syntax error census. + */ +export function normalizeFrontendOracle(spec, queryExpectation) { + const channel = contractChannel(spec); + requireObject(queryExpectation, `[${spec.ruleId}] query expectation`); + + const hasLegacy = Object.prototype.hasOwnProperty.call( + queryExpectation, + 'detectorCount' + ); + const hasFrontend = Object.prototype.hasOwnProperty.call( + queryExpectation, + 'frontend' + ); + if (hasLegacy && hasFrontend) { + throw new Error( + `[${spec.ruleId}] query expectation must use either detectorCount or frontend, not both.` + ); + } + + if (channel === 'lint') { + const frontend = hasFrontend + ? requireObject(queryExpectation.frontend, `[${spec.ruleId}] frontend`) + : { + count: queryExpectation.detectorCount, + severity: queryExpectation.severity, + matchMessage: queryExpectation.matchMessage, + }; + assertKnownKeys(frontend, LINT_FRONTEND_FIELDS, `[${spec.ruleId}] frontend`); + requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); + assertOptionalString(frontend.severity, `[${spec.ruleId}] frontend.severity`); + if ( + frontend.matchMessage !== undefined && + typeof frontend.matchMessage !== 'string' + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + ); + } + if ( + hasFrontend && + (queryExpectation.severity !== undefined || + queryExpectation.matchMessage !== undefined) + ) { + throw new Error( + `[${spec.ruleId}] severity and matchMessage must be nested under frontend when frontend is present.` + ); + } + return { + channel, + count: frontend.count, + severity: frontend.severity, + matchMessage: frontend.matchMessage, + }; + } + + if (hasLegacy) { + throw new Error( + `[${spec.ruleId}] syntax contracts must use frontend instead of detectorCount.` + ); + } + const frontend = requireObject( + queryExpectation.frontend, + `[${spec.ruleId}] frontend` + ); + assertKnownKeys(frontend, SYNTAX_FRONTEND_FIELDS, `[${spec.ruleId}] frontend`); + requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); + requireNonEmptyString(frontend.code, `[${spec.ruleId}] frontend.code`); + assertOptionalString(frontend.fixText, `[${spec.ruleId}] frontend.fixText`); + if ( + frontend.matchMessage !== undefined && + typeof frontend.matchMessage !== 'string' + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + ); + } + if (frontend.rawMessage !== undefined && typeof frontend.rawMessage !== 'boolean') { + throw new TypeError(`[${spec.ruleId}] frontend.rawMessage must be a boolean.`); + } + if (frontend.totalErrors !== undefined) { + requireNonNegativeInteger( + frontend.totalErrors, + `[${spec.ruleId}] frontend.totalErrors` + ); + } + for (const field of ['severity', 'matchMessage']) { + if (Object.prototype.hasOwnProperty.call(queryExpectation, field)) { + throw new Error( + `[${spec.ruleId}] syntax ${field} must be nested under frontend.` + ); + } + } + return { channel, ...frontend }; +} + +export function normalizeLintWiring(ruleId, wiring, label = 'wiring') { + requireNonEmptyString(ruleId, `${label}.id`); + requireObject(wiring, label); + const appliesTo = + wiring.appliesTo === undefined ? {} : requireObject(wiring.appliesTo, `${label}.appliesTo`); + const normalizedAppliesTo = {}; + for (const key of ['minVersion', 'maxVersion', 'engine']) { + assertOptionalString(appliesTo[key], `${label}.appliesTo.${key}`); + if (appliesTo[key] !== undefined) { + normalizedAppliesTo[key] = appliesTo[key]; + } + } + for (const key of [ + 'runtimeOnly', + 'needsContext', + 'needsExplain', + 'sourceScoped', + ]) { + if (wiring[key] !== undefined && typeof wiring[key] !== 'boolean') { + throw new TypeError(`${label}.${key} must be a boolean when present.`); + } + } + if (typeof wiring.enabled !== 'boolean') { + throw new TypeError(`${label}.enabled must be a boolean.`); + } + return { + id: ruleId, + detector: requireNonEmptyString(wiring.detector, `${label}.detector`), + enabled: wiring.enabled, + severity: requireNonEmptyString(wiring.severity, `${label}.severity`), + appliesTo: normalizedAppliesTo, + runtimeOnly: wiring.runtimeOnly === true, + needsContext: wiring.needsContext === true, + needsExplain: wiring.needsExplain === true, + sourceScoped: wiring.sourceScoped === true, + }; +} + function assertBackendOracle(oracle, ruleId, executionBackend) { const label = `[${ruleId}] ${executionBackend} backend oracle`; requireObject(oracle, label); @@ -206,6 +375,38 @@ export function assertContractSchema(spec) { ); } requireNonEmptyString(spec.ruleId, 'contract.ruleId'); + const channel = contractChannel(spec); + if (spec.wiring !== undefined) { + const wiring = requireObject(spec.wiring, `[${spec.ruleId}] contract.wiring`); + if (channel === 'syntax') { + const keys = Object.keys(wiring); + if (keys.length !== 1 || keys[0] !== 'code') { + throw new Error( + `[${spec.ruleId}] syntax wiring must contain only the stable error code.` + ); + } + requireNonEmptyString(wiring.code, `[${spec.ruleId}] contract.wiring.code`); + } else if (Object.prototype.hasOwnProperty.call(wiring, 'code')) { + throw new Error(`[${spec.ruleId}] lint wiring must not contain syntax code.`); + } + } + if (spec.queries !== undefined) { + requireObject(spec.queries, `[${spec.ruleId}] contract.queries`); + for (const [queryName, query] of Object.entries(spec.queries)) { + requireObject(query, `[${spec.ruleId}] contract.queries.${queryName}`); + const role = query.role === undefined ? 'trigger' : query.role; + if (!QUERY_ROLES.has(role)) { + throw new Error( + `[${spec.ruleId}] query "${queryName}" has invalid role ${describe(role)}.` + ); + } + if (role === 'suppression-control' && channel !== 'syntax') { + throw new Error( + `[${spec.ruleId}] suppression-control is valid only for syntax contracts.` + ); + } + } + } return spec.schemaVersion; } @@ -251,32 +452,11 @@ export function assertExactQueryCoverage(spec, expectation) { export function resolveBackendOracle(spec, queryExpectation, executionBackend) { const schemaVersion = assertContractSchema(spec); assertExecutionBackend(executionBackend); - requireObject(queryExpectation, `[${spec.ruleId}] query expectation`); - - requireNonNegativeInteger( - queryExpectation.detectorCount, - `[${spec.ruleId}] detectorCount` - ); - if ( - Object.prototype.hasOwnProperty.call(queryExpectation, 'severity') && - (typeof queryExpectation.severity !== 'string' || - queryExpectation.severity.length === 0) - ) { - throw new TypeError( - `[${spec.ruleId}] severity must be a non-empty string when present.` - ); - } - if ( - Object.prototype.hasOwnProperty.call(queryExpectation, 'matchMessage') && - typeof queryExpectation.matchMessage !== 'string' - ) { - throw new TypeError(`[${spec.ruleId}] matchMessage must be a string when present.`); - } - + const frontend = normalizeFrontendOracle(spec, queryExpectation); const detector = { - count: queryExpectation.detectorCount, - severity: queryExpectation.severity, - matchMessage: queryExpectation.matchMessage, + count: frontend.count, + severity: frontend.severity, + matchMessage: frontend.matchMessage, }; let oracle; @@ -308,6 +488,7 @@ export function resolveBackendOracle(spec, queryExpectation, executionBackend) { status: 'coverage-missing', executionBackend, detector, + frontend, oracle: undefined, reason: missingReason, }; @@ -319,6 +500,7 @@ export function resolveBackendOracle(spec, queryExpectation, executionBackend) { status: 'not-applicable', executionBackend, detector, + frontend, oracle, reason: oracle.reason, }; @@ -328,6 +510,7 @@ export function resolveBackendOracle(spec, queryExpectation, executionBackend) { status: 'applicable', executionBackend, detector, + frontend, oracle, reason: undefined, }; diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs index 82023a8b187..4a19a6b9b50 100644 --- a/scripts/ppl-lint/harvest-queries.mjs +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -66,18 +66,7 @@ function fatal(message) { process.exit(2); } -/** - * Directories under an OSD checkout that hold PPL lint tests. Kept explicit - * rather than globbing the whole repo: a wide sweep would pull in queries from - * autocomplete/highlighting suites that were never written as lint trigger or - * control cases, and a query harvested from the wrong intent produces a - * "disagreement" that is really just a query nobody claimed anything about. - */ -const LINT_TEST_DIRS = [ - 'packages/osd-monaco/src/ppl/lint/__tests__', - 'packages/osd-monaco/src/ppl/lint/hover/__tests__', - 'packages/osd-monaco/src/ppl/lint/explain/__tests__', -]; +const LINT_TEST_ROOT = 'packages/osd-monaco/src/ppl/lint'; /** * Benchmarks and repro captures are excluded. Bench files hold deliberately @@ -85,6 +74,13 @@ const LINT_TEST_DIRS = [ * they would dominate the corpus with near-duplicates. */ const EXCLUDED_FILE_PATTERNS = [/\.bench\.test\.ts$/, /\.verify\.test\.ts$/]; +const EXCLUDED_DIRS = new Set([ + '__fixtures__', + '__snapshots__', + 'fixtures', + 'generated', + 'target', +]); function parseArgs(argv) { const args = { @@ -142,18 +138,25 @@ function readRuleList(value) { .filter((s) => s && !s.startsWith('#')); } -/** Every lint test file under the OSD checkout, excluding benches. */ -function findTestFiles(osdRoot) { +/** Every lint test file under the lint package, excluding generated test data. */ +export function findTestFiles(osdRoot) { const files = []; - for (const dir of LINT_TEST_DIRS) { - const abs = path.join(osdRoot, dir); - if (!fs.existsSync(abs)) continue; - for (const name of fs.readdirSync(abs)) { + const root = path.join(osdRoot, LINT_TEST_ROOT); + const visit = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (!EXCLUDED_DIRS.has(entry.name)) { + visit(path.join(dir, entry.name)); + } + continue; + } + const name = entry.name; if (!name.endsWith('.test.ts') && !name.endsWith('.test.tsx')) continue; if (EXCLUDED_FILE_PATTERNS.some((re) => re.test(name))) continue; - files.push(path.join(abs, name)); + files.push(path.join(dir, name)); } - } + }; + if (fs.existsSync(root)) visit(root); return files.sort(); } @@ -621,6 +624,20 @@ function main() { 'detector output; there are no pinned expectations and this corpus must never fail a build.', index: args.index || null, sourceFiles: files.map((f) => path.relative(args.osd, f)), + ruleCoverage: args.catalogRules + .map((ruleId) => { + const entries = kept.filter((entry) => entry.ruleId === ruleId); + return { + ruleId, + filesScanned: [ + ...new Set(entries.map((entry) => entry.source.split(':')[0])), + ].sort(), + ownedQueryCount: entries.length, + explicitException: null, + }; + }) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + exceptions: [], queries: kept, unowned: args.keepUnowned ? unowned : [], stats: { diff --git a/scripts/ppl-lint/label-discovery.mjs b/scripts/ppl-lint/label-discovery.mjs index 3ee6c318860..487975c2fa3 100644 --- a/scripts/ppl-lint/label-discovery.mjs +++ b/scripts/ppl-lint/label-discovery.mjs @@ -306,11 +306,29 @@ function main() { const findings = labelled.filter((l) => l.finding); const byRule = new Map(); + for (const coverage of corpus.ruleCoverage || []) { + byRule.set(coverage.ruleId, { + triggers: [], + controls: [], + unknown: [], + suppressed: 0, + files: new Set(coverage.filesScanned || []), + explicitException: coverage.explicitException || null, + }); + } for (const row of labelled) { if (!byRule.has(row.ruleId)) { - byRule.set(row.ruleId, { triggers: [], controls: [], unknown: [], suppressed: 0 }); + byRule.set(row.ruleId, { + triggers: [], + controls: [], + unknown: [], + suppressed: 0, + files: new Set(), + explicitException: null, + }); } const bucket = byRule.get(row.ruleId); + if (row.source) bucket.files.add(row.source.split(':')[0]); if (row.suppressed) bucket.suppressed++; if (row.role === ROLES.TRIGGER) bucket.triggers.push(row.queryName); else if (row.role === ROLES.CONTROL) bucket.controls.push(row.queryName); @@ -347,10 +365,16 @@ function main() { triggerCoverage: [...byRule] .map(([ruleId, b]) => ({ ruleId, + filesScanned: [...b.files].sort(), triggers: b.triggers.length, controls: b.controls.length, unknown: b.unknown.length, suppressed: b.suppressed, + unattributedQueryCount: (corpus.stats && corpus.stats.unowned) || 0, + explicitException: b.explicitException, + coverageSatisfied: + b.triggers.length + b.controls.length + b.unknown.length > 0 || + !!b.explicitException, // Below two triggers, "every trigger relaxed" is a single observation and // cannot support a version-scoping decision. Flagged so the gap is visible // rather than implied by a number nobody reads. diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 7bd7de2ce61..305f55fdbb6 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -82,7 +82,9 @@ import { assertContractSchema, assertExactQueryCoverage, classifyBackendReportRow, + contractChannel, indexBackendReport, + normalizeLintWiring, normalizeTarget, resolveBackendOracle, } from './contract-schema.mjs'; @@ -90,6 +92,8 @@ import { // OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved // against the OSD checkout root, not this script's SQL-repo location. const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +const SYNTAX_MODULE = + 'src/plugins/data/public/antlr/opensearch_ppl/runtime_validation_core'; // The COMPILED-simplified surface: OSD's own checked-in grammar, used when the // engine cannot export a runtime bundle. See `PPL_LINT_SURFACE` below. const ANALYZER_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; @@ -174,7 +178,12 @@ function loadContracts() { if (!fs.existsSync(single)) { fatal(`Contract file not found: ${single}`); } - return [loadContractFile(single)]; + const contract = loadContractFile(single); + return { + contracts: [contract], + manifest: { contracts: [path.basename(single)] }, + manifestPath: '', + }; } if (!dir) { @@ -186,8 +195,8 @@ function loadContracts() { const manifestPath = path.join(dir, 'manifest.json'); let files; + let manifest; if (fs.existsSync(manifestPath)) { - let manifest; try { manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); } catch (error) { @@ -196,6 +205,9 @@ function loadContracts() { if (!Array.isArray(manifest.contracts)) { fatal(`manifest.json must have a "contracts" array of file names.`); } + if (new Set(manifest.contracts).size !== manifest.contracts.length) { + fatal(`manifest.json "contracts" contains duplicate file names.`); + } files = manifest.contracts.map((name) => path.join(dir, name)); } else { files = fs @@ -205,12 +217,13 @@ function loadContracts() { .map((f) => path.join(dir, f)); } - return files.map((file) => { + const contracts = files.map((file) => { if (!fs.existsSync(file)) { fatal(`Contract referenced by manifest not found: ${file}`); } return loadContractFile(file); }); + return { contracts, manifest: manifest || { contracts: files.map(path.basename) }, manifestPath }; } function loadOsd() { @@ -286,11 +299,17 @@ function loadOsd() { `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` ); } + const syntaxModule = resolveOsd(SYNTAX_MODULE, { optional: true }); + const validateSyntax = + syntaxModule && typeof syntaxModule.validateQueryWithBundle === 'function' + ? syntaxModule.validateQueryWithBundle + : undefined; return { surface: SURFACE, deserializeBundleOrThrow, lintQuery: lintQueryWithBundle, + validateSyntax, getBundledCatalog, getDetector, osdRoot, @@ -466,6 +485,9 @@ function selectExpectation(spec, version, isCalcite, failures, { allowMissing = */ function checkWiring(spec, catalog, getDetector, failures) { const { ruleId, wiring } = spec; + if (contractChannel(spec) === 'syntax') { + return { id: ruleId, syntaxCode: wiring && wiring.code }; + } const entry = catalog.find((c) => c.id === ruleId); if (!entry) { failures.push(`[${ruleId}] not present in the OSD bundled catalog.`); @@ -475,31 +497,20 @@ function checkWiring(spec, catalog, getDetector, failures) { return entry; // no wiring block to assert } - const checks = [ - ['detector', wiring.detector, entry.detector], - ['enabled', wiring.enabled, entry.enabled], - ['severity', wiring.severity, entry.severity], - ['runtimeOnly', !!wiring.runtimeOnly, !!entry.runtimeOnly], - ['needsContext', !!wiring.needsContext, !!entry.needsContext], - ['needsExplain', !!wiring.needsExplain, !!entry.needsExplain], - ]; - for (const [name, expected, actual] of checks) { - if (expected !== undefined && expected !== actual) { - failures.push( - `[${ruleId}] wiring.${name} expected ${JSON.stringify(expected)} but catalog has ${JSON.stringify(actual)}.` - ); - } + let expected; + let actual; + try { + expected = normalizeLintWiring(ruleId, wiring, `[${ruleId}] contract.wiring`); + actual = normalizeLintWiring(ruleId, entry, `[${ruleId}] catalog`); + } catch (error) { + failures.push(error.message); + return entry; } - - if (wiring.appliesTo) { - const a = entry.appliesTo || {}; - for (const key of ['minVersion', 'maxVersion', 'engine']) { - if (wiring.appliesTo[key] !== undefined && wiring.appliesTo[key] !== a[key]) { - failures.push( - `[${ruleId}] wiring.appliesTo.${key} expected ${JSON.stringify(wiring.appliesTo[key])} but catalog has ${JSON.stringify(a[key])}.` - ); - } - } + if (JSON.stringify(expected) !== JSON.stringify(actual)) { + failures.push( + `[${ruleId}] normalized wiring mismatch: contract=${JSON.stringify(expected)} ` + + `catalog=${JSON.stringify(actual)}.` + ); } if (wiring.detector && typeof getDetector === 'function' && typeof getDetector(wiring.detector) !== 'function') { @@ -553,16 +564,124 @@ function buildContext(spec, engineVersion) { return context; } +function equalSets(left, right) { + return left.size === right.size && [...left].every((value) => right.has(value)); +} + +function buildCensus(contracts, manifest, catalog) { + const problems = []; + const byFile = new Map( + contracts.map(({ file, spec }) => [path.basename(file), spec]) + ); + const resolveManifestRules = (field) => { + const names = manifest[field] || []; + if (!Array.isArray(names)) { + problems.push(`manifest.${field} must be an array.`); + return []; + } + if (new Set(names).size !== names.length) { + problems.push(`manifest.${field} contains duplicate file names.`); + } + const rules = []; + for (const name of names) { + const spec = byFile.get(name); + if (!spec) { + problems.push(`manifest.${field} references inactive or missing contract "${name}".`); + } else { + rules.push(spec.ruleId); + } + } + return rules; + }; + + const activeContractRules = contracts.map(({ spec }) => spec.ruleId).sort(); + const duplicateRuleIds = activeContractRules.filter( + (ruleId, index) => activeContractRules.indexOf(ruleId) !== index + ); + if (duplicateRuleIds.length > 0) { + problems.push(`active contracts contain duplicate rule IDs: ${duplicateRuleIds.join(', ')}.`); + } + + const activeLintRules = contracts + .filter(({ spec }) => contractChannel(spec) === 'lint') + .map(({ spec }) => spec.ruleId) + .sort(); + const activeSyntaxRules = contracts + .filter(({ spec }) => contractChannel(spec) === 'syntax') + .map(({ spec }) => spec.ruleId) + .sort(); + const enabledRules = catalog + .filter((rule) => rule.enabled) + .map((rule) => rule.id) + .sort(); + const defaultErrorRules = catalog + .filter((rule) => rule.enabled && rule.severity === 'error') + .map((rule) => rule.id) + .sort(); + const manifestDefaultErrorRules = resolveManifestRules('defaultError').sort(); + const requiredSyntaxFeatures = resolveManifestRules('requiredSyntaxFeatures').sort(); + + if (activeLintRules.length !== 12) { + problems.push(`expected 12 active lint contracts, found ${activeLintRules.length}.`); + } + if (requiredSyntaxFeatures.length !== 1) { + problems.push( + `expected one required syntax feature, found ${requiredSyntaxFeatures.length}.` + ); + } + if (activeContractRules.length !== 13) { + problems.push(`expected 13 active contracts, found ${activeContractRules.length}.`); + } + if (!equalSets(new Set(activeSyntaxRules), new Set(requiredSyntaxFeatures))) { + problems.push( + `active syntax contracts ${JSON.stringify(activeSyntaxRules)} do not equal ` + + `manifest.requiredSyntaxFeatures ${JSON.stringify(requiredSyntaxFeatures)}.` + ); + } + if (!equalSets(new Set(activeLintRules), new Set(enabledRules))) { + problems.push( + `active lint contracts ${JSON.stringify(activeLintRules)} do not equal enabled catalog ` + + `rules ${JSON.stringify(enabledRules)}.` + ); + } + if (!equalSets(new Set(manifestDefaultErrorRules), new Set(defaultErrorRules))) { + problems.push( + `manifest.defaultError rules ${JSON.stringify(manifestDefaultErrorRules)} do not equal ` + + `enabled error catalog rules ${JSON.stringify(defaultErrorRules)}.` + ); + } + + return { + enabledRules, + defaultErrorRules, + requiredSyntaxFeatures, + activeContractRules, + activeLintRules, + activeSyntaxRules, + manifestDefaultErrorRules, + passed: problems.length === 0, + problems, + }; +} + function main() { const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; const reportPath = process.env.PPL_LINT_REPORT; const target = loadTarget(); const backendReport = loadBackendReport(target); - const contracts = loadContracts(); + const { contracts, manifest, manifestPath } = loadContracts(); const osd = loadOsd(); - const { getBundledCatalog, getDetector, lintQuery, osdRoot, surface } = osd; + const { + getBundledCatalog, + getDetector, + lintQuery, + validateSyntax, + osdRoot, + surface, + } = osd; const catalog = getBundledCatalog(); + const census = buildCensus(contracts, manifest, catalog); // The compiled surface lints with OSD's own checked-in grammar, so there is no // candidate bundle to load. On the runtime surface a missing bundle stays a hard @@ -610,8 +729,24 @@ function main() { .filter((rule) => rule.enabled && rule.severity === 'error') .map((rule) => rule.id) .sort(), + enabledRules: census.enabledRules, + requiredSyntaxFeatures: census.requiredSyntaxFeatures, + activeContractRules: census.activeContractRules, + census: { + enforced: process.env.PPL_LINT_ENFORCE_CENSUS === '1', + manifest: manifestPath, + ...census, + }, results: [], }; + if (!census.passed) { + for (const problem of census.problems) { + log(`CENSUS REPORT-ONLY: ${problem}`); + } + if (process.env.PPL_LINT_ENFORCE_CENSUS === '1') { + failures.push(...census.problems.map((problem) => `[census] ${problem}`)); + } + } log(`OSD root: ${osdRoot}`); log( @@ -623,6 +758,11 @@ function main() { for (const { file, spec } of contracts) { const ruleId = spec.ruleId; const index = spec.index; + const channel = contractChannel(spec); + const entry = checkWiring(spec, catalog, getDetector, failures); + if (!entry) { + continue; + } // A contract runs on PR only when scheduled for PR; nightly runs everything. const contractSchedule = spec.schedule || 'pr'; @@ -650,6 +790,7 @@ function main() { for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { report.results.push({ ruleId, + channel, queryName, role: queryDef.role || 'trigger', query: (queryDef.query || '').split('{{index}}').join(index), @@ -662,11 +803,6 @@ function main() { continue; } - const entry = checkWiring(spec, catalog, getDetector, failures); - if (!entry) { - continue; - } - const context = buildContext(spec, engineVersion); const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures, { allowMissing: observeOnly, @@ -681,6 +817,7 @@ function main() { if (surface === 'compiled-simplified' && entry.runtimeOnly) { report.results.push({ ruleId, + channel, queryName, role, query, @@ -691,10 +828,23 @@ function main() { }); continue; } - const result = lintQuery(query, grammar, context); - const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + fatal( + `Syntax contract "${ruleId}" requires validateQueryWithBundle from ${SYNTAX_MODULE}. ` + + `Validate this SQL branch against the OSD headless-syntax PR.` + ); + } + const result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const matches = + channel === 'syntax' + ? result.errors || [] + : (result.diagnostics || []).filter((d) => d.ruleId === ruleId); report.results.push({ ruleId, + channel, queryName, role, query, @@ -730,9 +880,10 @@ function main() { } catch (error) { fatal(`Invalid contract ${file} query "${queryName}": ${error.message}`); } - const expectedCount = oracleSelection.detector.count; - const expectedSeverity = oracleSelection.detector.severity; - const expectedMessage = oracleSelection.detector.matchMessage; + const frontendOracle = oracleSelection.frontend; + const expectedCount = frontendOracle.count; + const expectedSeverity = frontendOracle.severity; + const expectedMessage = frontendOracle.matchMessage; // A `runtimeOnly` rule walks grammar productions that exist only in the // runtime bundle, so `lint_runner` skips it on the compiled surface. Its @@ -747,6 +898,7 @@ function main() { ); report.results.push({ ruleId, + channel, queryName, role, query, @@ -758,8 +910,22 @@ function main() { continue; } - const result = lintQuery(query, grammar, context); - const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + fatal( + `Syntax contract "${ruleId}" requires validateQueryWithBundle from ${SYNTAX_MODULE}. ` + + `Validate this SQL branch against the OSD headless-syntax PR.` + ); + } + const result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const allFrontendFindings = + channel === 'syntax' ? result.errors || [] : result.diagnostics || []; + const matches = + channel === 'syntax' + ? allFrontendFindings.filter((finding) => finding.code === frontendOracle.code) + : allFrontendFindings.filter((finding) => finding.ruleId === ruleId); const actual = matches.length; const ok = actual === expectedCount; @@ -769,23 +935,51 @@ function main() { ); const severityOk = + channel === 'syntax' || !expectedSeverity || actual === 0 || matches.every((m) => m.severity === expectedSeverity); const messageOk = !expectedMessage || matches.some((m) => (m.message || '').includes(expectedMessage)); + const fixOk = + channel !== 'syntax' || + frontendOracle.fixText === undefined || + matches.some((m) => m.fix && m.fix.text === frontendOracle.fixText); + const rawMessageOk = + channel !== 'syntax' || + frontendOracle.rawMessage === undefined || + matches.some((m) => + frontendOracle.rawMessage + ? typeof m.rawMessage === 'string' && m.rawMessage.length > 0 + : m.rawMessage === undefined + ); + const totalErrorsOk = + channel !== 'syntax' || + frontendOracle.totalErrors === undefined || + allFrontendFindings.length === frontendOracle.totalErrors; const resultEntry = { ruleId, + channel, queryName, role, query, expected: expectedCount, actual, - severities: matches.map((m) => m.severity), + severities: matches.map((m) => m.severity).filter(Boolean), severityMatched: severityOk, messageMatched: messageOk, + fixMatched: fixOk, + rawMessageMatched: rawMessageOk, + totalErrorsMatched: totalErrorsOk, + ...(channel === 'syntax' + ? { + code: frontendOracle.code, + codes: allFrontendFindings.map((finding) => finding.code).filter(Boolean), + totalErrors: allFrontendFindings.length, + } + : {}), executionBackend, backendOracleStatus: oracleSelection.status, }; @@ -805,6 +999,22 @@ function main() { `[${ruleId}/${queryName}] expected message to contain "${expectedMessage}" for: ${query}` ); } + if (!fixOk) { + failures.push( + `[${ruleId}/${queryName}] expected fix text "${frontendOracle.fixText}" for: ${query}` + ); + } + if (!rawMessageOk) { + failures.push( + `[${ruleId}/${queryName}] expected rawMessage=${frontendOracle.rawMessage} for: ${query}` + ); + } + if (!totalErrorsOk) { + failures.push( + `[${ruleId}/${queryName}] expected ${frontendOracle.totalErrors} total syntax error(s), ` + + `got ${allFrontendFindings.length} for: ${query}` + ); + } if (oracleSelection.status === 'not-applicable') { // Only the backend fixture is non-applicable. The detector still ran above and its @@ -894,6 +1104,17 @@ function main() { `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` ); } + if ( + role === 'suppression-control' && + (detectorFlagged || !backendRejected) + ) { + failures.push( + `[${ruleId}/${queryName}] differential: suppression control must retain a backend ` + + `syntax rejection without a "${frontendOracle.code}" suggestion, but frontend ` + + `${detectorFlagged ? 'suggested a rewrite' : 'did not suggest a rewrite'} and ` + + `backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } } } } @@ -902,16 +1123,6 @@ function main() { } } - // Nightly-only coverage: every enabled catalog rule must have a contract file. - if (schedule === 'nightly') { - const covered = new Set(contracts.map(({ spec }) => spec.ruleId)); - for (const rule of catalog) { - if (rule.enabled && !covered.has(rule.id)) { - failures.push(`[coverage] enabled catalog rule "${rule.id}" has no contract file.`); - } - } - } - if (reportPath) { report.failures = failures; try { From c3af7efaeff8905407ac93c85a673e4f97463ca8 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 11:14:22 -0700 Subject: [PATCH 71/78] feat(ci): enforce exact PPL lint frontend contracts Pin exact diagnostic, deterministic-fix, and AI-action behavior for all active schema-v4 contracts, and promote the complete 13-contract corpus to required PR validation. Enforce the shipping census while retaining dormant detector contracts as report-only observations. Propagate frontend assertion failures through required and multi-version reports with inline annotations. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 2 + .../workflows/ppl-lint-rule-validation.yml | 6 +- .../ppl-lint/contracts/agg-on-text.spec.json | 28 +- .../contracts/command-suggestion.spec.json | 13 +- .../contracts/division-by-zero.spec.json | 52 +- .../contracts/enabled-false-object.spec.json | 28 +- .../contracts/field-validation.spec.json | 142 +++- .../invalid-capture-group-name.spec.json | 76 ++- .../ppl-lint/contracts/manifest.json | 11 +- .../multisearch-min-subsearch.spec.json | 38 +- .../replace-wildcard-asymmetry.spec.json | 76 ++- .../contracts/rex-scan-cost.spec.json | 26 +- .../contracts/type-mismatch-numeric.spec.json | 28 +- .../contracts/union-min-datasets.spec.json | 38 +- ...ed-window-function-in-eventstats.spec.json | 114 +++- .../wildcard-source-zero-match.spec.json | 19 +- scripts/ppl-lint/README.md | 28 +- .../__tests__/aggregate-versions.test.mjs | 88 +++ scripts/ppl-lint/__tests__/annotate.test.mjs | 32 + .../__tests__/assemble-run-manifest.test.mjs | 73 ++- .../__tests__/contract-schema.test.mjs | 252 +++++++- .../__tests__/run-frontend-contract.test.mjs | 325 ++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 168 ++++- scripts/ppl-lint/annotate.mjs | 19 + scripts/ppl-lint/assemble-run-manifest.mjs | 96 ++- scripts/ppl-lint/contract-schema.mjs | 238 ++++++- scripts/ppl-lint/run-frontend-contract.mjs | 609 +++++++++++++++--- 27 files changed, 2375 insertions(+), 250 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 8e31ab55918..df1aad11185 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -837,6 +837,8 @@ jobs: env "${surface_env[@]}" "${observe_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_ENFORCE_CENSUS=1 \ + PPL_LINT_INCLUDE_DORMANT=1 \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ PPL_LINT_REPORT="$leg/detector-report.json" \ node -r ./src/setup_node_env \ diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 6f83c60a60b..7ffabbce2da 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -37,8 +37,8 @@ concurrency: # # Modes (design §3.4, §4.1.1): # - pull_request: SQL PR validation against the resolved OSD target. The ONLY -# enforcing mode; this is what branch protection pins to. Runs the fast -# schedule:pr subset. The committed default is `main` on the canonical repo; +# enforcing mode; this is what branch protection pins to. Runs all 13 active +# schedule:pr contracts. The committed default is `main` on the canonical repo; # it can be overridden by the OSD_REPO/OSD_REF repo variables — see the # "Resolve OSD ref" step. TEMPORARY: those repo variables are currently set to # the unmerged paired OSD branch that ships the headless lint API this job @@ -295,6 +295,8 @@ jobs: PPL_LINT_TARGET_MANIFEST: ${{ github.workspace }}/artifacts/target.json PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json + PPL_LINT_ENFORCE_CENSUS: '1' + PPL_LINT_INCLUDE_DORMANT: ${{ steps.schedule.outputs.value == 'nightly' && '1' || '0' }} run: | # pipefail so the runner's non-zero exit propagates through `tee` — # otherwise the pipeline takes tee's (success) status and a real diff --git a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json index 4c574aaa86c..ebec7087274 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json @@ -4,7 +4,7 @@ "channel": "lint", "note": "The standard engine accepts both text aggregations: avg(text) returns null while sum(text) returns a non-empty numeric result (observed as 0.0). The warning catches this misleading coercion rather than predicting backend rejection.", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "agg-on-text", "enabled": true, @@ -58,7 +58,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "text field" + "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -81,7 +88,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "text field" + "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -102,7 +116,13 @@ }, "avg-numeric-control": { "frontend": { - "count": 0 + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json index e1bb53ffb3b..021a37bb4b3 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json @@ -3,7 +3,7 @@ "ruleId": "command-suggestion", "channel": "syntax", "grammarSurface": "runtime-bundle", - "schedule": "nightly", + "schedule": "pr", "wiring": { "code": "UNKNOWN_COMMAND" }, @@ -47,8 +47,9 @@ "count": 1, "code": "UNKNOWN_COMMAND", "fixText": "where", - "matchMessage": "where", - "rawMessage": true + "matchMessage": "Unknown command \"wherre\". Did you mean \"where\"?", + "rawMessage": true, + "totalErrors": 1 }, "backends": { "standard": { @@ -71,6 +72,8 @@ "frontend": { "count": 0, "code": "UNKNOWN_COMMAND", + "fixText": null, + "rawMessage": false, "totalErrors": 0 }, "backends": { @@ -94,6 +97,8 @@ "frontend": { "count": 0, "code": "UNKNOWN_COMMAND", + "fixText": null, + "rawMessage": false, "totalErrors": 1 }, "backends": { @@ -117,6 +122,8 @@ "frontend": { "count": 0, "code": "UNKNOWN_COMMAND", + "fixText": null, + "rawMessage": false, "totalErrors": 1 }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 88c4226fd8e..b643094471b 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -50,8 +50,18 @@ "version": ">=0.0.0", "queries": { "divide-by-zero-literal": { - "detectorCount": 1, - "severity": "warning", + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "result-shape", @@ -70,8 +80,18 @@ } }, "divide-by-decimal-zero-literal": { - "detectorCount": 1, - "severity": "warning", + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "result-shape", @@ -90,7 +110,15 @@ } }, "divide-by-nonzero-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -109,8 +137,18 @@ } }, "modulo-by-zero-literal": { - "detectorCount": 1, - "severity": "warning", + "frontend": { + "count": 1, + "severity": "warning", + "messageEquals": "Dividing by zero returns no value (null) instead of an error.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json index b9ebc0eef91..4e222982589 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json @@ -4,7 +4,7 @@ "channel": "lint", "note": "The standard Calcite route can still project and filter enabled:false object values from _source. These result-shape oracles pin that observed acceptance; the warning communicates that the object is not indexed/searchable through ordinary OpenSearch field semantics.", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "enabled-false-object", "enabled": true, @@ -60,7 +60,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "not searchable" + "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -83,7 +90,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "not searchable" + "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -104,7 +118,13 @@ }, "indexed-field-control": { "frontend": { - "count": 0 + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 6225eaed87b..4094f0af7e0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -62,8 +62,18 @@ "note": "This rule has an empty appliesTo, so it ships to users on EVERY engine, including pre-3.4. Detector behavior is live-verified identical from 2.19 up (1/1/0 on the compiled surface at 2.19.0, 3.0.0, 3.5.0, 3.7.0). The backend oracle deliberately omits error.type/reason: this engine's wording for an unknown field has not been observed live, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. A compiled-surface leg records the real wording; pin it then.", "queries": { "unknown-field-existence": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -82,8 +92,27 @@ } }, "grok-field-slot-shape-typo": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "expectedText": "field=firstname", + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "rejection", @@ -102,7 +131,15 @@ } }, "known-field-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -126,8 +163,18 @@ "version": ">=3.4.0 <3.7.0", "queries": { "unknown-field-existence": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -154,8 +201,27 @@ } }, "grok-field-slot-shape-typo": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "expectedText": "field=firstname", + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "rejection", @@ -182,7 +248,15 @@ } }, "known-field-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -206,9 +280,18 @@ "version": ">=3.7.0", "queries": { "unknown-field-existence": { - "detectorCount": 1, - "severity": "error", - "matchMessage": "nonexistent_field", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Unknown field \"nonexistent_field\".", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -235,8 +318,27 @@ } }, "grok-field-slot-shape-typo": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "grok expects a field name here, not an expression.", + "deterministicFix": { + "offered": true, + "title": "Remove \"field=\" (use \"firstname\")", + "text": "firstname", + "range": { + "startLine": 1, + "startColumn": 48, + "endLine": 1, + "endColumn": 63 + }, + "expectedText": "field=firstname", + "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "rejection", @@ -263,7 +365,15 @@ } }, "known-field-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index 4c2cfc36c6f..b4ba3df32c8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -55,8 +55,18 @@ "engine": "calcite", "queries": { "rex-capture-name-underscore": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -83,8 +93,18 @@ } }, "rex-capture-name-hyphen": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -111,7 +131,15 @@ } }, "rex-capture-name-alphanumeric-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -136,8 +164,18 @@ "engine": "calcite", "queries": { "rex-capture-name-underscore": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -164,8 +202,18 @@ } }, "rex-capture-name-hyphen": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -192,7 +240,15 @@ } }, "rex-capture-name-alphanumeric-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index 3e83f493e0c..20a6995acb8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,6 +1,6 @@ { "schemaVersion": 4, - "description": "Active PPL frontend/backend compatibility corpus for the approved 12 detector rules plus the command-suggestion syntax feature. New contracts remain nightly until their runtime and analytics oracles are reviewed; census drift is report-only until the paired OSD default-alignment change lands.", + "description": "Required PPL frontend/backend compatibility corpus for the approved 12 detector rules plus the command-suggestion syntax feature.", "contracts": [ "agg-on-text.spec.json", "command-suggestion.spec.json", @@ -41,14 +41,6 @@ "requiredSyntaxFeatures": [ "command-suggestion.spec.json" ], - "pendingReview": [ - "agg-on-text.spec.json", - "command-suggestion.spec.json", - "enabled-false-object.spec.json", - "rex-scan-cost.spec.json", - "type-mismatch-numeric.spec.json", - "wildcard-source-zero-match.spec.json" - ], "nonEnforcing": [ "agg-on-text.spec.json", "division-by-zero.spec.json", @@ -61,7 +53,6 @@ "enforced": "Reviewed lint error contracts with deterministic backend behavior.", "defaultError": "Exact approved six-rule detector error census. command-suggestion is an error-channel feature but is intentionally excluded because it is not a detector.", "requiredSyntaxFeatures": "Syntax-channel features validated through the production runtime grammar listener.", - "pendingReview": "Nightly contracts whose standard and analytics observations must be reviewed before promotion to the required PR schedule.", "nonEnforcing": "Oracle-quality classification for warning, info, advisory, and result-shape contracts; scheduling determines execution, not this list.", "dormantContracts": "Preserved default-off detector regression contracts. They do not count toward active shipping coverage and must force-enable their detector when run." } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 21cdae63845..106ed504f01 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -52,8 +52,18 @@ "version": ">=3.4.0", "queries": { "multisearch-single-subsearch": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The multisearch command requires at least two subsearches.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -80,8 +90,18 @@ } }, "multisearch-single-subsearch-with-where": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The multisearch command requires at least two subsearches.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -108,7 +128,15 @@ } }, "multisearch-two-subsearches-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 29def9f3b8a..c4b310bbac8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -53,8 +53,18 @@ "engine": "calcite", "queries": { "replace-wildcard-count-mismatch": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -81,8 +91,18 @@ } }, "replace-wildcard-count-mismatch-reverse": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -109,7 +129,15 @@ } }, "replace-symmetric-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -134,8 +162,18 @@ "engine": "calcite", "queries": { "replace-wildcard-count-mismatch": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -162,8 +200,18 @@ } }, "replace-wildcard-count-mismatch-reverse": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -190,7 +238,15 @@ } }, "replace-symmetric-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json index 918eaacb0cd..5230aefe49f 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json @@ -3,7 +3,7 @@ "ruleId": "rex-scan-cost", "channel": "lint", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "rex-scan-cost", "enabled": true, @@ -52,7 +52,13 @@ "frontend": { "count": 1, "severity": "info", - "matchMessage": "every input row" + "messageEquals": "parse runs the pattern against every input row from text field \"email\", even when it finds no match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { @@ -69,7 +75,13 @@ "frontend": { "count": 1, "severity": "info", - "matchMessage": "every input row" + "messageEquals": "grok runs the pattern against every input row from text field \"email\", even when it finds no match.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { @@ -84,7 +96,13 @@ }, "plain-field-control": { "frontend": { - "count": 0 + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json index 0f3e14791f5..5589bdb79f7 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json @@ -3,7 +3,7 @@ "ruleId": "type-mismatch-numeric", "channel": "lint", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "type-mismatch-numeric", "enabled": true, @@ -56,7 +56,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "not a number" + "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -79,7 +86,14 @@ "frontend": { "count": 1, "severity": "warning", - "matchMessage": "not a number" + "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -100,7 +114,13 @@ }, "numeric-string-control": { "frontend": { - "count": 0 + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index 6de9db9a7d6..ca16823381a 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -55,8 +55,18 @@ "engine": "calcite", "queries": { "union-single-dataset": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The union command requires at least two datasets.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -83,8 +93,18 @@ } }, "union-single-dataset-with-fields": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "The union command requires at least two datasets.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -111,7 +131,15 @@ } }, "union-two-datasets-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index fda4d273fb9..6c017195e1a 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -49,8 +49,18 @@ "version": ">=3.4.0 <3.7.0", "queries": { "eventstats-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -77,8 +87,18 @@ } }, "eventstats-dense-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -105,7 +125,15 @@ } }, "eventstats-avg-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -129,8 +157,18 @@ "version": ">=3.7.0 <3.8.0", "queries": { "eventstats-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -157,8 +195,18 @@ } }, "eventstats-dense-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -185,7 +233,15 @@ } }, "eventstats-avg-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", @@ -209,8 +265,18 @@ "version": ">=3.8.0", "queries": { "eventstats-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -237,8 +303,18 @@ } }, "eventstats-dense-rank": { - "detectorCount": 1, - "severity": "error", + "frontend": { + "count": 1, + "severity": "error", + "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } + }, "backends": { "standard": { "kind": "rejection", @@ -265,7 +341,15 @@ } }, "eventstats-avg-control": { - "detectorCount": 0, + "frontend": { + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } + }, "backends": { "standard": { "kind": "result-shape", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json index df01303aefb..daf9920cab7 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json @@ -3,7 +3,7 @@ "ruleId": "wildcard-source-zero-match", "channel": "lint", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "wildcard-source-zero-match", "enabled": true, @@ -48,7 +48,14 @@ "frontend": { "count": 1, "severity": "info", - "matchMessage": "matches no known index" + "messageEquals": "Wildcard source pattern matches no known index.", + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": true, + "commandId": "ppl.lint.aiFix" + } }, "backends": { "standard": { @@ -69,7 +76,13 @@ }, "matching-wildcard-control": { "frontend": { - "count": 0 + "count": 0, + "deterministicFix": { + "offered": false + }, + "aiAction": { + "offered": false + } }, "backends": { "standard": { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f79a3052513..62b7d157283 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -50,12 +50,10 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j | `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | | `schedule` (nightly) | full corpus + coverage | `main` | No | -The active corpus contains 12 detector contracts plus the -`command-suggestion` syntax contract. Seven reviewed contracts currently declare -`schedule: "pr"`; the six new contracts remain `nightly` until their standard and -analytics observations are reviewed. A contract that runs also asserts: neither -the IT nor the frontend runner consults the manifest's `enforced` list, so any -contract on the PR schedule can fail the required check. +The required PR corpus contains 12 detector contracts plus the +`command-suggestion` syntax contract. All 13 declare `schedule: "pr"` and can +fail the required check. The nightly mode runs the same shipping corpus across +the supported version and execution-backend matrix. `workflow_dispatch` inputs: @@ -214,8 +212,6 @@ rule cannot be validated end to end. `rules_catalog.json`; it contains exactly six detector rules. - `requiredSyntaxFeatures` — `command-suggestion` only. Syntax features never appear in `defaultError` or the detector catalog. -- `pendingReview` — the six nightly contracts awaiting oracle review and PR - promotion. - `nonEnforcing` — oracle-quality classification for warning, info, advisory, and result-shape contracts. Scheduling determines whether a contract runs. - `dormantContracts` — four preserved default-off detector contracts. They do @@ -410,8 +406,8 @@ different places a developer looks: The required single-version lane follows the same rule: frontend and backend failures with a `[rule/query]` identity anchor on that contract's `ruleId`. -Shipping-census findings anchor on `manifest.json` (as warnings while census -enforcement is report-only). Artifact and job failures without a trustworthy +Shipping-census findings anchor on `manifest.json` and fail the required lane. +Artifact and job failures without a trustworthy repository location remain file-less rather than pointing at a guessed line. Without the annotations the only thing above the summary is `Process completed @@ -470,10 +466,9 @@ node --test "scripts/ppl-lint/__tests__/*.test.mjs" ## Discovery corpus (harvested, never enforced) -The enforced corpus is hand-pinned, which is what lets a mismatch red the build — -and also why it is small (about one trigger per rule). One trigger is not enough to -tell a full engine fix from a partial one, so the `discovery` job builds a second, -much larger corpus that pins nothing. +The required corpus is hand-pinned, which is what lets a mismatch red the build. +The `discovery` job builds a larger unpinned corpus to distinguish full engine +fixes from partial behavior changes. ``` harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report @@ -487,9 +482,8 @@ harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-front (matched as a prefix, so `describe('rex-scan-cost (compiled surface)')` counts). A query with no rule-owning ancestor is recorded unattributed and dropped rather than guessed at. Indices are rewritten onto the fixture index; JS string escapes - are unescaped so the query matches what the test actually linted. Against OSD - `main` today this yields **~109 queries across 12 rules** versus 27 across 11 in - the enforced corpus. + are unescaped so the query matches what the test actually linted. The harvested + corpus is substantially larger than the curated 13-contract required corpus. Each file's **lint context** is harvested alongside its queries. Seven of the nineteen rules are `needsContext: true` and self-suppress without a `typeMap`, so diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index b498f594745..b42c825384d 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -119,6 +119,7 @@ function writeLeg({ grammarHash = `sha256:${version}`, surface = 'runtime-bundle', explicitIdentity = true, + censusEnforced = false, }) { const dir = makeTmp(`ppl-lint-leg-${version}-`); const target = { @@ -168,6 +169,20 @@ function writeLeg({ severities: c.severities || (c.detector > 0 ? ['error'] : []), severityMatched: c.severityMatched ?? true, messageMatched: c.messageMatched ?? true, + ...Object.fromEntries( + [ + 'deterministicFixMatched', + 'aiActionMatched', + 'actionDecisionMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ] + .filter((field) => c[field] !== undefined) + .map((field) => [field, c[field]]) + ), + ...(c.assertions ? { assertions: c.assertions } : {}), + ...(c.mismatches ? { mismatches: c.mismatches } : {}), ...(explicitIdentity ? { executionBackend } : {}), }); backend.push({ @@ -200,6 +215,10 @@ function writeLeg({ surface, results, ...(defaultErrorRules !== null ? { defaultErrorRules } : {}), + enabledRules: [SPEC.ruleId], + activeContractRules: [SPEC.ruleId], + requiredSyntaxFeatures: [], + census: { enforced: censusEnforced }, }) ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); @@ -625,6 +644,51 @@ test('paired detector reports must be identical across execution backends', () = assert.match(stderr, /detector parity failed for union-min-datasets::trigger/); }); +test('identical exact-action mismatches across versions cannot aggregate green', () => { + const badCase = { + detector: 1, + rejected: true, + deterministicFixMatched: false, + assertions: { deterministicFix: false }, + mismatches: [ + { + field: 'deterministicFix', + expected: { offered: false }, + actual: { offered: true }, + }, + ], + }; + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: badCase, + control: { detector: 0, rejected: false }, + }, + }), + '3.8.0': writeLeg({ + version: '3.8.0', + cases: { + trigger: badCase, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const actionDrifts = report.drifts.filter( + (drift) => drift.driftClass === 'frontend-contract-mismatch' + ); + assert.equal(actionDrifts.length, 2); + assert.ok( + actionDrifts.every((drift) => + drift.frontendAssertions.includes('deterministicFixMatched') + ) + ); + assert.ok(report.matrix.every((row) => row.status === 'drift')); +}); + test('target and detector execution identities must match', () => { const dir = writeLeg({ version: '3.8.0', @@ -1048,6 +1112,7 @@ test('a default-error rule with no contract file fails the check', () => { version: '3.7.0', cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, defaultErrorRules: ['union-min-datasets', 'brand-new-error-rule'], + censusEnforced: true, }), }; const { status, report, stdout } = run({ contracts: writeContracts(), legs }); @@ -1071,6 +1136,29 @@ test('a census matching the manifest keeps the check green', () => { assert.equal(report.result.missingContractCount, 0); }); +test('an enforced shipping census mismatch fails aggregation', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + censusEnforced: true, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.passed, false); + assert.ok(report.result.blockingShippingCensusProblems > 0); + assert.equal(report.shippingCensus.blocking, true); + assert.match(stdout, /shipping census problem/); + assert.match(stdout, /CENSUS ENFORCED/); + assert.doesNotMatch(stdout, /CENSUS REPORT-ONLY/); + assert.match(stdout, /### Shipping census/); +}); + test('a schema-v2 detector report without a census fails closed', () => { const dir = writeLeg({ version: '3.8.0', diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs index 1209c1b0637..c0aca6f5d33 100644 --- a/scripts/ppl-lint/__tests__/annotate.test.mjs +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -180,6 +180,38 @@ test('an unvalidated rule has no file to point at', () => { assert.match(annotations[0].message, /manifest\.defaultError/); }); +test('shipping census errors point to manifest.json', () => { + const annotations = buildAnnotations( + { + shippingCensus: { + passed: false, + blocking: true, + problems: ['active lint rules do not equal enabled OSD rules'], + }, + }, + { + contractsDir: '/workspace/contracts', + workspace: '/workspace', + readFile: (_dir, file) => + file === 'manifest.json' + ? '{\n "schemaVersion": 4,\n "contracts": []\n}\n' + : undefined, + } + ); + + assert.deepEqual(annotations, [ + { + level: 'error', + file: 'contracts/manifest.json', + line: 3, + title: 'PPL lint shipping census mismatch', + message: + 'active lint rules do not equal enabled OSD rules\n' + + 'FIX: align the active SQL manifest with the approved OSD shipping catalog.', + }, + ]); +}); + test('paths are repo-relative so GitHub can render them inline', () => { // An absolute path still annotates the run, but never attaches to the diff. assert.equal( diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs index a3ee6f10acd..7e2c1afa062 100644 --- a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -184,6 +184,77 @@ test('syntax-specific frontend mismatches fail artifact validation', () => { const result = run(dir); assert.notEqual(result.status, 0); - assert.match(result.stderr, /did not match its (fix|raw-message|total-error) assertion/); + assert.match( + result.stderr, + /did not match its (syntax-fix|raw-parser-error|total-error) assertion/ + ); } }); + +test('exact deterministic and AI action mismatches fail artifacts and summary rows', () => { + for (const field of ['deterministicFixMatched', 'aiActionMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + detector.results[0].assertions = { + [field === 'deterministicFixMatched' ? 'deterministicFix' : 'aiAction']: false, + }; + detector.results[0].mismatches = [ + { + field: field === 'deterministicFixMatched' ? 'deterministicFix' : 'aiAction', + expected: { offered: false }, + actual: { offered: true }, + }, + ]; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not match its (deterministic-fix|AI-action) assertion/); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); + } +}); + +test('report-only dormant rows do not affect the required manifest result or active set', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results.push({ + ruleId: 'dormant-rule', + queryName: 'trigger', + role: 'trigger', + expected: 1, + actual: 0, + severities: [], + severityMatched: false, + messageMatched: false, + assertions: { count: false }, + mismatches: [{ field: 'count', expected: 1, actual: 0 }], + executionBackend: 'standard', + reportOnly: true, + }); + fs.writeFileSync(file, JSON.stringify(detector)); + const backendFile = path.join(dir, 'artifacts', 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend.push({ + ruleId: 'dormant-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'report-only observation failed', + }); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const summary = path.join(dir, 'summary.md'); + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.equal(result.status, 0, result.stderr); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.deepEqual(manifest.validationSet, ['advisory-rule']); + assert.equal(manifest.result.passed, true); + assert.match(fs.readFileSync(summary, 'utf8'), /dormant-rule.*Report only/); +}); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs index 914dd31569f..5bb44957bf1 100644 --- a/scripts/ppl-lint/__tests__/contract-schema.test.mjs +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test'; import { assertContractSchema, assertExactQueryCoverage, + assertShippingFrontendOracles, classifyBackendReportRow, contractChannel, indexBackendReport, @@ -508,11 +509,189 @@ test('missing channel remains a backwards-compatible lint contract', () => { channel: 'lint', count: 1, severity: 'warning', - matchMessage: undefined, + messageEquals: undefined, + deterministicFix: undefined, + aiAction: undefined, } ); }); +test('schema-v4 lint frontend normalizes exact message, fix, and AI action oracles', () => { + const contract = spec(4); + const frontend = normalizeFrontendOracle(contract, { + frontend: { + count: 1, + severity: 'warning', + messageEquals: 'Use a non-zero divisor.', + deterministicFix: { + offered: true, + title: 'Replace zero', + text: '1', + range: { + startLine: 1, + startColumn: 20, + endLine: 1, + endColumn: 21, + }, + expectedText: '0', + appliedQuery: 'source=t | eval x = 1', + }, + aiAction: { offered: false }, + }, + }); + + assert.deepEqual(frontend, { + channel: 'lint', + count: 1, + severity: 'warning', + messageEquals: 'Use a non-zero divisor.', + deterministicFix: { + offered: true, + title: 'Replace zero', + text: '1', + range: { + startLine: 1, + startColumn: 20, + endLine: 1, + endColumn: 21, + }, + expectedText: '0', + appliedQuery: 'source=t | eval x = 1', + }, + aiAction: { offered: false }, + }); +}); + +test('matchMessage remains available only to schema-v3 lint contracts', () => { + assert.equal( + normalizeFrontendOracle(spec(3), { + frontend: { count: 1, matchMessage: 'legacy substring' }, + }).matchMessage, + 'legacy substring' + ); + assert.throws( + () => + normalizeFrontendOracle(spec(4), { + frontend: { count: 1, matchMessage: 'not exact' }, + }), + /matchMessage is not valid/ + ); +}); + +test('schema-v4 action payloads fail closed on partial or extra fields', () => { + const contract = spec(4); + for (const [frontend, expected] of [ + [ + { count: 1, deterministicFix: { offered: false, title: 'unexpected' } }, + /must contain only offered/, + ], + [ + { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 0, startColumn: 0, endLine: 1, endColumn: 1 }, + appliedQuery: 'x', + }, + }, + /startLine must be a positive integer/, + ], + [ + { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 1 }, + appliedQuery: 'x', + }, + }, + /expectedText must be a string/, + ], + [ + { + count: 1, + deterministicFix: { + offered: true, + title: 'Fix', + text: 'x', + range: { startLine: 1, startColumn: 2, endLine: 1, endColumn: 1 }, + expectedText: 'y', + appliedQuery: 'x', + }, + }, + /must end at or after its start/, + ], + [ + { count: 1, aiAction: { offered: true } }, + /commandId must be a non-empty string/, + ], + [ + { count: 1, aiAction: { offered: false, commandId: 'ppl.lint.aiFix' } }, + /must contain only offered/, + ], + ]) { + assert.throws(() => normalizeFrontendOracle(contract, { frontend }), expected); + } +}); + +test('active lint contracts require exact messages and explicit exclusive action modes', () => { + const contract = spec(4); + const expectation = structuredClone(contract.expectations[0]); + expectation.queries.trigger = { + frontend: { + count: 1, + severity: 'error', + messageEquals: 'Exact diagnostic.', + deterministicFix: { offered: false }, + aiAction: { offered: true, commandId: 'ppl.lint.aiFix' }, + }, + backends: { + standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + analytics: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + }, + }; + expectation.queries.control.frontend = { + count: 0, + deterministicFix: { offered: false }, + aiAction: { offered: false }, + }; + delete expectation.queries.control.detectorCount; + + assert.doesNotThrow(() => assertShippingFrontendOracles(contract, expectation)); + + const missingMessage = structuredClone(expectation); + delete missingMessage.queries.trigger.frontend.messageEquals; + assert.throws( + () => assertShippingFrontendOracles(contract, missingMessage), + /messageEquals is required/ + ); + + const missingSeverity = structuredClone(expectation); + delete missingSeverity.queries.trigger.frontend.severity; + assert.throws( + () => assertShippingFrontendOracles(contract, missingSeverity), + /severity is required/ + ); + + const simultaneousActions = structuredClone(expectation); + simultaneousActions.queries.trigger.frontend.deterministicFix = { + offered: true, + title: 'Fix', + text: 'fixed', + range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 3 }, + expectedText: 'bad', + appliedQuery: 'fixed', + }; + assert.throws( + () => assertShippingFrontendOracles(contract, simultaneousActions), + /cannot offer deterministic and AI actions together/ + ); +}); + test('syntax frontend assertions normalize stable code, fix, raw message, and error census', () => { const contract = { schemaVersion: 4, @@ -545,6 +724,77 @@ test('syntax frontend assertions normalize stable code, fix, raw message, and er assert.equal(assertContractSchema(contract), 4); }); +test('active syntax contracts require explicit fix, raw-message, and total-error assertions', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + trigger: { role: 'trigger', query: 'source=t | wherre a > 1' }, + }, + }; + const expectation = { + queries: { + trigger: { + frontend: { + count: 1, + code: 'UNKNOWN_COMMAND', + fixText: 'where', + matchMessage: 'Unknown command "wherre". Did you mean "where"?', + rawMessage: true, + totalErrors: 1, + }, + }, + }, + }; + assert.doesNotThrow(() => assertShippingFrontendOracles(contract, expectation)); + + delete expectation.queries.trigger.frontend.fixText; + assert.throws( + () => assertShippingFrontendOracles(contract, expectation), + /fixText must explicitly assert/ + ); +}); + +test('syntax supports explicit fix absence and requires frontend code to match wiring', () => { + const contract = { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + wiring: { code: 'UNKNOWN_COMMAND' }, + queries: { + suppressed: { role: 'suppression-control', query: 'source=t | zzzzzzzz' }, + }, + }; + assert.deepEqual( + normalizeFrontendOracle(contract, { + frontend: { + count: 0, + code: 'UNKNOWN_COMMAND', + fixText: null, + rawMessage: true, + totalErrors: 1, + }, + }), + { + channel: 'syntax', + count: 0, + code: 'UNKNOWN_COMMAND', + fixText: null, + rawMessage: true, + totalErrors: 1, + } + ); + assert.throws( + () => + normalizeFrontendOracle(contract, { + frontend: { count: 0, code: 'OTHER_ERROR' }, + }), + /does not match contract\.wiring\.code/ + ); +}); + test('lint and syntax frontend fields cannot cross channels', () => { assert.throws( () => diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs new file mode 100644 index 00000000000..6b482125211 --- /dev/null +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -0,0 +1,325 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + assertActiveShippingContracts, + buildCensus, + evaluateFrontendAssertions, + selectManifestContractNames, +} from '../run-frontend-contract.mjs'; + +const RANGE = { + startLine: 1, + startColumn: 11, + endLine: 1, + endColumn: 14, +}; + +test('exact lint assertions materialize the effective deterministic edit', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + ruleId: 'example-rule', + severity: 'warning', + message: 'Replace bad.', + range: RANGE, + fix: { + title: 'Replace bad', + text: 'good', + expectedText: 'bad', + }, + }, + ], + frontendOracle: { + severity: 'warning', + messageEquals: 'Replace bad.', + deterministicFix: { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'bad', + appliedQuery: 'source=t | good', + }, + aiAction: { offered: false }, + }, + decideAction: ({ hasDeterministicFix }) => ({ + kind: hasDeterministicFix ? 'deterministic' : 'ai', + commandId: 'ppl.lint.aiFix', + }), + }); + + assert.deepEqual(result.mismatches, []); + assert.deepEqual(result.assertions, { + severity: true, + message: true, + deterministicFix: true, + aiAction: true, + actionDecision: true, + }); + assert.deepEqual(result.deterministicFixActual, { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'bad', + appliedQuery: 'source=t | good', + }); +}); + +test('deterministic fixes require the production helper to choose the deterministic action', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + severity: 'warning', + message: 'Replace bad.', + range: RANGE, + fix: { title: 'Replace bad', text: 'good', expectedText: 'bad' }, + }, + ], + frontendOracle: { + deterministicFix: { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'bad', + appliedQuery: 'source=t | good', + }, + aiAction: { offered: false }, + }, + decideAction: () => ({ kind: 'none' }), + }); + + assert.equal(result.deterministicFixMatched, true); + assert.equal(result.aiActionMatched, true); + assert.equal(result.actionDecisionMatched, false); + assert.equal( + result.mismatches.find(({ field }) => field === 'actionDecision')?.actual[0], + 'none' + ); +}); + +test('AI action identity and exact messages produce field-specific mismatches', () => { + let decisionInput; + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + ruleId: 'example-rule', + severity: 'warning', + message: 'Different message.', + range: RANGE, + }, + ], + frontendOracle: { + messageEquals: 'Expected message.', + deterministicFix: { offered: false }, + aiAction: { offered: true, commandId: 'ppl.lint.aiFix' }, + }, + decideAction: (input) => { + decisionInput = input; + return { kind: 'ai', commandId: 'wrong.command' }; + }, + }); + + assert.deepEqual( + result.mismatches.map(({ field }) => field), + ['message', 'aiAction'] + ); + assert.deepEqual(result.aiActionActual, { + offered: true, + commandId: 'wrong.command', + }); + assert.equal(decisionInput.enableAIFeatures, true); + assert.equal(decisionInput.hasAiFixHandler, true); + assert.equal(decisionInput.aiAgentAvailableForSource, true); + assert.equal(decisionInput.aiFixEligible, true); +}); + +test('deterministic expectedText must match the source slice', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [ + { + message: 'Replace bad.', + range: RANGE, + fix: { + title: 'Replace bad', + text: 'good', + expectedText: 'stale', + }, + }, + ], + frontendOracle: { + deterministicFix: { + offered: true, + title: 'Replace bad', + text: 'good', + range: RANGE, + expectedText: 'stale', + appliedQuery: 'source=t | good', + }, + }, + }); + + assert.equal(result.deterministicFixMatched, false); + assert.equal(result.deterministicFixActual.expectedTextMatchesSource, false); +}); + +test('AI assertions fail closed when the production decision export is unavailable', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t', + matches: [], + frontendOracle: { aiAction: { offered: false } }, + }); + + assert.equal(result.aiActionMatched, false); + assert.equal(result.mismatches[0].field, 'aiAction'); + assert.equal(result.aiActionActual.unavailable, true); +}); + +test('AI assertions report production decision errors instead of passing absence', () => { + const result = evaluateFrontendAssertions({ + channel: 'lint', + query: 'source=t | bad', + matches: [{ message: 'Bad', range: RANGE }], + frontendOracle: { aiAction: { offered: false } }, + decideAction: () => { + throw new Error('decision failed'); + }, + }); + + assert.equal(result.aiActionMatched, false); + assert.match(result.aiActionActual.error, /decision failed/); + assert.equal(result.mismatches[0].field, 'aiAction'); +}); + +test('syntax suppression checks raw parser errors outside the suggestion code filter', () => { + const parserErrors = [ + { + code: 'PARSER_ERROR', + message: 'Unexpected command.', + rawMessage: "mismatched input 'zzzzzzzz'", + }, + ]; + const result = evaluateFrontendAssertions({ + channel: 'syntax', + query: 'source=t | zzzzzzzz', + matches: [], + allFrontendFindings: parserErrors, + frontendOracle: { + fixText: null, + rawMessage: true, + totalErrors: 1, + }, + }); + + assert.deepEqual(result.mismatches, []); + assert.deepEqual(result.assertions, { + syntaxFix: true, + rawParserError: true, + totalErrors: true, + }); + + const withUnexpectedFix = evaluateFrontendAssertions({ + channel: 'syntax', + query: 'source=t | zzzzzzzz', + matches: [], + allFrontendFindings: [ + { + ...parserErrors[0], + fix: { title: 'Rewrite', text: 'where' }, + }, + ], + frontendOracle: { fixText: null, rawMessage: true }, + }); + assert.equal(withUnexpectedFix.syntaxFixMatched, false); + assert.equal(withUnexpectedFix.mismatches[0].field, 'syntaxFix'); +}); + +test('dormant manifest contracts are opt-in and remain tagged report-only', () => { + const manifest = { + contracts: ['active.spec.json'], + dormantContracts: ['dormant.spec.json'], + }; + assert.deepEqual(selectManifestContractNames(manifest), [ + { name: 'active.spec.json', reportOnly: false }, + ]); + assert.deepEqual(selectManifestContractNames(manifest, true), [ + { name: 'active.spec.json', reportOnly: false }, + { name: 'dormant.spec.json', reportOnly: true }, + ]); + assert.throws( + () => + selectManifestContractNames( + { + contracts: ['same.spec.json'], + dormantContracts: ['same.spec.json'], + }, + true + ), + /cannot be both active and dormant/ + ); +}); + +test('discovery mode accepts legacy generated specs without shipping oracles', () => { + const contract = { + file: 'generated.discovery.spec.json', + spec: { + schemaVersion: 3, + ruleId: 'generated-rule', + queries: { trigger: { role: 'trigger', query: 'source=t' } }, + expectations: [{ version: '', queries: { trigger: { detectorCount: 0 } } }], + }, + }; + + assert.doesNotThrow(() => + assertActiveShippingContracts([contract], { discovery: true }) + ); + assert.throws( + () => assertActiveShippingContracts([contract]), + /active shipping contracts must use schemaVersion 4/ + ); +}); + +test('shipping census rejects duplicate detector IDs and syntax features in the catalog', () => { + const syntaxContract = { + file: 'command-suggestion.spec.json', + spec: { + schemaVersion: 4, + ruleId: 'command-suggestion', + channel: 'syntax', + }, + }; + const census = buildCensus( + [syntaxContract], + { + contracts: ['command-suggestion.spec.json'], + defaultError: [], + requiredSyntaxFeatures: ['command-suggestion.spec.json'], + }, + [ + { id: 'command-suggestion', enabled: false, severity: 'error' }, + { id: 'duplicate-rule', enabled: false, severity: 'info' }, + { id: 'duplicate-rule', enabled: false, severity: 'info' }, + ] + ); + + assert.ok(census.problems.some((problem) => /duplicate rule IDs/.test(problem))); + assert.ok( + census.problems.some((problem) => /outside the detector catalog/.test(problem)) + ); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index a8c29560fb1..1a9abf03b0b 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -266,6 +266,37 @@ function normalizeDetectorReport(detector, target) { if (typeof entry.messageMatched !== 'boolean') { throw new TypeError(`detector report row ${key}.messageMatched must be a boolean`); } + for (const field of [ + 'deterministicFixMatched', + 'aiActionMatched', + 'actionDecisionMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (entry[field] !== undefined && typeof entry[field] !== 'boolean') { + throw new TypeError(`detector report row ${key}.${field} must be a boolean`); + } + } + if (entry.assertions !== undefined) { + if ( + !entry.assertions || + typeof entry.assertions !== 'object' || + Array.isArray(entry.assertions) + ) { + throw new TypeError(`detector report row ${key}.assertions must be a JSON object`); + } + for (const [field, matched] of Object.entries(entry.assertions)) { + if (typeof matched !== 'boolean') { + throw new TypeError( + `detector report row ${key}.assertions.${field} must be a boolean` + ); + } + } + } + if (entry.mismatches !== undefined && !Array.isArray(entry.mismatches)) { + throw new TypeError(`detector report row ${key}.mismatches must be a JSON array`); + } } results.set(key, entry); } @@ -578,6 +609,16 @@ function detectorParityValue(entry) { typeof entry.rawMessageMatched === 'boolean' ? entry.rawMessageMatched : undefined, totalErrorsMatched: typeof entry.totalErrorsMatched === 'boolean' ? entry.totalErrorsMatched : undefined, + deterministicFixMatched: + typeof entry.deterministicFixMatched === 'boolean' + ? entry.deterministicFixMatched + : undefined, + aiActionMatched: + typeof entry.aiActionMatched === 'boolean' ? entry.aiActionMatched : undefined, + assertions: entry.assertions, + mismatches: entry.mismatches, + deterministicFix: entry.deterministicFix, + aiAction: entry.aiAction, code: entry.code, codes: entry.codes, totalErrors: entry.totalErrors, @@ -596,6 +637,9 @@ function assertDetectorParity(pair) { for (const key of keys) { const standardRow = standard.get(key); const analyticsRow = analytics.get(key); + if (standardRow?.reportOnly === true || analyticsRow?.reportOnly === true) { + continue; + } if (!standardRow || !analyticsRow) { fatal( `detector parity failed for ${key}: standard row=${!!standardRow}, ` + @@ -852,10 +896,61 @@ function readBackendObservation(backendEntry, detectorResult) { backendMismatch: verdict.backendMismatch, severityMatched: detectorResult ? detectorResult.severityMatched : undefined, messageMatched: detectorResult ? detectorResult.messageMatched : undefined, + deterministicFixMatched: detectorResult + ? detectorResult.deterministicFixMatched + : undefined, + aiActionMatched: detectorResult ? detectorResult.aiActionMatched : undefined, + fixMatched: detectorResult ? detectorResult.fixMatched : undefined, + rawMessageMatched: detectorResult ? detectorResult.rawMessageMatched : undefined, + totalErrorsMatched: detectorResult + ? detectorResult.totalErrorsMatched + : undefined, + assertions: detectorResult ? detectorResult.assertions : undefined, + mismatches: detectorResult ? detectorResult.mismatches : undefined, }, }; } +function failedExtendedFrontendAssertions(entry, frontendOracle) { + if (!entry) { + return []; + } + const failures = new Set(); + for (const field of [ + 'deterministicFixMatched', + 'aiActionMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (entry[field] === false) { + failures.add(field); + } + } + for (const [field, matched] of Object.entries(entry.assertions || {})) { + if ( + matched === false && + field !== 'count' && + field !== 'severity' && + (field !== 'message' || frontendOracle.messageEquals !== undefined) + ) { + failures.add(field); + } + } + for (const mismatch of entry.mismatches || []) { + const field = mismatch && mismatch.field; + if ( + typeof field === 'string' && + field !== 'count' && + field !== 'severity' && + (field !== 'message' || frontendOracle.messageEquals !== undefined) + ) { + failures.add(field); + } + } + return [...failures].sort(); +} + /** * Check an out-of-scope rule for the one drift that still matters there: the * engine rejects a trigger query, but the rule's `appliesTo` excludes this @@ -1036,12 +1131,16 @@ function main() { const missingContracts = auditDefaultErrorCensus(legs, specs, enforcedRules); const shippingCensus = auditShippingCensus(legs, specs, manifest); const blockCensusDrift = !shippingCensus.available || shippingCensus.enforced; + shippingCensus.blocking = blockCensusDrift; + const blockingShippingCensusProblems = blockCensusDrift + ? shippingCensus.problems + : []; for (const entry of missingContracts) { entry.blocking = blockCensusDrift; } if (!shippingCensus.passed) { for (const problem of shippingCensus.problems) { - log(`CENSUS REPORT-ONLY: ${problem}`); + log(`CENSUS ${blockCensusDrift ? 'ENFORCED' : 'REPORT-ONLY'}: ${problem}`); } } @@ -1306,6 +1405,51 @@ function main() { ruleNotApplicable++; continue; } + const failedFrontendAssertions = failedExtendedFrontendAssertions( + detectorResult, + oracleSelection.frontend + ); + if (failedFrontendAssertions.length > 0) { + addDrift( + { + ruleId, + version: leg.version, + driftVersion: leg.version, + queryName, + role, + query, + driftClass: 'frontend-contract-mismatch', + evidence: + `${ruleId} @ ${leg.version} [${queryName}]: frontend assertion(s) failed: ` + + failedFrontendAssertions.join(', '), + frontendAssertions: failedFrontendAssertions, + frontendMismatches: detectorResult.mismatches || [], + remediation: { + action: 'update-detector', + target: + spec.detectorPath || + `packages/osd-monaco/src/ppl/lint/rules/${ruleId.replace(/-/g, '_')}.ts`, + detail: + `Reproduce this contract query against the reported OSD commit and candidate ` + + `grammar. Restore the exact message/action/fix behavior, or update the contract ` + + `only after confirming an intentional product change.`, + }, + }, + leg, + { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + } + ); + ruleDrifts++; + compared++; + if (role === 'trigger') { + triggersCompared++; + } + continue; + } const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { // No comparable pair, so there is nothing to classify. Attempting it @@ -1627,6 +1771,7 @@ function main() { coverageHoles.filter((h) => h.enforced && !h.blocking).length, missingContractCount: missingContracts.length, blockingMissingContractCount: blockingMissingContracts.length, + blockingShippingCensusProblems: blockingShippingCensusProblems.length, enforcedInconclusive: enforcedInconclusive.length, // An inconclusive default-error rule fails too: "we could not check" must // never render as "it is fine". @@ -1634,6 +1779,7 @@ function main() { enforcedDrifts.length === 0 && enforcedHoles.length === 0 && blockingMissingContracts.length === 0 && + blockingShippingCensusProblems.length === 0 && enforcedInconclusive.length === 0, }, }; @@ -1668,7 +1814,8 @@ function main() { console.error( `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + `${enforcedHoles.length} coverage hole(s), ${blockingMissingContracts.length} unvalidated ` + - `default-error rule(s) and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` + `default-error rule(s), ${blockingShippingCensusProblems.length} shipping census problem(s), ` + + `and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` ); process.exit(1); } @@ -1702,6 +1849,11 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { if (report.result.missingContractCount) { reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); } + if (report.result.blockingShippingCensusProblems) { + reasons.push( + `${report.result.blockingShippingCensusProblems} shipping census problem(s)` + ); + } lines.push( // Name the surface when a leg is not the default runtime-bundle one, so a // reader knows a column speaks for OSD's compiled grammar rather than the @@ -1786,6 +1938,18 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { lines.push(''); } + if (report.shippingCensus && !report.shippingCensus.passed) { + lines.push('### Shipping census'); + lines.push(''); + for (const problem of report.shippingCensus.problems || []) { + lines.push( + `- ${report.shippingCensus.blocking ? '**ENFORCED:**' : '**REPORT ONLY:**'} ${problem}. ` + + `Align \`manifest.json\` with the approved OSD shipping catalog.` + ); + } + lines.push(''); + } + if (coverageHoles.length > 0) { lines.push('### Coverage holes'); lines.push(''); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs index 9c9d6bf2c16..6cb2e031a9c 100644 --- a/scripts/ppl-lint/annotate.mjs +++ b/scripts/ppl-lint/annotate.mjs @@ -215,6 +215,25 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r }); } + const shippingCensus = report.shippingCensus; + if (shippingCensus && shippingCensus.passed === false) { + const manifestText = contractText('manifest.json'); + for (const problem of shippingCensus.problems || []) { + const blocking = shippingCensus.blocking !== false; + annotations.push({ + level: blocking ? 'error' : 'warning', + file: contractRepoPath(contractsDir, 'manifest.json', workspace), + line: findJsonKeyLine(manifestText, manifestKeyFor(problem)), + title: 'PPL lint shipping census mismatch', + message: + `${problem}\n` + + (blocking + ? 'FIX: align the active SQL manifest with the approved OSD shipping catalog.' + : 'REPORT ONLY: align the active SQL manifest with the approved OSD shipping catalog before enabling census enforcement.'), + }); + } + } + return annotations; } diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index 4d0cd183503..5d875e593bb 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -46,6 +46,47 @@ function readJson(file, errors) { return undefined; } +function failedFrontendAssertions(entry) { + const failures = new Set(); + for (const [field, label] of [ + ['severityMatched', 'severity'], + ['messageMatched', 'message'], + ]) { + if (entry[field] !== true) { + failures.add(label); + } + } + for (const [field, label] of [ + ['deterministicFixMatched', 'deterministic-fix'], + ['aiActionMatched', 'AI-action'], + ['actionDecisionMatched', 'action-decision'], + ['fixMatched', 'syntax-fix'], + ['rawMessageMatched', 'raw-parser-error'], + ['totalErrorsMatched', 'total-error'], + ]) { + if (entry[field] === false) { + failures.add(label); + } + } + if ( + entry.assertions && + typeof entry.assertions === 'object' && + !Array.isArray(entry.assertions) + ) { + for (const [field, matched] of Object.entries(entry.assertions)) { + if (matched === false) { + failures.add(field); + } + } + } + for (const mismatch of Array.isArray(entry.mismatches) ? entry.mismatches : []) { + if (mismatch && typeof mismatch.field === 'string') { + failures.add(mismatch.field); + } + } + return [...failures].sort(); +} + function main() { const artifactErrors = []; const targetRaw = readJson(path.join(ARTIFACTS, 'target.json'), artifactErrors) || {}; @@ -118,6 +159,7 @@ function main() { artifactErrors.push('detector-report.json must contain a non-empty results array'); } const detectorKeys = new Set(); + const reportOnlyDetectorKeys = new Set(); for (const entry of Array.isArray(detector.results) ? detector.results : []) { const key = `${entry.ruleId}::${entry.queryName}`; if (!entry.ruleId || !entry.queryName) { @@ -128,6 +170,10 @@ function main() { artifactErrors.push(`detector-report.json contains duplicate row ${key}`); } detectorKeys.add(key); + if (entry.reportOnly === true) { + reportOnlyDetectorKeys.add(key); + continue; + } if (entry.executionBackend !== executionBackend) { artifactErrors.push( `detector row ${key} executionBackend ${JSON.stringify(entry.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` @@ -143,23 +189,28 @@ function main() { `detector row ${key} count mismatch: expected ${entry.expected}, got ${entry.actual}` ); } - if (entry.severityMatched !== true) { - artifactErrors.push(`detector row ${key} did not match its severity assertion`); + if ( + entry.assertions !== undefined && + (!entry.assertions || + typeof entry.assertions !== 'object' || + Array.isArray(entry.assertions) || + Object.values(entry.assertions).some((matched) => typeof matched !== 'boolean')) + ) { + artifactErrors.push(`detector row ${key} assertions must contain only booleans`); } - if (entry.messageMatched !== true) { - artifactErrors.push(`detector row ${key} did not match its message assertion`); + if (entry.mismatches !== undefined && !Array.isArray(entry.mismatches)) { + artifactErrors.push(`detector row ${key} mismatches must be an array`); } - for (const [field, label] of [ - ['fixMatched', 'fix'], - ['rawMessageMatched', 'raw-message'], - ['totalErrorsMatched', 'total-error'], - ]) { - if (entry[field] === false) { - artifactErrors.push(`detector row ${key} did not match its ${label} assertion`); - } + for (const assertion of failedFrontendAssertions(entry)) { + artifactErrors.push( + `detector row ${key} did not match its ${assertion} assertion` + ); } } for (const [key, entry] of backendByKey) { + if (reportOnlyDetectorKeys.has(key)) { + continue; + } if (!detectorKeys.has(key)) { artifactErrors.push(`backend row ${key} has no matching detector row`); } @@ -192,7 +243,11 @@ function main() { // The selected validation set is the set of rules the detector run actually // evaluated (post schedule filtering). const validationSet = Array.from( - new Set((detector.results || []).map((r) => r.ruleId)) + new Set( + (detector.results || []) + .filter((row) => row.reportOnly !== true) + .map((row) => row.ruleId) + ) ).sort(); const manifest = { @@ -286,14 +341,17 @@ function writeSummary(manifest, detector, backend) { ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` : 'accepted'; const ok = - r.actual === r.expected && - r.severityMatched === true && - r.messageMatched === true && - !!be && - be.outcome === 'pass'; + r.reportOnly === true + ? undefined + : r.actual === r.expected && + failedFrontendAssertions(r).length === 0 && + !!be && + be.outcome === 'pass'; lines.push( `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + - `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ok ? 'Pass' : 'Fail'} |` + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ + ok === undefined ? 'Report only' : ok ? 'Pass' : 'Fail' + } |` ); } diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs index ca3c5a809c2..b1da53617fa 100644 --- a/scripts/ppl-lint/contract-schema.mjs +++ b/scripts/ppl-lint/contract-schema.mjs @@ -8,7 +8,14 @@ const CONTRACT_SCHEMA_VERSIONS = new Set([3, 4]); const APPLICABLE_BACKEND_KINDS = new Set(['rejection', 'result-shape', 'advisory']); const CONTRACT_CHANNELS = new Set(['lint', 'syntax']); const QUERY_ROLES = new Set(['trigger', 'control', 'suppression-control']); -const LINT_FRONTEND_FIELDS = new Set(['count', 'severity', 'matchMessage']); +const LINT_V3_FRONTEND_FIELDS = new Set(['count', 'severity', 'matchMessage']); +const LINT_V4_FRONTEND_FIELDS = new Set([ + 'count', + 'severity', + 'messageEquals', + 'deterministicFix', + 'aiAction', +]); const SYNTAX_FRONTEND_FIELDS = new Set([ 'count', 'code', @@ -40,6 +47,13 @@ function requireNonEmptyString(value, label) { return value; } +function requireString(value, label) { + if (typeof value !== 'string') { + throw new TypeError(`${label} must be a string.`); + } + return value; +} + function requireNonNegativeInteger(value, label) { if (!Number.isInteger(value) || value < 0) { throw new TypeError(`${label} must be a non-negative integer.`); @@ -61,6 +75,85 @@ function assertKnownKeys(value, allowed, label) { } } +function normalizeRange(value, label) { + const range = requireObject(value, label); + assertKnownKeys( + range, + new Set(['startLine', 'startColumn', 'endLine', 'endColumn']), + label + ); + for (const field of ['startLine', 'endLine']) { + if (!Number.isInteger(range[field]) || range[field] < 1) { + throw new TypeError(`${label}.${field} must be a positive integer.`); + } + } + for (const field of ['startColumn', 'endColumn']) { + requireNonNegativeInteger(range[field], `${label}.${field}`); + } + if ( + range.endLine < range.startLine || + (range.endLine === range.startLine && range.endColumn < range.startColumn) + ) { + throw new Error(`${label} must end at or after its start.`); + } + return { + startLine: range.startLine, + startColumn: range.startColumn, + endLine: range.endLine, + endColumn: range.endColumn, + }; +} + +function normalizeDeterministicFix(value, label) { + const fix = requireObject(value, label); + if (typeof fix.offered !== 'boolean') { + throw new TypeError(`${label}.offered must be a boolean.`); + } + const allowed = new Set([ + 'offered', + 'title', + 'text', + 'range', + 'expectedText', + 'appliedQuery', + ]); + assertKnownKeys(fix, allowed, label); + if (!fix.offered) { + if (Object.keys(fix).length !== 1) { + throw new Error(`${label} must contain only offered when no fix is expected.`); + } + return { offered: false }; + } + + const normalized = { + offered: true, + title: requireNonEmptyString(fix.title, `${label}.title`), + text: requireString(fix.text, `${label}.text`), + range: normalizeRange(fix.range, `${label}.range`), + expectedText: requireString(fix.expectedText, `${label}.expectedText`), + appliedQuery: requireString(fix.appliedQuery, `${label}.appliedQuery`), + }; + return normalized; +} + +function normalizeAiAction(value, label) { + const action = requireObject(value, label); + if (typeof action.offered !== 'boolean') { + throw new TypeError(`${label}.offered must be a boolean.`); + } + assertKnownKeys(action, new Set(['offered', 'commandId']), label); + if (!action.offered) { + if (Object.keys(action).length !== 1) { + throw new Error(`${label} must contain only offered when no AI action is expected.`); + } + return { offered: false }; + } + return { + offered: true, + commandId: requireNonEmptyString(action.commandId, `${label}.commandId`), + }; +} + export function contractChannel(spec) { requireObject(spec, 'contract'); const channel = spec.channel === undefined ? 'lint' : spec.channel; @@ -75,9 +168,10 @@ export function contractChannel(spec) { /** * Normalize legacy detector fields and channel-specific frontend assertions. * - * Callers continue to receive `count`, `severity`, and `matchMessage` for lint - * contracts while syntax contracts can assert stable parser error identity, - * quick-fix text, raw-message preservation, and the total syntax error census. + * Schema-v3 lint contracts retain substring messages. Schema-v4 lint contracts + * assert exact messages and exact deterministic/AI action availability. Syntax + * contracts assert stable parser error identity, quick-fix presence or absence, + * raw-message preservation, and the total syntax error census. */ export function normalizeFrontendOracle(spec, queryExpectation) { const channel = contractChannel(spec); @@ -103,33 +197,72 @@ export function normalizeFrontendOracle(spec, queryExpectation) { : { count: queryExpectation.detectorCount, severity: queryExpectation.severity, - matchMessage: queryExpectation.matchMessage, + ...(queryExpectation.matchMessage !== undefined + ? { matchMessage: queryExpectation.matchMessage } + : {}), + ...(queryExpectation.messageEquals !== undefined + ? { messageEquals: queryExpectation.messageEquals } + : {}), + ...(queryExpectation.deterministicFix !== undefined + ? { deterministicFix: queryExpectation.deterministicFix } + : {}), + ...(queryExpectation.aiAction !== undefined + ? { aiAction: queryExpectation.aiAction } + : {}), }; - assertKnownKeys(frontend, LINT_FRONTEND_FIELDS, `[${spec.ruleId}] frontend`); + const allowed = + spec.schemaVersion === 3 ? LINT_V3_FRONTEND_FIELDS : LINT_V4_FRONTEND_FIELDS; + assertKnownKeys(frontend, allowed, `[${spec.ruleId}] frontend`); requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); assertOptionalString(frontend.severity, `[${spec.ruleId}] frontend.severity`); - if ( - frontend.matchMessage !== undefined && - typeof frontend.matchMessage !== 'string' - ) { - throw new TypeError( - `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + if (spec.schemaVersion === 3) { + if ( + frontend.matchMessage !== undefined && + typeof frontend.matchMessage !== 'string' + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.matchMessage must be a string when present.` + ); + } + } else { + assertOptionalString( + frontend.messageEquals, + `[${spec.ruleId}] frontend.messageEquals` ); } if ( hasFrontend && - (queryExpectation.severity !== undefined || - queryExpectation.matchMessage !== undefined) + ['severity', 'matchMessage', 'messageEquals', 'deterministicFix', 'aiAction'].some( + (field) => queryExpectation[field] !== undefined + ) ) { throw new Error( - `[${spec.ruleId}] severity and matchMessage must be nested under frontend when frontend is present.` + `[${spec.ruleId}] lint assertions must be nested under frontend when frontend is present.` ); } return { channel, count: frontend.count, severity: frontend.severity, - matchMessage: frontend.matchMessage, + ...(spec.schemaVersion === 3 + ? { matchMessage: frontend.matchMessage } + : { + messageEquals: frontend.messageEquals, + deterministicFix: + frontend.deterministicFix === undefined + ? undefined + : normalizeDeterministicFix( + frontend.deterministicFix, + `[${spec.ruleId}] frontend.deterministicFix` + ), + aiAction: + frontend.aiAction === undefined + ? undefined + : normalizeAiAction( + frontend.aiAction, + `[${spec.ruleId}] frontend.aiAction` + ), + }), }; } @@ -145,7 +278,15 @@ export function normalizeFrontendOracle(spec, queryExpectation) { assertKnownKeys(frontend, SYNTAX_FRONTEND_FIELDS, `[${spec.ruleId}] frontend`); requireNonNegativeInteger(frontend.count, `[${spec.ruleId}] frontend.count`); requireNonEmptyString(frontend.code, `[${spec.ruleId}] frontend.code`); - assertOptionalString(frontend.fixText, `[${spec.ruleId}] frontend.fixText`); + if ( + frontend.fixText !== undefined && + frontend.fixText !== null && + (typeof frontend.fixText !== 'string' || frontend.fixText.length === 0) + ) { + throw new TypeError( + `[${spec.ruleId}] frontend.fixText must be a non-empty string or null when present.` + ); + } if ( frontend.matchMessage !== undefined && typeof frontend.matchMessage !== 'string' @@ -170,9 +311,66 @@ export function normalizeFrontendOracle(spec, queryExpectation) { ); } } + if (spec.wiring && frontend.code !== spec.wiring.code) { + throw new Error( + `[${spec.ruleId}] frontend.code ${describe(frontend.code)} does not match ` + + `contract.wiring.code ${describe(spec.wiring.code)}.` + ); + } return { channel, ...frontend }; } +/** + * Active shipping contracts are stricter than dormant compatibility contracts: + * every lint finding pins its exact message and action mode, every lint control + * pins action absence, and every syntax case pins fix/raw-error/error-count state. + */ +export function assertShippingFrontendOracles(spec, expectation) { + if (spec.schemaVersion !== 4) { + throw new Error(`[${spec.ruleId}] active shipping contracts must use schemaVersion 4.`); + } + for (const queryName of assertExactQueryCoverage(spec, expectation)) { + const frontend = normalizeFrontendOracle(spec, expectation.queries[queryName]); + const label = `[${spec.ruleId}/${queryName}] frontend`; + + if (frontend.channel === 'syntax') { + if (frontend.fixText === undefined) { + throw new Error(`${label}.fixText must explicitly assert fix presence or absence.`); + } + if (frontend.rawMessage === undefined) { + throw new Error(`${label}.rawMessage must be explicitly asserted.`); + } + if (frontend.totalErrors === undefined) { + throw new Error(`${label}.totalErrors must be explicitly asserted.`); + } + if (frontend.count > 0 && frontend.matchMessage === undefined) { + throw new Error(`${label}.matchMessage is required for a syntax finding.`); + } + continue; + } + + if (frontend.deterministicFix === undefined) { + throw new Error(`${label}.deterministicFix must be explicitly asserted.`); + } + if (frontend.aiAction === undefined) { + throw new Error(`${label}.aiAction must be explicitly asserted.`); + } + if (frontend.count > 0) { + if (frontend.severity === undefined) { + throw new Error(`${label}.severity is required for a lint finding.`); + } + if (frontend.messageEquals === undefined) { + throw new Error(`${label}.messageEquals is required for a lint finding.`); + } + if (frontend.deterministicFix.offered && frontend.aiAction.offered) { + throw new Error(`${label} cannot offer deterministic and AI actions together.`); + } + } else if (frontend.deterministicFix.offered || frontend.aiAction.offered) { + throw new Error(`${label} must not offer actions when no finding is expected.`); + } + } +} + export function normalizeLintWiring(ruleId, wiring, label = 'wiring') { requireNonEmptyString(ruleId, `${label}.id`); requireObject(wiring, label); @@ -453,11 +651,7 @@ export function resolveBackendOracle(spec, queryExpectation, executionBackend) { const schemaVersion = assertContractSchema(spec); assertExecutionBackend(executionBackend); const frontend = normalizeFrontendOracle(spec, queryExpectation); - const detector = { - count: frontend.count, - severity: frontend.severity, - matchMessage: frontend.matchMessage, - }; + const { channel: _channel, ...detector } = frontend; let oracle; let missingReason; diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 305f55fdbb6..68438d4632e 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -77,10 +77,12 @@ import fs from 'fs'; import path from 'path'; import { createRequire } from 'module'; +import { fileURLToPath } from 'url'; import { assertContractSchema, assertExactQueryCoverage, + assertShippingFrontendOracles, classifyBackendReportRow, contractChannel, indexBackendReport, @@ -105,6 +107,8 @@ const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; // supports older checkouts (that is the coverage it adds), so fall back to the // source module, which `setup_node_env` transpiles on require anyway. const CATALOG_SOURCE_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; +const ACTION_DECISION_MODULE = + 'packages/osd-monaco/src/ppl/lint/action_decision'; const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; /** @@ -169,6 +173,54 @@ function loadContractFile(file) { return undefined; // unreachable } +export function assertActiveShippingContracts(contracts, { discovery = false } = {}) { + if (discovery) { + return; + } + for (const { file, spec } of contracts) { + for (const expectation of spec.expectations) { + try { + assertShippingFrontendOracles(spec, expectation); + } catch (error) { + throw new Error(`Invalid active shipping contract ${file}: ${error.message}`); + } + } + } +} + +export function selectManifestContractNames(manifest, includeDormant = false) { + if (!Array.isArray(manifest.contracts)) { + throw new TypeError('manifest.json must have a "contracts" array of file names.'); + } + if (new Set(manifest.contracts).size !== manifest.contracts.length) { + throw new Error('manifest.json "contracts" contains duplicate file names.'); + } + const active = manifest.contracts.map((name) => ({ name, reportOnly: false })); + if (!includeDormant) { + return active; + } + if (!Array.isArray(manifest.dormantContracts)) { + throw new TypeError( + 'PPL_LINT_INCLUDE_DORMANT=1 requires manifest.json "dormantContracts" to be an array.' + ); + } + if (new Set(manifest.dormantContracts).size !== manifest.dormantContracts.length) { + throw new Error('manifest.json "dormantContracts" contains duplicate file names.'); + } + const activeNames = new Set(manifest.contracts); + for (const name of manifest.dormantContracts) { + if (activeNames.has(name)) { + throw new Error( + `manifest.json contract "${name}" cannot be both active and dormant.` + ); + } + } + return [ + ...active, + ...manifest.dormantContracts.map((name) => ({ name, reportOnly: true })), + ]; +} + /** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ function loadContracts() { const dir = process.env.PPL_LINT_CONTRACT_DIR; @@ -179,8 +231,16 @@ function loadContracts() { fatal(`Contract file not found: ${single}`); } const contract = loadContractFile(single); + try { + assertActiveShippingContracts([contract], { + discovery: process.env.PPL_LINT_DISCOVERY === '1', + }); + } catch (error) { + fatal(error.message); + } return { - contracts: [contract], + contracts: [{ ...contract, reportOnly: false }], + activeContracts: [contract], manifest: { contracts: [path.basename(single)] }, manifestPath: '', }; @@ -195,6 +255,7 @@ function loadContracts() { const manifestPath = path.join(dir, 'manifest.json'); let files; + let selectedFiles; let manifest; if (fs.existsSync(manifestPath)) { try { @@ -202,28 +263,69 @@ function loadContracts() { } catch (error) { fatal(`Invalid contract manifest ${manifestPath}: ${error.message}`); } - if (!Array.isArray(manifest.contracts)) { - fatal(`manifest.json must have a "contracts" array of file names.`); - } - if (new Set(manifest.contracts).size !== manifest.contracts.length) { - fatal(`manifest.json "contracts" contains duplicate file names.`); + try { + selectedFiles = selectManifestContractNames( + manifest, + process.env.PPL_LINT_INCLUDE_DORMANT === '1' + ); + } catch (error) { + fatal(error.message); } - files = manifest.contracts.map((name) => path.join(dir, name)); + files = selectedFiles.map(({ name }) => path.join(dir, name)); } else { files = fs .readdirSync(dir) .filter((f) => f.endsWith('.spec.json')) .sort() .map((f) => path.join(dir, f)); + selectedFiles = files.map((file) => ({ + name: path.basename(file), + reportOnly: false, + })); } - const contracts = files.map((file) => { + const contracts = files.map((file, index) => { if (!fs.existsSync(file)) { fatal(`Contract referenced by manifest not found: ${file}`); } - return loadContractFile(file); + return { + ...loadContractFile(file), + reportOnly: selectedFiles[index].reportOnly, + }; }); - return { contracts, manifest: manifest || { contracts: files.map(path.basename) }, manifestPath }; + const activeContracts = contracts + .filter(({ reportOnly }) => !reportOnly) + .map(({ file, spec }) => ({ file, spec })); + try { + assertActiveShippingContracts(activeContracts, { + discovery: process.env.PPL_LINT_DISCOVERY === '1', + }); + } catch (error) { + fatal(error.message); + } + return { + contracts, + activeContracts, + manifest: manifest || { contracts: files.map(path.basename) }, + manifestPath, + }; +} + +function resolveActionDecision(module) { + if (!module) { + return undefined; + } + for (const name of [ + 'decidePPLLintAction', + 'decidePPLDiagnosticAction', + 'getPPLDiagnosticActionDecision', + 'decideDiagnosticAction', + ]) { + if (typeof module[name] === 'function') { + return module[name]; + } + } + return undefined; } function loadOsd() { @@ -260,6 +362,7 @@ function loadOsd() { // so a checkout without the built export can still run the compiled surface. const catalogModule = resolveOsd(CATALOG_MODULE, { optional: true }) || resolveOsd(CATALOG_SOURCE_MODULE); + const actionDecisionModule = resolveOsd(ACTION_DECISION_MODULE, { optional: true }); const { getBundledCatalog } = catalogModule; const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); if (typeof getBundledCatalog !== 'function') { @@ -276,6 +379,7 @@ function loadOsd() { fatal(`PPLLanguageAnalyzer not found in ${ANALYZER_MODULE}.`); } const analyzer = new PPLLanguageAnalyzer(); + const headless = resolveOsd(HEADLESS_MODULE, { optional: true }); return { surface: SURFACE, // Same (query, grammar, context) shape as the bundle path so the main loop @@ -286,6 +390,10 @@ function loadOsd() { }, getBundledCatalog, getDetector, + decideAction: + resolveActionDecision(headless) || + resolveActionDecision(catalogModule) || + resolveActionDecision(actionDecisionModule), osdRoot, }; } @@ -310,6 +418,10 @@ function loadOsd() { deserializeBundleOrThrow, lintQuery: lintQueryWithBundle, validateSyntax, + decideAction: + resolveActionDecision(headless) || + resolveActionDecision(catalogModule) || + resolveActionDecision(actionDecisionModule), getBundledCatalog, getDetector, osdRoot, @@ -486,6 +598,9 @@ function selectExpectation(spec, version, isCalcite, failures, { allowMissing = function checkWiring(spec, catalog, getDetector, failures) { const { ruleId, wiring } = spec; if (contractChannel(spec) === 'syntax') { + if (!wiring) { + failures.push(`[${ruleId}] contract.wiring is required for strict syntax wiring.`); + } return { id: ruleId, syntaxCode: wiring && wiring.code }; } const entry = catalog.find((c) => c.id === ruleId); @@ -494,7 +609,8 @@ function checkWiring(spec, catalog, getDetector, failures) { return undefined; } if (!wiring) { - return entry; // no wiring block to assert + failures.push(`[${ruleId}] contract.wiring is required for strict catalog comparison.`); + return entry; } let expected; @@ -564,11 +680,300 @@ function buildContext(spec, engineVersion) { return context; } +function rangeOffsets(query, range) { + const lineStarts = [0]; + for (let index = 0; index < query.length; index += 1) { + if (query[index] === '\n') { + lineStarts.push(index + 1); + } + } + const offset = (line, column) => { + const lineStart = lineStarts[line - 1]; + if (lineStart === undefined) { + throw new Error(`range line ${line} is outside a ${lineStarts.length}-line query`); + } + const lineEnd = lineStarts[line] === undefined ? query.length : lineStarts[line] - 1; + if (lineStart + column > lineEnd) { + throw new Error(`range column ${column} is outside query line ${line}`); + } + return lineStart + column; + }; + return { + start: offset(range.startLine, range.startColumn), + end: offset(range.endLine, range.endColumn), + }; +} + +function materializeDeterministicFix(query, diagnostic) { + if (!diagnostic.fix) { + return undefined; + } + const range = diagnostic.fix.range || diagnostic.range; + const { start, end } = rangeOffsets(query, range); + const sourceText = query.slice(start, end); + const expectedTextMatchesSource = + diagnostic.fix.expectedText === undefined || + diagnostic.fix.expectedText === sourceText; + return { + offered: true, + title: diagnostic.fix.title, + text: diagnostic.fix.text, + range: { + startLine: range.startLine, + startColumn: range.startColumn, + endLine: range.endLine, + endColumn: range.endColumn, + }, + ...(diagnostic.fix.expectedText !== undefined + ? { expectedText: diagnostic.fix.expectedText } + : {}), + ...(!expectedTextMatchesSource + ? { expectedTextMatchesSource: false } + : {}), + appliedQuery: query.slice(0, start) + diagnostic.fix.text + query.slice(end), + }; +} + +function normalizeActionDecision(decision) { + if (typeof decision === 'string') { + return { kind: decision }; + } + if (!decision || typeof decision !== 'object' || Array.isArray(decision)) { + return { kind: 'invalid', value: decision }; + } + return { + kind: decision.kind || decision.type || decision.action, + commandId: decision.commandId || (decision.command && decision.command.id), + }; +} + +function exactEqual(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function evaluateFrontendAssertions({ + channel, + query, + matches, + allFrontendFindings = matches, + frontendOracle, + decideAction, +}) { + const assertions = {}; + const mismatches = []; + const record = (field, matched, expected, actual) => { + assertions[field] = matched; + if (!matched) { + mismatches.push({ field, expected, actual }); + } + return matched; + }; + + const severityMatched = + channel === 'syntax' || + !frontendOracle.severity || + matches.length === 0 || + matches.every((finding) => finding.severity === frontendOracle.severity); + if (channel === 'lint' && frontendOracle.severity !== undefined) { + record( + 'severity', + severityMatched, + frontendOracle.severity, + matches.map((finding) => finding.severity) + ); + } + + let messageMatched = true; + if (frontendOracle.matchMessage !== undefined) { + messageMatched = matches.some((finding) => + String(finding.message || '').includes(frontendOracle.matchMessage) + ); + record( + 'message', + messageMatched, + { contains: frontendOracle.matchMessage }, + matches.map((finding) => finding.message) + ); + } else if (frontendOracle.messageEquals !== undefined) { + messageMatched = + matches.length > 0 && + matches.every((finding) => finding.message === frontendOracle.messageEquals); + record( + 'message', + messageMatched, + { equals: frontendOracle.messageEquals }, + matches.map((finding) => finding.message) + ); + } + + let deterministicFixMatched = true; + let deterministicFixActual; + if (frontendOracle.deterministicFix !== undefined) { + const fixes = matches + .map((diagnostic) => materializeDeterministicFix(query, diagnostic)) + .filter(Boolean); + deterministicFixActual = + fixes.length === 0 + ? { offered: false } + : fixes.length === 1 + ? fixes[0] + : { offered: true, count: fixes.length, fixes }; + deterministicFixMatched = record( + 'deterministicFix', + exactEqual(frontendOracle.deterministicFix, deterministicFixActual), + frontendOracle.deterministicFix, + deterministicFixActual + ); + } + + let aiActionMatched = true; + let aiActionActual; + let actionDecisionMatched = true; + let actionDecisionActual; + if (frontendOracle.aiAction !== undefined) { + let decisions = []; + if (typeof decideAction !== 'function') { + aiActionActual = { + unavailable: true, + reason: 'production headless action-decision export is unavailable', + }; + actionDecisionActual = aiActionActual; + } else { + try { + for (const diagnostic of matches) { + decisions.push( + normalizeActionDecision( + decideAction({ + channel, + diagnostic, + hasDeterministicFix: !!diagnostic.fix, + aiFixEligible: diagnostic.aiFix?.eligible !== false, + enableAIFeatures: true, + hasAiFixHandler: true, + chatWired: true, + aiAgentAvailableForSource: true, + }) + ) + ); + } + } catch (error) { + aiActionActual = { + error: + `production action decision failed: ` + + `${error instanceof Error ? error.message : String(error)}`, + }; + actionDecisionActual = aiActionActual; + } + const invalidDecision = decisions.find( + (decision) => !['deterministic', 'ai', 'none'].includes(decision.kind) + ); + if (aiActionActual === undefined && invalidDecision) { + aiActionActual = { invalidDecision }; + } + if (aiActionActual === undefined) { + const actions = decisions + .filter((decision) => decision.kind === 'ai') + .map((decision) => ({ + offered: true, + ...(decision.commandId !== undefined + ? { commandId: decision.commandId } + : {}), + })); + aiActionActual = + actions.length === 0 + ? { offered: false } + : actions.length === 1 + ? actions[0] + : { offered: true, count: actions.length, actions }; + } + if (actionDecisionActual === undefined) { + actionDecisionActual = decisions.map((decision) => decision.kind); + } + } + aiActionMatched = record( + 'aiAction', + exactEqual(frontendOracle.aiAction, aiActionActual), + frontendOracle.aiAction, + aiActionActual + ); + const expectedDecisionKind = frontendOracle.deterministicFix?.offered + ? 'deterministic' + : frontendOracle.aiAction.offered + ? 'ai' + : 'none'; + const expectedDecisions = matches.map(() => expectedDecisionKind); + actionDecisionMatched = record( + 'actionDecision', + exactEqual(expectedDecisions, actionDecisionActual), + expectedDecisions, + actionDecisionActual + ); + } + + let syntaxFixMatched = true; + if (channel === 'syntax' && frontendOracle.fixText !== undefined) { + const fixes = allFrontendFindings + .filter((finding) => finding.fix) + .map((finding) => finding.fix.text); + const expected = + frontendOracle.fixText === null + ? { offered: false } + : { offered: true, text: frontendOracle.fixText }; + const actual = + fixes.length === 0 + ? { offered: false } + : fixes.length === 1 + ? { offered: true, text: fixes[0] } + : { offered: true, count: fixes.length, texts: fixes }; + syntaxFixMatched = record('syntaxFix', exactEqual(expected, actual), expected, actual); + } + + let rawMessageMatched = true; + if (channel === 'syntax' && frontendOracle.rawMessage !== undefined) { + const rawMessages = allFrontendFindings + .map((finding) => finding.rawMessage) + .filter((message) => typeof message === 'string' && message.length > 0); + const actual = rawMessages.length > 0; + rawMessageMatched = record( + 'rawParserError', + actual === frontendOracle.rawMessage, + frontendOracle.rawMessage, + actual + ); + } + + let totalErrorsMatched = true; + if (channel === 'syntax' && frontendOracle.totalErrors !== undefined) { + totalErrorsMatched = record( + 'totalErrors', + allFrontendFindings.length === frontendOracle.totalErrors, + frontendOracle.totalErrors, + allFrontendFindings.length + ); + } + + return { + assertions, + mismatches, + severityMatched, + messageMatched, + deterministicFixMatched, + deterministicFixActual, + aiActionMatched, + aiActionActual, + actionDecisionMatched, + actionDecisionActual, + syntaxFixMatched, + rawMessageMatched, + totalErrorsMatched, + }; +} + function equalSets(left, right) { return left.size === right.size && [...left].every((value) => right.has(value)); } -function buildCensus(contracts, manifest, catalog) { +export function buildCensus(contracts, manifest, catalog) { const problems = []; const byFile = new Map( contracts.map(({ file, spec }) => [path.basename(file), spec]) @@ -610,6 +1015,27 @@ function buildCensus(contracts, manifest, catalog) { .filter(({ spec }) => contractChannel(spec) === 'syntax') .map(({ spec }) => spec.ruleId) .sort(); + const catalogRuleIds = catalog.map((rule) => rule.id).sort(); + const duplicateCatalogRuleIds = catalogRuleIds.filter( + (ruleId, index) => catalogRuleIds.indexOf(ruleId) !== index + ); + if (duplicateCatalogRuleIds.length > 0) { + problems.push( + `catalog contains duplicate rule IDs: ${[ + ...new Set(duplicateCatalogRuleIds), + ].join(', ')}.` + ); + } + const syntaxRulesInCatalog = activeSyntaxRules.filter((ruleId) => + catalogRuleIds.includes(ruleId) + ); + if (syntaxRulesInCatalog.length > 0) { + problems.push( + `syntax features must remain outside the detector catalog: ${syntaxRulesInCatalog.join( + ', ' + )}.` + ); + } const enabledRules = catalog .filter((rule) => rule.enabled) .map((rule) => rule.id) @@ -629,6 +1055,12 @@ function buildCensus(contracts, manifest, catalog) { `expected one required syntax feature, found ${requiredSyntaxFeatures.length}.` ); } + if (!equalSets(new Set(requiredSyntaxFeatures), new Set(['command-suggestion']))) { + problems.push( + `required syntax features must equal ["command-suggestion"], found ` + + `${JSON.stringify(requiredSyntaxFeatures)}.` + ); + } if (activeContractRules.length !== 13) { problems.push(`expected 13 active contracts, found ${activeContractRules.length}.`); } @@ -669,7 +1101,7 @@ function main() { const reportPath = process.env.PPL_LINT_REPORT; const target = loadTarget(); const backendReport = loadBackendReport(target); - const { contracts, manifest, manifestPath } = loadContracts(); + const { contracts, activeContracts, manifest, manifestPath } = loadContracts(); const osd = loadOsd(); const { @@ -677,11 +1109,12 @@ function main() { getDetector, lintQuery, validateSyntax, + decideAction, osdRoot, surface, } = osd; const catalog = getBundledCatalog(); - const census = buildCensus(contracts, manifest, catalog); + const census = buildCensus(activeContracts, manifest, catalog); // The compiled surface lints with OSD's own checked-in grammar, so there is no // candidate bundle to load. On the runtime surface a missing bundle stays a hard @@ -698,6 +1131,7 @@ function main() { } const failures = []; + const reportOnlyFailures = []; // Contracts this surface did not score, recorded so the report says a rule was // skipped for surface rather than leaving its absence unexplained. const skippedForSurface = []; @@ -719,6 +1153,8 @@ function main() { observeAnalytics, observeOnly, differential: !!backendReport, + includedDormant: process.env.PPL_LINT_INCLUDE_DORMANT === '1', + reportOnlyFailures, // Census of the rules that ship enabled at ERROR severity, read from the OSD // catalog this run linted with. The multi-version aggregator enforces its // `defaultError` manifest set against this list, so a rule that becomes @@ -755,18 +1191,19 @@ function main() { `contracts=${contracts.length}` ); - for (const { file, spec } of contracts) { + for (const { file, spec, reportOnly = false } of contracts) { const ruleId = spec.ruleId; const index = spec.index; const channel = contractChannel(spec); - const entry = checkWiring(spec, catalog, getDetector, failures); + const scoringFailures = reportOnly ? reportOnlyFailures : failures; + const entry = checkWiring(spec, catalog, getDetector, scoringFailures); if (!entry) { continue; } // A contract runs on PR only when scheduled for PR; nightly runs everything. const contractSchedule = spec.schedule || 'pr'; - if (schedule === 'pr' && contractSchedule !== 'pr') { + if (!reportOnly && schedule === 'pr' && contractSchedule !== 'pr') { log(`SKIP ${ruleId} (schedule=${contractSchedule}, running ${schedule}) — ${path.basename(file)}`); continue; } @@ -796,6 +1233,7 @@ function main() { query: (queryDef.query || '').split('{{index}}').join(index), surface, executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), outcome: 'not-applicable', notApplicable: `contract declares grammarSurface "${contractSurface}"`, }); @@ -804,7 +1242,7 @@ function main() { } const context = buildContext(spec, engineVersion); - const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures, { + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, scoringFailures, { allowMissing: observeOnly, }); if (!expectation) { @@ -823,6 +1261,7 @@ function main() { query, surface, executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), outcome: 'not-applicable', notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', }); @@ -850,6 +1289,7 @@ function main() { query, surface, executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), expected: 0, actual: matches.length, severities: matches.map((m) => m.severity), @@ -882,8 +1322,6 @@ function main() { } const frontendOracle = oracleSelection.frontend; const expectedCount = frontendOracle.count; - const expectedSeverity = frontendOracle.severity; - const expectedMessage = frontendOracle.matchMessage; // A `runtimeOnly` rule walks grammar productions that exist only in the // runtime bundle, so `lint_runner` skips it on the compiled surface. Its @@ -904,6 +1342,7 @@ function main() { query, surface, executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), outcome: 'not-applicable', notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', }); @@ -934,30 +1373,27 @@ function main() { `expected ${expectedCount}, got ${actual} — ${query}` ); - const severityOk = - channel === 'syntax' || - !expectedSeverity || - actual === 0 || - matches.every((m) => m.severity === expectedSeverity); - const messageOk = - !expectedMessage || - matches.some((m) => (m.message || '').includes(expectedMessage)); - const fixOk = - channel !== 'syntax' || - frontendOracle.fixText === undefined || - matches.some((m) => m.fix && m.fix.text === frontendOracle.fixText); - const rawMessageOk = - channel !== 'syntax' || - frontendOracle.rawMessage === undefined || - matches.some((m) => - frontendOracle.rawMessage - ? typeof m.rawMessage === 'string' && m.rawMessage.length > 0 - : m.rawMessage === undefined - ); - const totalErrorsOk = - channel !== 'syntax' || - frontendOracle.totalErrors === undefined || - allFrontendFindings.length === frontendOracle.totalErrors; + const evaluated = evaluateFrontendAssertions({ + channel, + query, + matches, + allFrontendFindings, + frontendOracle, + decideAction, + }); + const assertions = { count: ok, ...evaluated.assertions }; + const mismatches = [ + ...(ok + ? [] + : [ + { + field: 'count', + expected: expectedCount, + actual, + }, + ]), + ...evaluated.mismatches, + ]; const resultEntry = { ruleId, @@ -968,11 +1404,25 @@ function main() { expected: expectedCount, actual, severities: matches.map((m) => m.severity).filter(Boolean), - severityMatched: severityOk, - messageMatched: messageOk, - fixMatched: fixOk, - rawMessageMatched: rawMessageOk, - totalErrorsMatched: totalErrorsOk, + severityMatched: evaluated.severityMatched, + messageMatched: evaluated.messageMatched, + deterministicFixMatched: evaluated.deterministicFixMatched, + aiActionMatched: evaluated.aiActionMatched, + actionDecisionMatched: evaluated.actionDecisionMatched, + fixMatched: evaluated.syntaxFixMatched, + rawMessageMatched: evaluated.rawMessageMatched, + totalErrorsMatched: evaluated.totalErrorsMatched, + assertions, + mismatches, + ...(evaluated.deterministicFixActual !== undefined + ? { deterministicFix: evaluated.deterministicFixActual } + : {}), + ...(evaluated.aiActionActual !== undefined + ? { aiAction: evaluated.aiActionActual } + : {}), + ...(evaluated.actionDecisionActual !== undefined + ? { actionDecision: evaluated.actionDecisionActual } + : {}), ...(channel === 'syntax' ? { code: frontendOracle.code, @@ -980,39 +1430,16 @@ function main() { totalErrors: allFrontendFindings.length, } : {}), + ...(reportOnly ? { reportOnly: true } : {}), executionBackend, backendOracleStatus: oracleSelection.status, }; - if (!ok) { - failures.push( - `[${ruleId}/${queryName}] expected ${expectedCount} "${ruleId}" diagnostic(s), got ${actual} for: ${query}` - ); - } - if (!severityOk) { - failures.push( - `[${ruleId}/${queryName}] expected severity "${expectedSeverity}" for: ${query}` - ); - } - if (!messageOk) { - failures.push( - `[${ruleId}/${queryName}] expected message to contain "${expectedMessage}" for: ${query}` - ); - } - if (!fixOk) { - failures.push( - `[${ruleId}/${queryName}] expected fix text "${frontendOracle.fixText}" for: ${query}` - ); - } - if (!rawMessageOk) { - failures.push( - `[${ruleId}/${queryName}] expected rawMessage=${frontendOracle.rawMessage} for: ${query}` - ); - } - if (!totalErrorsOk) { - failures.push( - `[${ruleId}/${queryName}] expected ${frontendOracle.totalErrors} total syntax error(s), ` + - `got ${allFrontendFindings.length} for: ${query}` + for (const mismatch of mismatches) { + scoringFailures.push( + `[${ruleId}/${queryName}] frontend.${mismatch.field} mismatch: ` + + `expected ${JSON.stringify(mismatch.expected)}, got ` + + `${JSON.stringify(mismatch.actual)} for: ${query}` ); } @@ -1026,7 +1453,7 @@ function main() { resultEntry.reason = oracleSelection.reason; resultEntry.coverageMissing = oracleSelection.reason; if (!observeAnalytics) { - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` ); } @@ -1041,7 +1468,9 @@ function main() { if (backendReport) { const be = backendReport.get(`${ruleId}::${queryName}`); if (!be) { - failures.push(`[${ruleId}/${queryName}] no backend report entry (backend did not run this query).`); + scoringFailures.push( + `[${ruleId}/${queryName}] no backend report entry (backend did not run this query).` + ); } else { const backendObservation = classifyBackendReportRow(be); if (oracleSelection.status !== 'applicable') { @@ -1051,7 +1480,7 @@ function main() { resultEntry.backendOutcome = backendObservation.status; } else if (backendObservation.status !== 'observed') { resultEntry.backendOutcome = backendObservation.status; - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + `(outcome=${JSON.stringify(backendObservation.status)}).` ); @@ -1061,7 +1490,7 @@ function main() { const backendRejected = backendObservation.rejected; resultEntry.backendRejected = backendRejected; if (backendRejected !== expectRejected) { - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` ); @@ -1090,7 +1519,7 @@ function main() { // the pairing rule is scoped to the rules it makes sense for. const detectorFlagged = actual > 0; if (role === 'trigger' && expectRejected && detectorFlagged !== backendRejected) { - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` ); @@ -1099,7 +1528,7 @@ function main() { // query the rule has to stay quiet on. Unlike a trigger, that claim does // not vary with `backend.kind`. if (role === 'control' && (detectorFlagged || backendRejected)) { - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` ); @@ -1108,7 +1537,7 @@ function main() { role === 'suppression-control' && (detectorFlagged || !backendRejected) ) { - failures.push( + scoringFailures.push( `[${ruleId}/${queryName}] differential: suppression control must retain a backend ` + `syntax rejection without a "${frontendOracle.code}" suggestion, but frontend ` + `${detectorFlagged ? 'suggested a rewrite' : 'did not suggest a rewrite'} and ` + @@ -1141,7 +1570,15 @@ function main() { process.exit(1); } + if (reportOnlyFailures.length > 0) { + log( + `REPORT-ONLY: ${reportOnlyFailures.length} dormant contract problem(s):\n- ` + + reportOnlyFailures.join('\n- ') + ); + } log(`PASS: all contracts agreed with the OSD detectors on the candidate bundle (schedule=${schedule}).`); } -main(); +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} From 72a8b9ed96e57991e2ed35b1c58357a3e5c27204 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 13:30:38 -0700 Subject: [PATCH 72/78] fix(ci): isolate PPL lint contract failures Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 10 +- .../workflows/ppl-lint-rule-validation.yml | 5 +- .../remote/PplLintRuleValidationIT.java | 8 +- .../ppl-lint/contracts/agg-on-text.spec.json | 11 - .../contracts/command-suggestion.spec.json | 149 ---- .../contracts/division-by-zero.spec.json | 15 - .../contracts/enabled-false-object.spec.json | 11 - .../contracts/field-validation.spec.json | 33 - .../invalid-capture-group-name.spec.json | 22 - .../ppl-lint/contracts/manifest.json | 11 +- .../multisearch-min-subsearch.spec.json | 11 - .../replace-wildcard-asymmetry.spec.json | 22 - .../contracts/rex-scan-cost.spec.json | 12 +- .../contracts/type-mismatch-numeric.spec.json | 11 - .../contracts/union-min-datasets.spec.json | 11 - ...ed-window-function-in-eventstats.spec.json | 33 - .../wildcard-source-zero-match.spec.json | 7 - scripts/ppl-lint/README.md | 38 +- .../__tests__/aggregate-versions.test.mjs | 63 +- .../__tests__/assemble-run-manifest.test.mjs | 46 +- .../__tests__/contract-schema.test.mjs | 51 +- .../__tests__/run-frontend-contract.test.mjs | 149 ++-- scripts/ppl-lint/aggregate-versions.mjs | 51 +- scripts/ppl-lint/assemble-run-manifest.mjs | 20 +- scripts/ppl-lint/contract-schema.mjs | 53 +- scripts/ppl-lint/run-frontend-contract.mjs | 701 +++++++++--------- 26 files changed, 600 insertions(+), 954 deletions(-) delete mode 100644 integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index df1aad11185..294197584f4 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -9,11 +9,10 @@ name: PPL lint multi-version validation # happens to be. A rule that is correct on main can be a false positive on 3.6 or # a false negative on 3.7, and nothing notices. # -# This workflow validates every active shipping contract — 12 detector rules and -# the command-suggestion syntax feature where its runtime grammar surface is -# available — against released engine versions plus the PR build. The aggregate -# still records the exact default-error census separately, but --all-rules makes -# warning/info omissions and syntax regressions visible too. +# This workflow validates all 12 active detector contracts against released +# engine versions plus the PR build. The aggregate still records the exact +# default-error census separately, but --all-rules makes warning/info omissions +# visible too. # # Shape — a per-version matrix of observation legs, then one aggregation: # @@ -837,7 +836,6 @@ jobs: env "${surface_env[@]}" "${observe_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_ENFORCE_CENSUS=1 \ PPL_LINT_INCLUDE_DORMANT=1 \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ PPL_LINT_REPORT="$leg/detector-report.json" \ diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 7ffabbce2da..c218b7fbda3 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -37,8 +37,8 @@ concurrency: # # Modes (design §3.4, §4.1.1): # - pull_request: SQL PR validation against the resolved OSD target. The ONLY -# enforcing mode; this is what branch protection pins to. Runs all 13 active -# schedule:pr contracts. The committed default is `main` on the canonical repo; +# enforcing mode; this is what branch protection pins to. Runs all 12 active +# detector contracts. The committed default is `main` on the canonical repo; # it can be overridden by the OSD_REPO/OSD_REF repo variables — see the # "Resolve OSD ref" step. TEMPORARY: those repo variables are currently set to # the unmerged paired OSD branch that ships the headless lint API this job @@ -295,7 +295,6 @@ jobs: PPL_LINT_TARGET_MANIFEST: ${{ github.workspace }}/artifacts/target.json PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json - PPL_LINT_ENFORCE_CENSUS: '1' PPL_LINT_INCLUDE_DORMANT: ${{ steps.schedule.outputs.value == 'nightly' && '1' || '0' }} run: | # pipefail so the runner's non-zero exit propagates through `tee` — diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index c6a503fddaf..7410eb32023 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -60,13 +60,13 @@ *

    While the ephemeral cluster is alive, the test also exports the candidate runtime grammar * bundle it built ({@code GET /_plugins/_ppl/_grammar}) and a small target manifest pairing the * bundle with the backend version and grammar hash. These become workflow artifacts that the - * detector-validation job injects into OSD's headless lint and syntax APIs, so both halves validate - * against the SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code + * detector-validation job injects into OSD's headless lint API, so both halves validate against the + * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. * *

    The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips - * contracts declaring {@code schedule: "nightly"}, while nightly runs all 13 active contracts. The - * filter holds new detector and syntax contracts back from PR runs while their standard and + * contracts declaring {@code schedule: "nightly"}, while nightly runs all 12 active detector + * contracts. The filter holds new detector contracts back from PR runs while their standard and * analytics oracles are still settling. * *

    Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's diff --git a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json index ebec7087274..5b9ece1480b 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/agg-on-text.spec.json @@ -61,10 +61,6 @@ "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -91,10 +87,6 @@ "messageEquals": "Numeric aggregation on a text field may return no value (null), because text is not stored as a number.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -119,9 +111,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json deleted file mode 100644 index 021a37bb4b3..00000000000 --- a/integ-test/src/test/resources/ppl-lint/contracts/command-suggestion.spec.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "schemaVersion": 4, - "ruleId": "command-suggestion", - "channel": "syntax", - "grammarSurface": "runtime-bundle", - "schedule": "pr", - "wiring": { - "code": "UNKNOWN_COMMAND" - }, - "backendFixture": { - "indices": [ - "ACCOUNT" - ], - "clusterSettings": { - "calcite": true, - "calciteFallback": false - } - }, - "frontendContext": { - "isCalcite": true - }, - "index": "opensearch-sql_test_index_account", - "queries": { - "misspelled-command": { - "role": "trigger", - "query": "source={{index}} | wherre age > 1" - }, - "valid-command-control": { - "role": "control", - "query": "source={{index}} | where age > 1" - }, - "unrecognizable-command": { - "role": "suppression-control", - "query": "source={{index}} | zzzzzzzz" - }, - "incomplete-expression": { - "role": "suppression-control", - "query": "source={{index}} | where age >" - } - }, - "expectations": [ - { - "version": ">=0.0.0", - "queries": { - "misspelled-command": { - "frontend": { - "count": 1, - "code": "UNKNOWN_COMMAND", - "fixText": "where", - "matchMessage": "Unknown command \"wherre\". Did you mean \"where\"?", - "rawMessage": true, - "totalErrors": 1 - }, - "backends": { - "standard": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - }, - "analytics": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - } - } - }, - "valid-command-control": { - "frontend": { - "count": 0, - "code": "UNKNOWN_COMMAND", - "fixText": null, - "rawMessage": false, - "totalErrors": 0 - }, - "backends": { - "standard": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true - } - }, - "analytics": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true - } - } - } - }, - "unrecognizable-command": { - "frontend": { - "count": 0, - "code": "UNKNOWN_COMMAND", - "fixText": null, - "rawMessage": false, - "totalErrors": 1 - }, - "backends": { - "standard": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - }, - "analytics": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - } - } - }, - "incomplete-expression": { - "frontend": { - "count": 0, - "code": "UNKNOWN_COMMAND", - "fixText": null, - "rawMessage": false, - "totalErrors": 1 - }, - "backends": { - "standard": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - }, - "analytics": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400 - } - } - } - } - } - } - ] -} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index b643094471b..5bb88408554 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -56,10 +56,6 @@ "messageEquals": "Dividing by zero returns no value (null) instead of an error.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -86,10 +82,6 @@ "messageEquals": "Dividing by zero returns no value (null) instead of an error.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -114,9 +106,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -143,10 +132,6 @@ "messageEquals": "Dividing by zero returns no value (null) instead of an error.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json index 4e222982589..366a7f85bb1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/enabled-false-object.spec.json @@ -63,10 +63,6 @@ "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -93,10 +89,6 @@ "messageEquals": "This field is stored but not searchable, so PPL returns null for it.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -121,9 +113,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 4094f0af7e0..f9c68833a02 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -68,10 +68,6 @@ "messageEquals": "Unknown field \"nonexistent_field\".", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -106,11 +102,7 @@ "endLine": 1, "endColumn": 63 }, - "expectedText": "field=firstname", "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" - }, - "aiAction": { - "offered": false } }, "backends": { @@ -135,9 +127,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -169,10 +158,6 @@ "messageEquals": "Unknown field \"nonexistent_field\".", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -215,11 +200,7 @@ "endLine": 1, "endColumn": 63 }, - "expectedText": "field=firstname", "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" - }, - "aiAction": { - "offered": false } }, "backends": { @@ -252,9 +233,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -286,10 +264,6 @@ "messageEquals": "Unknown field \"nonexistent_field\".", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -332,11 +306,7 @@ "endLine": 1, "endColumn": 63 }, - "expectedText": "field=firstname", "appliedQuery": "source=opensearch-sql_test_index_account | grok firstname \"%{WORD:w}\"" - }, - "aiAction": { - "offered": false } }, "backends": { @@ -369,9 +339,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index b4ba3df32c8..b6a5779ff08 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -61,10 +61,6 @@ "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -99,10 +95,6 @@ "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -135,9 +127,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -170,10 +159,6 @@ "messageEquals": "Capture group name \"user_name\" is invalid. Start with a letter and use only letters and numbers.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -208,10 +193,6 @@ "messageEquals": "Capture group name \"user-name\" is invalid. Start with a letter and use only letters and numbers.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -244,9 +225,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index 20a6995acb8..fe720056ba0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,9 +1,8 @@ { "schemaVersion": 4, - "description": "Required PPL frontend/backend compatibility corpus for the approved 12 detector rules plus the command-suggestion syntax feature.", + "description": "PPL frontend/backend compatibility corpus for the approved 12 detector rules.", "contracts": [ "agg-on-text.spec.json", - "command-suggestion.spec.json", "division-by-zero.spec.json", "enabled-false-object.spec.json", "field-validation.spec.json", @@ -38,9 +37,7 @@ "union-min-datasets.spec.json", "unsupported-window-function-in-eventstats.spec.json" ], - "requiredSyntaxFeatures": [ - "command-suggestion.spec.json" - ], + "requiredSyntaxFeatures": [], "nonEnforcing": [ "agg-on-text.spec.json", "division-by-zero.spec.json", @@ -51,8 +48,8 @@ ], "notes": { "enforced": "Reviewed lint error contracts with deterministic backend behavior.", - "defaultError": "Exact approved six-rule detector error census. command-suggestion is an error-channel feature but is intentionally excluded because it is not a detector.", - "requiredSyntaxFeatures": "Syntax-channel features validated through the production runtime grammar listener.", + "defaultError": "Exact approved six-rule detector error census.", + "requiredSyntaxFeatures": "Reserved for future syntax-channel compatibility contracts; currently empty.", "nonEnforcing": "Oracle-quality classification for warning, info, advisory, and result-shape contracts; scheduling determines execution, not this list.", "dormantContracts": "Preserved default-off detector regression contracts. They do not count toward active shipping coverage and must force-enable their detector when run." } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 106ed504f01..f13d9d4408c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -58,10 +58,6 @@ "messageEquals": "The multisearch command requires at least two subsearches.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -96,10 +92,6 @@ "messageEquals": "The multisearch command requires at least two subsearches.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -132,9 +124,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index c4b310bbac8..0ff54ce9fc3 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -59,10 +59,6 @@ "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -97,10 +93,6 @@ "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -133,9 +125,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -168,10 +157,6 @@ "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -206,10 +191,6 @@ "messageEquals": "The replace match and replacement have different numbers of \"*\" wildcards. The counts must match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -242,9 +223,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json index 5230aefe49f..43d02469f67 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/rex-scan-cost.spec.json @@ -6,7 +6,7 @@ "schedule": "pr", "wiring": { "detector": "rex-scan-cost", - "enabled": true, + "enabled": false, "severity": "info", "runtimeOnly": false, "needsContext": true, @@ -25,6 +25,7 @@ }, "frontendContext": { "isCalcite": true, + "forceEnable": true, "deriveFromMapping": { "email": "text" } @@ -55,9 +56,6 @@ "messageEquals": "parse runs the pattern against every input row from text field \"email\", even when it finds no match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -78,9 +76,6 @@ "messageEquals": "grok runs the pattern against every input row from text field \"email\", even when it finds no match.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -99,9 +94,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json index 5589bdb79f7..635f862a9d4 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/type-mismatch-numeric.spec.json @@ -59,10 +59,6 @@ "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -89,10 +85,6 @@ "messageEquals": "This field is numeric, but the compared value is not a number, so the comparison returns no rows.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -117,9 +109,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index ca16823381a..60551229d9b 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -61,10 +61,6 @@ "messageEquals": "The union command requires at least two datasets.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -99,10 +95,6 @@ "messageEquals": "The union command requires at least two datasets.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -135,9 +127,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 6c017195e1a..478ed49c0be 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -55,10 +55,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -93,10 +89,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -129,9 +121,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -163,10 +152,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -201,10 +186,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -237,9 +218,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { @@ -271,10 +249,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -309,10 +283,6 @@ "messageEquals": "This window function is not supported in eventstats/streamstats. Only row_number is supported.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -345,9 +315,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json index daf9920cab7..43e94902d56 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/wildcard-source-zero-match.spec.json @@ -51,10 +51,6 @@ "messageEquals": "Wildcard source pattern matches no known index.", "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": true, - "commandId": "ppl.lint.aiFix" } }, "backends": { @@ -79,9 +75,6 @@ "count": 0, "deterministicFix": { "offered": false - }, - "aiAction": { - "offered": false } }, "backends": { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 62b7d157283..63cc7797409 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -50,10 +50,9 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j | `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | | `schedule` (nightly) | full corpus + coverage | `main` | No | -The required PR corpus contains 12 detector contracts plus the -`command-suggestion` syntax contract. All 13 declare `schedule: "pr"` and can -fail the required check. The nightly mode runs the same shipping corpus across -the supported version and execution-backend matrix. +The required PR corpus contains 12 detector contracts. The nightly mode runs +the same active corpus plus four dormant report-only contracts across the +supported version and execution-backend matrix. `workflow_dispatch` inputs: @@ -210,8 +209,8 @@ rule cannot be validated end to end. backend behavior. - `defaultError` — every rule that ships **enabled at error severity** in OSD's `rules_catalog.json`; it contains exactly six detector rules. -- `requiredSyntaxFeatures` — `command-suggestion` only. Syntax features never - appear in `defaultError` or the detector catalog. +- `requiredSyntaxFeatures` — reserved for future syntax compatibility contracts; + currently empty. - `nonEnforcing` — oracle-quality classification for warning, info, advisory, and result-shape contracts. Scheduling determines whether a contract runs. - `dormantContracts` — four preserved default-off detector contracts. They do @@ -363,13 +362,15 @@ rule with only one pinned trigger gets an explicit warning that a "fully relaxed verdict rests on a single observation. That is the gap the discovery corpus below closes. -Four guards keep the check from passing vacuously. Each exists because "we could -not check" must never render as "it is fine": +Three hard guards keep the check from passing vacuously. The shipping census is +also recorded, but remains report-only until the paired OSD default-alignment +change lands: -- A rule that is default-error in OSD's catalog but has no contract file fails the - run. The detector runner records the catalog's default-error census in - `detector-report.json`, and the aggregate step compares it against - `manifest.defaultError` — so a new error rule cannot land unvalidated. +- A rule that is default-error in OSD's catalog but has no contract file is + reported in the shipping census. The detector runner records the catalog's + default-error census in `detector-report.json`, and the aggregate step compares + it against `manifest.defaultError`. This becomes blocking when census + enforcement is enabled after OSD defaults are aligned. - A leg whose artifacts are missing is a hard failure, never a silently dropped version. The aggregate step also checks that every version the plan asked for produced a report, so a dead observe job cannot shrink the matrix into a green @@ -406,9 +407,14 @@ different places a developer looks: The required single-version lane follows the same rule: frontend and backend failures with a `[rule/query]` identity anchor on that contract's `ruleId`. -Shipping-census findings anchor on `manifest.json` and fail the required lane. -Artifact and job failures without a trustworthy -repository location remain file-less rather than pointing at a guessed line. +An individual detector/query execution error is recorded as an `error` row and +does not stop the remaining contracts from running or prevent +`detector-report.json` from being uploaded. The required check still fails after +the complete report is written, with the failing rule/query named directly. +Shipping-census findings anchor on `manifest.json` and remain report-only until +the paired OSD default-alignment change lands. Artifact and job failures without +a trustworthy repository location remain file-less rather than pointing at a +guessed line. Without the annotations the only thing above the summary is `Process completed with exit code 1`, so the natural next click lands in raw job logs rather than the @@ -483,7 +489,7 @@ harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-front A query with no rule-owning ancestor is recorded unattributed and dropped rather than guessed at. Indices are rewritten onto the fixture index; JS string escapes are unescaped so the query matches what the test actually linted. The harvested - corpus is substantially larger than the curated 13-contract required corpus. + corpus is substantially larger than the curated 12-contract required corpus. Each file's **lint context** is harvested alongside its queries. Seven of the nineteen rules are `needsContext: true` and self-suppress without a `typeMap`, so diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index b42c825384d..df46ab6acd3 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -172,8 +172,6 @@ function writeLeg({ ...Object.fromEntries( [ 'deterministicFixMatched', - 'aiActionMatched', - 'actionDecisionMatched', 'fixMatched', 'rawMessageMatched', 'totalErrorsMatched', @@ -183,6 +181,8 @@ function writeLeg({ ), ...(c.assertions ? { assertions: c.assertions } : {}), ...(c.mismatches ? { mismatches: c.mismatches } : {}), + ...(c.detectorOutcome ? { outcome: c.detectorOutcome } : {}), + ...(c.detectorError ? { error: c.detectorError } : {}), ...(explicitIdentity ? { executionBackend } : {}), }); backend.push({ @@ -644,7 +644,7 @@ test('paired detector reports must be identical across execution backends', () = assert.match(stderr, /detector parity failed for union-min-datasets::trigger/); }); -test('identical exact-action mismatches across versions cannot aggregate green', () => { +test('identical deterministic-fix mismatches across versions cannot aggregate green', () => { const badCase = { detector: 1, rejected: true, @@ -689,6 +689,42 @@ test('identical exact-action mismatches across versions cannot aggregate green', assert.ok(report.matrix.every((row) => row.status === 'drift')); }); +test('a detector execution error is infrastructure evidence, not detector drift', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { + detector: 0, + rejected: true, + detectorOutcome: 'error', + detectorError: 'detector crashed', + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + }, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.drifts.length, 0); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match( + report.inconclusive[0].reasons.join(' '), + /trigger \(frontend execution failed: detector crashed\)/ + ); + assert.doesNotMatch(stdout, /update-detector/); +}); + test('target and detector execution identities must match', () => { const dir = writeLeg({ version: '3.8.0', @@ -1159,6 +1195,27 @@ test('an enforced shipping census mismatch fails aggregation', () => { assert.match(stdout, /### Shipping census/); }); +test('a report-only shipping census mismatch remains green', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.result.passed, true); + assert.equal(report.result.blockingShippingCensusProblems, 0); + assert.ok(report.shippingCensus.problems.length > 0); + assert.equal(report.shippingCensus.blocking, false); + assert.match(stdout, /CENSUS REPORT-ONLY/); + assert.doesNotMatch(stdout, /CENSUS ENFORCED/); +}); + test('a schema-v2 detector report without a census fails closed', () => { const dir = writeLeg({ version: '3.8.0', diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs index 7e2c1afa062..cb3b1497181 100644 --- a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -173,6 +173,42 @@ test('detector severity and message mismatches fail the manifest and summary', ( } }); +test('a detector execution error fails with its rule row instead of missing-artifact noise', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0] = { + ...detector.results[0], + expected: 0, + actual: 0, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + outcome: 'error', + error: 'detector crashed', + }; + detector.failures = [ + '[advisory-rule/trigger] frontend.execution failed: detector crashed', + ]; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /advisory-rule::trigger execution failed: detector crashed/); + assert.doesNotMatch(result.stderr, /count mismatch/); + assert.doesNotMatch(result.stderr, /did not match its execution assertion/); + assert.doesNotMatch(result.stderr, /required artifact is missing/); + assert.doesNotMatch(result.stderr, /has no matching detector row/); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*Error.*Fail/); +}); + test('syntax-specific frontend mismatches fail artifact validation', () => { for (const field of ['fixMatched', 'rawMessageMatched', 'totalErrorsMatched']) { const dir = makeRun(); @@ -191,19 +227,19 @@ test('syntax-specific frontend mismatches fail artifact validation', () => { } }); -test('exact deterministic and AI action mismatches fail artifacts and summary rows', () => { - for (const field of ['deterministicFixMatched', 'aiActionMatched']) { +test('exact deterministic fix mismatches fail artifacts and summary rows', () => { + for (const field of ['deterministicFixMatched']) { const dir = makeRun(); validArtifacts(dir); const file = path.join(dir, 'artifacts', 'detector-report.json'); const detector = JSON.parse(fs.readFileSync(file, 'utf8')); detector.results[0][field] = false; detector.results[0].assertions = { - [field === 'deterministicFixMatched' ? 'deterministicFix' : 'aiAction']: false, + deterministicFix: false, }; detector.results[0].mismatches = [ { - field: field === 'deterministicFixMatched' ? 'deterministicFix' : 'aiAction', + field: 'deterministicFix', expected: { offered: false }, actual: { offered: true }, }, @@ -213,7 +249,7 @@ test('exact deterministic and AI action mismatches fail artifacts and summary ro const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); assert.notEqual(result.status, 0); - assert.match(result.stderr, /did not match its (deterministic-fix|AI-action) assertion/); + assert.match(result.stderr, /did not match its deterministic-fix assertion/); assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); } }); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs index 5bb44957bf1..0aa55eeabb6 100644 --- a/scripts/ppl-lint/__tests__/contract-schema.test.mjs +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -511,12 +511,11 @@ test('missing channel remains a backwards-compatible lint contract', () => { severity: 'warning', messageEquals: undefined, deterministicFix: undefined, - aiAction: undefined, } ); }); -test('schema-v4 lint frontend normalizes exact message, fix, and AI action oracles', () => { +test('schema-v4 lint frontend normalizes exact message and fix oracles', () => { const contract = spec(4); const frontend = normalizeFrontendOracle(contract, { frontend: { @@ -536,7 +535,6 @@ test('schema-v4 lint frontend normalizes exact message, fix, and AI action oracl expectedText: '0', appliedQuery: 'source=t | eval x = 1', }, - aiAction: { offered: false }, }, }); @@ -558,7 +556,6 @@ test('schema-v4 lint frontend normalizes exact message, fix, and AI action oracl expectedText: '0', appliedQuery: 'source=t | eval x = 1', }, - aiAction: { offered: false }, }); }); @@ -578,7 +575,7 @@ test('matchMessage remains available only to schema-v3 lint contracts', () => { ); }); -test('schema-v4 action payloads fail closed on partial or extra fields', () => { +test('schema-v4 deterministic-fix payloads fail closed on partial or extra fields', () => { const contract = spec(4); for (const [frontend, expected] of [ [ @@ -605,40 +602,34 @@ test('schema-v4 action payloads fail closed on partial or extra fields', () => { offered: true, title: 'Fix', text: 'x', - range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 1 }, + range: { startLine: 1, startColumn: 2, endLine: 1, endColumn: 1 }, + expectedText: 'y', appliedQuery: 'x', }, }, - /expectedText must be a string/, + /must end at or after its start/, ], - [ - { + ]) { + assert.throws(() => normalizeFrontendOracle(contract, { frontend }), expected); + } + + assert.doesNotThrow(() => + normalizeFrontendOracle(contract, { + frontend: { count: 1, deterministicFix: { offered: true, title: 'Fix', text: 'x', - range: { startLine: 1, startColumn: 2, endLine: 1, endColumn: 1 }, - expectedText: 'y', + range: { startLine: 1, startColumn: 0, endLine: 1, endColumn: 1 }, appliedQuery: 'x', }, }, - /must end at or after its start/, - ], - [ - { count: 1, aiAction: { offered: true } }, - /commandId must be a non-empty string/, - ], - [ - { count: 1, aiAction: { offered: false, commandId: 'ppl.lint.aiFix' } }, - /must contain only offered/, - ], - ]) { - assert.throws(() => normalizeFrontendOracle(contract, { frontend }), expected); - } + }) + ); }); -test('active lint contracts require exact messages and explicit exclusive action modes', () => { +test('active lint contracts require exact messages and deterministic-fix behavior', () => { const contract = spec(4); const expectation = structuredClone(contract.expectations[0]); expectation.queries.trigger = { @@ -647,7 +638,6 @@ test('active lint contracts require exact messages and explicit exclusive action severity: 'error', messageEquals: 'Exact diagnostic.', deterministicFix: { offered: false }, - aiAction: { offered: true, commandId: 'ppl.lint.aiFix' }, }, backends: { standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, @@ -657,7 +647,6 @@ test('active lint contracts require exact messages and explicit exclusive action expectation.queries.control.frontend = { count: 0, deterministicFix: { offered: false }, - aiAction: { offered: false }, }; delete expectation.queries.control.detectorCount; @@ -677,8 +666,8 @@ test('active lint contracts require exact messages and explicit exclusive action /severity is required/ ); - const simultaneousActions = structuredClone(expectation); - simultaneousActions.queries.trigger.frontend.deterministicFix = { + const fixWithoutFinding = structuredClone(expectation); + fixWithoutFinding.queries.control.frontend.deterministicFix = { offered: true, title: 'Fix', text: 'fixed', @@ -687,8 +676,8 @@ test('active lint contracts require exact messages and explicit exclusive action appliedQuery: 'fixed', }; assert.throws( - () => assertShippingFrontendOracles(contract, simultaneousActions), - /cannot offer deterministic and AI actions together/ + () => assertShippingFrontendOracles(contract, fixWithoutFinding), + /must not offer a deterministic fix/ ); }); diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs index 6b482125211..ff0b48f9100 100644 --- a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -9,6 +9,7 @@ import { test } from 'node:test'; import { assertActiveShippingContracts, buildCensus, + buildFrontendExecutionError, evaluateFrontendAssertions, selectManifestContractNames, } from '../run-frontend-contract.mjs'; @@ -48,12 +49,7 @@ test('exact lint assertions materialize the effective deterministic edit', () => expectedText: 'bad', appliedQuery: 'source=t | good', }, - aiAction: { offered: false }, }, - decideAction: ({ hasDeterministicFix }) => ({ - kind: hasDeterministicFix ? 'deterministic' : 'ai', - commandId: 'ppl.lint.aiFix', - }), }); assert.deepEqual(result.mismatches, []); @@ -61,8 +57,6 @@ test('exact lint assertions materialize the effective deterministic edit', () => severity: true, message: true, deterministicFix: true, - aiAction: true, - actionDecision: true, }); assert.deepEqual(result.deterministicFixActual, { offered: true, @@ -74,43 +68,7 @@ test('exact lint assertions materialize the effective deterministic edit', () => }); }); -test('deterministic fixes require the production helper to choose the deterministic action', () => { - const result = evaluateFrontendAssertions({ - channel: 'lint', - query: 'source=t | bad', - matches: [ - { - severity: 'warning', - message: 'Replace bad.', - range: RANGE, - fix: { title: 'Replace bad', text: 'good', expectedText: 'bad' }, - }, - ], - frontendOracle: { - deterministicFix: { - offered: true, - title: 'Replace bad', - text: 'good', - range: RANGE, - expectedText: 'bad', - appliedQuery: 'source=t | good', - }, - aiAction: { offered: false }, - }, - decideAction: () => ({ kind: 'none' }), - }); - - assert.equal(result.deterministicFixMatched, true); - assert.equal(result.aiActionMatched, true); - assert.equal(result.actionDecisionMatched, false); - assert.equal( - result.mismatches.find(({ field }) => field === 'actionDecision')?.actual[0], - 'none' - ); -}); - -test('AI action identity and exact messages produce field-specific mismatches', () => { - let decisionInput; +test('exact message mismatches are field-specific', () => { const result = evaluateFrontendAssertions({ channel: 'lint', query: 'source=t | bad', @@ -125,26 +83,10 @@ test('AI action identity and exact messages produce field-specific mismatches', frontendOracle: { messageEquals: 'Expected message.', deterministicFix: { offered: false }, - aiAction: { offered: true, commandId: 'ppl.lint.aiFix' }, - }, - decideAction: (input) => { - decisionInput = input; - return { kind: 'ai', commandId: 'wrong.command' }; }, }); - assert.deepEqual( - result.mismatches.map(({ field }) => field), - ['message', 'aiAction'] - ); - assert.deepEqual(result.aiActionActual, { - offered: true, - commandId: 'wrong.command', - }); - assert.equal(decisionInput.enableAIFeatures, true); - assert.equal(decisionInput.hasAiFixHandler, true); - assert.equal(decisionInput.aiAgentAvailableForSource, true); - assert.equal(decisionInput.aiFixEligible, true); + assert.deepEqual(result.mismatches.map(({ field }) => field), ['message']); }); test('deterministic expectedText must match the source slice', () => { @@ -178,33 +120,45 @@ test('deterministic expectedText must match the source slice', () => { assert.equal(result.deterministicFixActual.expectedTextMatchesSource, false); }); -test('AI assertions fail closed when the production decision export is unavailable', () => { - const result = evaluateFrontendAssertions({ - channel: 'lint', - query: 'source=t', - matches: [], - frontendOracle: { aiAction: { offered: false } }, - }); - - assert.equal(result.aiActionMatched, false); - assert.equal(result.mismatches[0].field, 'aiAction'); - assert.equal(result.aiActionActual.unavailable, true); -}); - -test('AI assertions report production decision errors instead of passing absence', () => { - const result = evaluateFrontendAssertions({ - channel: 'lint', - query: 'source=t | bad', - matches: [{ message: 'Bad', range: RANGE }], - frontendOracle: { aiAction: { offered: false } }, - decideAction: () => { - throw new Error('decision failed'); - }, - }); - - assert.equal(result.aiActionMatched, false); - assert.match(result.aiActionActual.error, /decision failed/); - assert.equal(result.mismatches[0].field, 'aiAction'); +test('a frontend execution error remains a complete report row', () => { + assert.deepEqual( + buildFrontendExecutionError({ + ruleId: 'example-rule', + channel: 'lint', + queryName: 'trigger', + role: 'trigger', + query: 'source=t | bad', + expected: 0, + surface: 'runtime-bundle', + executionBackend: 'standard', + error: new Error('detector crashed'), + }), + { + ruleId: 'example-rule', + channel: 'lint', + queryName: 'trigger', + role: 'trigger', + query: 'source=t | bad', + expected: 0, + actual: 0, + severities: [], + severityMatched: true, + messageMatched: true, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: 'detector crashed', + }, + ], + outcome: 'error', + error: 'detector crashed', + surface: 'runtime-bundle', + executionBackend: 'standard', + backendOracleStatus: 'error', + } + ); }); test('syntax suppression checks raw parser errors outside the suggestion code filter', () => { @@ -295,31 +249,26 @@ test('discovery mode accepts legacy generated specs without shipping oracles', ( ); }); -test('shipping census rejects duplicate detector IDs and syntax features in the catalog', () => { - const syntaxContract = { - file: 'command-suggestion.spec.json', +test('shipping census rejects duplicate detector IDs', () => { + const lintContract = { + file: 'example.spec.json', spec: { schemaVersion: 4, - ruleId: 'command-suggestion', - channel: 'syntax', + ruleId: 'example', }, }; const census = buildCensus( - [syntaxContract], + [lintContract], { - contracts: ['command-suggestion.spec.json'], + contracts: ['example.spec.json'], defaultError: [], - requiredSyntaxFeatures: ['command-suggestion.spec.json'], + requiredSyntaxFeatures: [], }, [ - { id: 'command-suggestion', enabled: false, severity: 'error' }, { id: 'duplicate-rule', enabled: false, severity: 'info' }, { id: 'duplicate-rule', enabled: false, severity: 'info' }, ] ); assert.ok(census.problems.some((problem) => /duplicate rule IDs/.test(problem))); - assert.ok( - census.problems.some((problem) => /outside the detector catalog/.test(problem)) - ); }); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 1a9abf03b0b..7ceb86ae6df 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -268,8 +268,6 @@ function normalizeDetectorReport(detector, target) { } for (const field of [ 'deterministicFixMatched', - 'aiActionMatched', - 'actionDecisionMatched', 'fixMatched', 'rawMessageMatched', 'totalErrorsMatched', @@ -613,12 +611,9 @@ function detectorParityValue(entry) { typeof entry.deterministicFixMatched === 'boolean' ? entry.deterministicFixMatched : undefined, - aiActionMatched: - typeof entry.aiActionMatched === 'boolean' ? entry.aiActionMatched : undefined, assertions: entry.assertions, mismatches: entry.mismatches, deterministicFix: entry.deterministicFix, - aiAction: entry.aiAction, code: entry.code, codes: entry.codes, totalErrors: entry.totalErrors, @@ -825,11 +820,11 @@ function auditShippingCensus(legs, specs, manifest) { if (activeLintRules.size !== 12) { problems.push(`expected 12 active lint contracts, found ${activeLintRules.size}`); } - if (requiredSyntaxRules.size !== 1) { - problems.push(`expected one required syntax feature, found ${requiredSyntaxRules.size}`); + if (requiredSyntaxRules.size !== 0) { + problems.push(`expected no required syntax features, found ${requiredSyntaxRules.size}`); } - if (activeRules.size !== 13) { - problems.push(`expected 13 active contracts, found ${activeRules.size}`); + if (activeRules.size !== 12) { + problems.push(`expected 12 active contracts, found ${activeRules.size}`); } if (!setsEqual(activeLintRules, enabledRules)) { problems.push( @@ -882,9 +877,10 @@ function auditShippingCensus(legs, specs, manifest) { function readBackendObservation(backendEntry, detectorResult) { const verdict = backendVerdict(backendEntry); const hasVerdict = typeof verdict.backendRejected === 'boolean'; + const detectorUsable = !!detectorResult && detectorResult.outcome !== 'error'; return { - usable: hasVerdict && !!detectorResult, + usable: hasVerdict && detectorUsable, observed: { detectorCount: detectorResult ? detectorResult.actual : 0, severities: detectorResult ? detectorResult.severities || [] : [], @@ -899,7 +895,6 @@ function readBackendObservation(backendEntry, detectorResult) { deterministicFixMatched: detectorResult ? detectorResult.deterministicFixMatched : undefined, - aiActionMatched: detectorResult ? detectorResult.aiActionMatched : undefined, fixMatched: detectorResult ? detectorResult.fixMatched : undefined, rawMessageMatched: detectorResult ? detectorResult.rawMessageMatched : undefined, totalErrorsMatched: detectorResult @@ -911,6 +906,20 @@ function readBackendObservation(backendEntry, detectorResult) { }; } +function unusableObservationReason(detectorResult) { + if (!detectorResult) { + return 'no detector result'; + } + if (detectorResult.outcome === 'error') { + const message = + typeof detectorResult.error === 'string' && detectorResult.error.length > 0 + ? detectorResult.error + : 'unknown frontend execution error'; + return `frontend execution failed: ${message}`; + } + return 'no engine verdict'; +} + function failedExtendedFrontendAssertions(entry, frontendOracle) { if (!entry) { return []; @@ -918,7 +927,6 @@ function failedExtendedFrontendAssertions(entry, frontendOracle) { const failures = new Set(); for (const field of [ 'deterministicFixMatched', - 'aiActionMatched', 'fixMatched', 'rawMessageMatched', 'totalErrorsMatched', @@ -972,9 +980,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify, divergentCases }) { const detectorResult = leg.detector.resultsByKey.get(rowKey); const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { - unusable.push( - `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` - ); + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); continue; } observations.set(queryName, observed); @@ -1322,13 +1328,18 @@ function main() { ); } } + if (detectorResult && detectorResult.outcome === 'error') { + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } if (oracleSelection.status === 'coverage-missing') { const { usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { - unusable.push( - `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` - ); + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); if (role === 'trigger') { unobservedTriggers.push(queryName); } @@ -1457,9 +1468,7 @@ function main() { // verdict and no detector row looks exactly like "the detector went // silent", and the report would tell the engineer to go fix a detector // that is fine. Record it as not compared and move on. - unusable.push( - `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` - ); + unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); if (role === 'trigger') { unobservedTriggers.push(queryName); } diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index 5d875e593bb..7047ffad28f 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -58,8 +58,6 @@ function failedFrontendAssertions(entry) { } for (const [field, label] of [ ['deterministicFixMatched', 'deterministic-fix'], - ['aiActionMatched', 'AI-action'], - ['actionDecisionMatched', 'action-decision'], ['fixMatched', 'syntax-fix'], ['rawMessageMatched', 'raw-parser-error'], ['totalErrorsMatched', 'total-error'], @@ -182,6 +180,14 @@ function main() { if (!backendByKey.has(key)) { artifactErrors.push(`detector row ${key} has no matching backend row`); } + if (entry.outcome === 'error') { + const message = + typeof entry.error === 'string' && entry.error.length > 0 + ? entry.error + : 'unknown frontend execution error'; + artifactErrors.push(`detector row ${key} execution failed: ${message}`); + continue; + } if (!Number.isInteger(entry.expected) || !Number.isInteger(entry.actual)) { artifactErrors.push(`detector row ${key} must contain integer expected/actual counts`); } else if (entry.actual !== entry.expected) { @@ -332,7 +338,12 @@ function writeSummary(manifest, detector, backend) { for (const r of detector.results || []) { const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); - const detectorCell = `${r.actual}/${r.expected}${r.severities && r.severities.length ? ` (${r.severities.join(',')})` : ''}`; + const detectorCell = + r.outcome === 'error' + ? 'Error' + : `${r.actual}/${r.expected}${ + r.severities && r.severities.length ? ` (${r.severities.join(',')})` : '' + }`; const backendCell = !be ? '—' : typeof be.rejected !== 'boolean' @@ -343,7 +354,8 @@ function writeSummary(manifest, detector, backend) { const ok = r.reportOnly === true ? undefined - : r.actual === r.expected && + : r.outcome !== 'error' && + r.actual === r.expected && failedFrontendAssertions(r).length === 0 && !!be && be.outcome === 'pass'; diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs index b1da53617fa..3b8e4bf68f9 100644 --- a/scripts/ppl-lint/contract-schema.mjs +++ b/scripts/ppl-lint/contract-schema.mjs @@ -14,7 +14,6 @@ const LINT_V4_FRONTEND_FIELDS = new Set([ 'severity', 'messageEquals', 'deterministicFix', - 'aiAction', ]); const SYNTAX_FRONTEND_FIELDS = new Set([ 'count', @@ -130,30 +129,14 @@ function normalizeDeterministicFix(value, label) { title: requireNonEmptyString(fix.title, `${label}.title`), text: requireString(fix.text, `${label}.text`), range: normalizeRange(fix.range, `${label}.range`), - expectedText: requireString(fix.expectedText, `${label}.expectedText`), + ...(fix.expectedText === undefined + ? {} + : { expectedText: requireString(fix.expectedText, `${label}.expectedText`) }), appliedQuery: requireString(fix.appliedQuery, `${label}.appliedQuery`), }; return normalized; } -function normalizeAiAction(value, label) { - const action = requireObject(value, label); - if (typeof action.offered !== 'boolean') { - throw new TypeError(`${label}.offered must be a boolean.`); - } - assertKnownKeys(action, new Set(['offered', 'commandId']), label); - if (!action.offered) { - if (Object.keys(action).length !== 1) { - throw new Error(`${label} must contain only offered when no AI action is expected.`); - } - return { offered: false }; - } - return { - offered: true, - commandId: requireNonEmptyString(action.commandId, `${label}.commandId`), - }; -} - export function contractChannel(spec) { requireObject(spec, 'contract'); const channel = spec.channel === undefined ? 'lint' : spec.channel; @@ -169,9 +152,8 @@ export function contractChannel(spec) { * Normalize legacy detector fields and channel-specific frontend assertions. * * Schema-v3 lint contracts retain substring messages. Schema-v4 lint contracts - * assert exact messages and exact deterministic/AI action availability. Syntax - * contracts assert stable parser error identity, quick-fix presence or absence, - * raw-message preservation, and the total syntax error census. + * assert exact messages and deterministic fixes. Syntax contracts retain their + * legacy parser error, quick-fix, and raw-message assertions. */ export function normalizeFrontendOracle(spec, queryExpectation) { const channel = contractChannel(spec); @@ -206,9 +188,6 @@ export function normalizeFrontendOracle(spec, queryExpectation) { ...(queryExpectation.deterministicFix !== undefined ? { deterministicFix: queryExpectation.deterministicFix } : {}), - ...(queryExpectation.aiAction !== undefined - ? { aiAction: queryExpectation.aiAction } - : {}), }; const allowed = spec.schemaVersion === 3 ? LINT_V3_FRONTEND_FIELDS : LINT_V4_FRONTEND_FIELDS; @@ -232,7 +211,7 @@ export function normalizeFrontendOracle(spec, queryExpectation) { } if ( hasFrontend && - ['severity', 'matchMessage', 'messageEquals', 'deterministicFix', 'aiAction'].some( + ['severity', 'matchMessage', 'messageEquals', 'deterministicFix'].some( (field) => queryExpectation[field] !== undefined ) ) { @@ -255,13 +234,6 @@ export function normalizeFrontendOracle(spec, queryExpectation) { frontend.deterministicFix, `[${spec.ruleId}] frontend.deterministicFix` ), - aiAction: - frontend.aiAction === undefined - ? undefined - : normalizeAiAction( - frontend.aiAction, - `[${spec.ruleId}] frontend.aiAction` - ), }), }; } @@ -322,8 +294,7 @@ export function normalizeFrontendOracle(spec, queryExpectation) { /** * Active shipping contracts are stricter than dormant compatibility contracts: - * every lint finding pins its exact message and action mode, every lint control - * pins action absence, and every syntax case pins fix/raw-error/error-count state. + * every lint finding pins its exact message and deterministic-fix behavior. */ export function assertShippingFrontendOracles(spec, expectation) { if (spec.schemaVersion !== 4) { @@ -352,9 +323,6 @@ export function assertShippingFrontendOracles(spec, expectation) { if (frontend.deterministicFix === undefined) { throw new Error(`${label}.deterministicFix must be explicitly asserted.`); } - if (frontend.aiAction === undefined) { - throw new Error(`${label}.aiAction must be explicitly asserted.`); - } if (frontend.count > 0) { if (frontend.severity === undefined) { throw new Error(`${label}.severity is required for a lint finding.`); @@ -362,11 +330,8 @@ export function assertShippingFrontendOracles(spec, expectation) { if (frontend.messageEquals === undefined) { throw new Error(`${label}.messageEquals is required for a lint finding.`); } - if (frontend.deterministicFix.offered && frontend.aiAction.offered) { - throw new Error(`${label} cannot offer deterministic and AI actions together.`); - } - } else if (frontend.deterministicFix.offered || frontend.aiAction.offered) { - throw new Error(`${label} must not offer actions when no finding is expected.`); + } else if (frontend.deterministicFix.offered) { + throw new Error(`${label} must not offer a deterministic fix when no finding is expected.`); } } } diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 68438d4632e..0875f902338 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -44,7 +44,8 @@ * backend behavior for each query agrees with the observed detector output * — a trigger the detector flags is one the backend rejected; a control the * detector passes is one the backend accepted (design §3.2, §4.3). - * 4. Coverage (nightly only): every enabled catalog rule has a contract file. + * 4. Coverage census: records whether every enabled catalog rule has a + * contract file. Census drift is report-only unless explicitly enforced. * * ## Two grammar surfaces * @@ -107,8 +108,6 @@ const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; // supports older checkouts (that is the coverage it adds), so fall back to the // source module, which `setup_node_env` transpiles on require anyway. const CATALOG_SOURCE_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; -const ACTION_DECISION_MODULE = - 'packages/osd-monaco/src/ppl/lint/action_decision'; const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; /** @@ -311,23 +310,6 @@ function loadContracts() { }; } -function resolveActionDecision(module) { - if (!module) { - return undefined; - } - for (const name of [ - 'decidePPLLintAction', - 'decidePPLDiagnosticAction', - 'getPPLDiagnosticActionDecision', - 'decideDiagnosticAction', - ]) { - if (typeof module[name] === 'function') { - return module[name]; - } - } - return undefined; -} - function loadOsd() { const osdRoot = process.cwd(); const require = createRequire(path.join(osdRoot, 'noop.js')); @@ -362,7 +344,6 @@ function loadOsd() { // so a checkout without the built export can still run the compiled surface. const catalogModule = resolveOsd(CATALOG_MODULE, { optional: true }) || resolveOsd(CATALOG_SOURCE_MODULE); - const actionDecisionModule = resolveOsd(ACTION_DECISION_MODULE, { optional: true }); const { getBundledCatalog } = catalogModule; const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); if (typeof getBundledCatalog !== 'function') { @@ -379,7 +360,6 @@ function loadOsd() { fatal(`PPLLanguageAnalyzer not found in ${ANALYZER_MODULE}.`); } const analyzer = new PPLLanguageAnalyzer(); - const headless = resolveOsd(HEADLESS_MODULE, { optional: true }); return { surface: SURFACE, // Same (query, grammar, context) shape as the bundle path so the main loop @@ -390,10 +370,6 @@ function loadOsd() { }, getBundledCatalog, getDetector, - decideAction: - resolveActionDecision(headless) || - resolveActionDecision(catalogModule) || - resolveActionDecision(actionDecisionModule), osdRoot, }; } @@ -418,10 +394,6 @@ function loadOsd() { deserializeBundleOrThrow, lintQuery: lintQueryWithBundle, validateSyntax, - decideAction: - resolveActionDecision(headless) || - resolveActionDecision(catalogModule) || - resolveActionDecision(actionDecisionModule), getBundledCatalog, getDetector, osdRoot, @@ -582,7 +554,7 @@ function selectExpectation(spec, version, isCalcite, failures, { allowMissing = failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); } } else { - fatal( + failures.push( `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} ` + '(exactly one required).' ); @@ -734,19 +706,6 @@ function materializeDeterministicFix(query, diagnostic) { }; } -function normalizeActionDecision(decision) { - if (typeof decision === 'string') { - return { kind: decision }; - } - if (!decision || typeof decision !== 'object' || Array.isArray(decision)) { - return { kind: 'invalid', value: decision }; - } - return { - kind: decision.kind || decision.type || decision.action, - commandId: decision.commandId || (decision.command && decision.command.id), - }; -} - function exactEqual(left, right) { return JSON.stringify(left) === JSON.stringify(right); } @@ -757,7 +716,6 @@ export function evaluateFrontendAssertions({ matches, allFrontendFindings = matches, frontendOracle, - decideAction, }) { const assertions = {}; const mismatches = []; @@ -826,90 +784,6 @@ export function evaluateFrontendAssertions({ ); } - let aiActionMatched = true; - let aiActionActual; - let actionDecisionMatched = true; - let actionDecisionActual; - if (frontendOracle.aiAction !== undefined) { - let decisions = []; - if (typeof decideAction !== 'function') { - aiActionActual = { - unavailable: true, - reason: 'production headless action-decision export is unavailable', - }; - actionDecisionActual = aiActionActual; - } else { - try { - for (const diagnostic of matches) { - decisions.push( - normalizeActionDecision( - decideAction({ - channel, - diagnostic, - hasDeterministicFix: !!diagnostic.fix, - aiFixEligible: diagnostic.aiFix?.eligible !== false, - enableAIFeatures: true, - hasAiFixHandler: true, - chatWired: true, - aiAgentAvailableForSource: true, - }) - ) - ); - } - } catch (error) { - aiActionActual = { - error: - `production action decision failed: ` + - `${error instanceof Error ? error.message : String(error)}`, - }; - actionDecisionActual = aiActionActual; - } - const invalidDecision = decisions.find( - (decision) => !['deterministic', 'ai', 'none'].includes(decision.kind) - ); - if (aiActionActual === undefined && invalidDecision) { - aiActionActual = { invalidDecision }; - } - if (aiActionActual === undefined) { - const actions = decisions - .filter((decision) => decision.kind === 'ai') - .map((decision) => ({ - offered: true, - ...(decision.commandId !== undefined - ? { commandId: decision.commandId } - : {}), - })); - aiActionActual = - actions.length === 0 - ? { offered: false } - : actions.length === 1 - ? actions[0] - : { offered: true, count: actions.length, actions }; - } - if (actionDecisionActual === undefined) { - actionDecisionActual = decisions.map((decision) => decision.kind); - } - } - aiActionMatched = record( - 'aiAction', - exactEqual(frontendOracle.aiAction, aiActionActual), - frontendOracle.aiAction, - aiActionActual - ); - const expectedDecisionKind = frontendOracle.deterministicFix?.offered - ? 'deterministic' - : frontendOracle.aiAction.offered - ? 'ai' - : 'none'; - const expectedDecisions = matches.map(() => expectedDecisionKind); - actionDecisionMatched = record( - 'actionDecision', - exactEqual(expectedDecisions, actionDecisionActual), - expectedDecisions, - actionDecisionActual - ); - } - let syntaxFixMatched = true; if (channel === 'syntax' && frontendOracle.fixText !== undefined) { const fixes = allFrontendFindings @@ -959,16 +833,53 @@ export function evaluateFrontendAssertions({ messageMatched, deterministicFixMatched, deterministicFixActual, - aiActionMatched, - aiActionActual, - actionDecisionMatched, - actionDecisionActual, syntaxFixMatched, rawMessageMatched, totalErrorsMatched, }; } +export function buildFrontendExecutionError({ + ruleId, + channel, + queryName, + role, + query, + expected = 0, + surface, + executionBackend, + error, + reportOnly = false, +}) { + const message = error instanceof Error ? error.message : String(error); + return { + ruleId, + channel, + queryName, + role, + query, + expected: Number.isInteger(expected) ? expected : 0, + actual: 0, + severities: [], + severityMatched: true, + messageMatched: true, + assertions: { execution: false }, + mismatches: [ + { + field: 'execution', + expected: 'completed', + actual: message, + }, + ], + outcome: 'error', + error: message, + surface, + executionBackend, + backendOracleStatus: 'error', + ...(reportOnly ? { reportOnly: true } : {}), + }; +} + function equalSets(left, right) { return left.size === right.size && [...left].every((value) => right.has(value)); } @@ -1050,19 +961,14 @@ export function buildCensus(contracts, manifest, catalog) { if (activeLintRules.length !== 12) { problems.push(`expected 12 active lint contracts, found ${activeLintRules.length}.`); } - if (requiredSyntaxFeatures.length !== 1) { + if (requiredSyntaxFeatures.length !== 0) { problems.push( - `expected one required syntax feature, found ${requiredSyntaxFeatures.length}.` - ); - } - if (!equalSets(new Set(requiredSyntaxFeatures), new Set(['command-suggestion']))) { - problems.push( - `required syntax features must equal ["command-suggestion"], found ` + + `required syntax features must be empty, found ` + `${JSON.stringify(requiredSyntaxFeatures)}.` ); } - if (activeContractRules.length !== 13) { - problems.push(`expected 13 active contracts, found ${activeContractRules.length}.`); + if (activeContractRules.length !== 12) { + problems.push(`expected 12 active contracts, found ${activeContractRules.length}.`); } if (!equalSets(new Set(activeSyntaxRules), new Set(requiredSyntaxFeatures))) { problems.push( @@ -1109,7 +1015,6 @@ function main() { getDetector, lintQuery, validateSyntax, - decideAction, osdRoot, surface, } = osd; @@ -1156,11 +1061,9 @@ function main() { includedDormant: process.env.PPL_LINT_INCLUDE_DORMANT === '1', reportOnlyFailures, // Census of the rules that ship enabled at ERROR severity, read from the OSD - // catalog this run linted with. The multi-version aggregator enforces its - // `defaultError` manifest set against this list, so a rule that becomes - // default-error in OSD without a contract file cannot slip through - // unvalidated — and the aggregator does not need its own OSD checkout to - // notice (design: default-error is the set users cannot opt out of). + // catalog this run linted with. The multi-version aggregator compares its + // `defaultError` manifest set against this list without needing its own OSD + // checkout. Drift remains report-only unless census enforcement is enabled. defaultErrorRules: catalog .filter((rule) => rule.enabled && rule.severity === 'error') .map((rule) => rule.id) @@ -1191,6 +1094,38 @@ function main() { `contracts=${contracts.length}` ); + const recordExecutionError = ({ + spec, + channel, + queryName, + queryDef, + expected, + error, + reportOnly, + scoringFailures, + }) => { + const query = (queryDef.query || '').split('{{index}}').join(spec.index); + const message = error instanceof Error ? error.message : String(error); + log(` FAIL ${spec.ruleId}/${queryName}: execution error — ${message}`); + scoringFailures.push( + `[${spec.ruleId}/${queryName}] frontend.execution failed: ${message}` + ); + report.results.push( + buildFrontendExecutionError({ + ruleId: spec.ruleId, + channel, + queryName, + role: queryDef.role || 'trigger', + query, + expected, + surface, + executionBackend, + error: message, + reportOnly, + }) + ); + }; + for (const { file, spec, reportOnly = false } of contracts) { const ruleId = spec.ruleId; const index = spec.index; @@ -1198,6 +1133,17 @@ function main() { const scoringFailures = reportOnly ? reportOnlyFailures : failures; const entry = checkWiring(spec, catalog, getDetector, scoringFailures); if (!entry) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `OSD catalog entry "${ruleId}" is unavailable`, + reportOnly, + scoringFailures, + }); + } continue; } @@ -1247,6 +1193,17 @@ function main() { }); if (!expectation) { if (!observeOnly) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `no unique expectation matches backend version ${engineVersion || 'unknown'}`, + reportOnly, + scoringFailures, + }); + } continue; } for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { @@ -1267,37 +1224,48 @@ function main() { }); continue; } - if (channel === 'syntax' && typeof validateSyntax !== 'function') { - fatal( - `Syntax contract "${ruleId}" requires validateQueryWithBundle from ${SYNTAX_MODULE}. ` + - `Validate this SQL branch against the OSD headless-syntax PR.` - ); + try { + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + throw new Error( + `syntax validation requires validateQueryWithBundle from ${SYNTAX_MODULE}` + ); + } + const result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const matches = + channel === 'syntax' + ? result.errors || [] + : (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + report.results.push({ + ruleId, + channel, + queryName, + role, + query, + surface, + executionBackend, + ...(reportOnly ? { reportOnly: true } : {}), + expected: 0, + actual: matches.length, + severities: matches.map((m) => m.severity), + severityMatched: true, + messageMatched: true, + backendOracleStatus: 'coverage-missing', + expectationStatus: 'coverage-missing', + }); + } catch (error) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error, + reportOnly, + scoringFailures, + }); } - const result = - channel === 'syntax' - ? validateSyntax(query, grammar) - : lintQuery(query, grammar, context); - const matches = - channel === 'syntax' - ? result.errors || [] - : (result.diagnostics || []).filter((d) => d.ruleId === ruleId); - report.results.push({ - ruleId, - channel, - queryName, - role, - query, - surface, - executionBackend, - ...(reportOnly ? { reportOnly: true } : {}), - expected: 0, - actual: matches.length, - severities: matches.map((m) => m.severity), - severityMatched: true, - messageMatched: true, - backendOracleStatus: 'coverage-missing', - expectationStatus: 'coverage-missing', - }); } continue; } @@ -1307,7 +1275,18 @@ function main() { try { assertExactQueryCoverage(spec, expectation); } catch (error) { - fatal(`Invalid contract ${file}: ${error.message}`); + for (const [queryName, queryDef] of Object.entries(queries)) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `invalid contract ${file}: ${error.message}`, + reportOnly, + scoringFailures, + }); + } + continue; } for (const queryName of Object.keys(queries)) { const queryDef = queries[queryName]; @@ -1318,7 +1297,16 @@ function main() { try { oracleSelection = resolveBackendOracle(spec, expected, executionBackend); } catch (error) { - fatal(`Invalid contract ${file} query "${queryName}": ${error.message}`); + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + error: `invalid contract ${file} query "${queryName}": ${error.message}`, + reportOnly, + scoringFailures, + }); + continue; } const frontendOracle = oracleSelection.frontend; const expectedCount = frontendOracle.count; @@ -1349,206 +1337,191 @@ function main() { continue; } - if (channel === 'syntax' && typeof validateSyntax !== 'function') { - fatal( - `Syntax contract "${ruleId}" requires validateQueryWithBundle from ${SYNTAX_MODULE}. ` + - `Validate this SQL branch against the OSD headless-syntax PR.` - ); - } - const result = - channel === 'syntax' - ? validateSyntax(query, grammar) - : lintQuery(query, grammar, context); - const allFrontendFindings = - channel === 'syntax' ? result.errors || [] : result.diagnostics || []; - const matches = - channel === 'syntax' - ? allFrontendFindings.filter((finding) => finding.code === frontendOracle.code) - : allFrontendFindings.filter((finding) => finding.ruleId === ruleId); - const actual = matches.length; - const ok = actual === expectedCount; - - log( - ` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${queryName} (${role}): ` + - `expected ${expectedCount}, got ${actual} — ${query}` - ); + try { + let result; + if (channel === 'syntax' && typeof validateSyntax !== 'function') { + throw new Error( + `syntax validation requires validateQueryWithBundle from ${SYNTAX_MODULE}` + ); + } + result = + channel === 'syntax' + ? validateSyntax(query, grammar) + : lintQuery(query, grammar, context); + const allFrontendFindings = + channel === 'syntax' ? result.errors || [] : result.diagnostics || []; + const matches = + channel === 'syntax' + ? allFrontendFindings.filter((finding) => finding.code === frontendOracle.code) + : allFrontendFindings.filter((finding) => finding.ruleId === ruleId); + const actual = matches.length; + const ok = actual === expectedCount; - const evaluated = evaluateFrontendAssertions({ - channel, - query, - matches, - allFrontendFindings, - frontendOracle, - decideAction, - }); - const assertions = { count: ok, ...evaluated.assertions }; - const mismatches = [ - ...(ok - ? [] - : [ - { - field: 'count', - expected: expectedCount, - actual, - }, - ]), - ...evaluated.mismatches, - ]; - - const resultEntry = { - ruleId, - channel, - queryName, - role, - query, - expected: expectedCount, - actual, - severities: matches.map((m) => m.severity).filter(Boolean), - severityMatched: evaluated.severityMatched, - messageMatched: evaluated.messageMatched, - deterministicFixMatched: evaluated.deterministicFixMatched, - aiActionMatched: evaluated.aiActionMatched, - actionDecisionMatched: evaluated.actionDecisionMatched, - fixMatched: evaluated.syntaxFixMatched, - rawMessageMatched: evaluated.rawMessageMatched, - totalErrorsMatched: evaluated.totalErrorsMatched, - assertions, - mismatches, - ...(evaluated.deterministicFixActual !== undefined - ? { deterministicFix: evaluated.deterministicFixActual } - : {}), - ...(evaluated.aiActionActual !== undefined - ? { aiAction: evaluated.aiActionActual } - : {}), - ...(evaluated.actionDecisionActual !== undefined - ? { actionDecision: evaluated.actionDecisionActual } - : {}), - ...(channel === 'syntax' - ? { - code: frontendOracle.code, - codes: allFrontendFindings.map((finding) => finding.code).filter(Boolean), - totalErrors: allFrontendFindings.length, - } - : {}), - ...(reportOnly ? { reportOnly: true } : {}), - executionBackend, - backendOracleStatus: oracleSelection.status, - }; - - for (const mismatch of mismatches) { - scoringFailures.push( - `[${ruleId}/${queryName}] frontend.${mismatch.field} mismatch: ` + - `expected ${JSON.stringify(mismatch.expected)}, got ` + - `${JSON.stringify(mismatch.actual)} for: ${query}` + log( + ` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${queryName} (${role}): ` + + `expected ${expectedCount}, got ${actual} — ${query}` ); - } - if (oracleSelection.status === 'not-applicable') { - // Only the backend fixture is non-applicable. The detector still ran above and its - // count/severity/message assertions remain ordinary, comparable frontend evidence. - resultEntry.backendOracleReason = oracleSelection.reason; - } else if (oracleSelection.status === 'coverage-missing') { - resultEntry.outcome = 'coverage-missing'; - resultEntry.coverage = 'missing'; - resultEntry.reason = oracleSelection.reason; - resultEntry.coverageMissing = oracleSelection.reason; - if (!observeAnalytics) { + const evaluated = evaluateFrontendAssertions({ + channel, + query, + matches, + allFrontendFindings, + frontendOracle, + }); + const assertions = { count: ok, ...evaluated.assertions }; + const mismatches = [ + ...(ok + ? [] + : [ + { + field: 'count', + expected: expectedCount, + actual, + }, + ]), + ...evaluated.mismatches, + ]; + + const resultEntry = { + ruleId, + channel, + queryName, + role, + query, + expected: expectedCount, + actual, + severities: matches.map((m) => m.severity).filter(Boolean), + severityMatched: evaluated.severityMatched, + messageMatched: evaluated.messageMatched, + deterministicFixMatched: evaluated.deterministicFixMatched, + fixMatched: evaluated.syntaxFixMatched, + rawMessageMatched: evaluated.rawMessageMatched, + totalErrorsMatched: evaluated.totalErrorsMatched, + assertions, + mismatches, + ...(evaluated.deterministicFixActual !== undefined + ? { deterministicFix: evaluated.deterministicFixActual } + : {}), + ...(channel === 'syntax' + ? { + code: frontendOracle.code, + codes: allFrontendFindings.map((finding) => finding.code).filter(Boolean), + totalErrors: allFrontendFindings.length, + } + : {}), + ...(reportOnly ? { reportOnly: true } : {}), + executionBackend, + backendOracleStatus: oracleSelection.status, + }; + + for (const mismatch of mismatches) { scoringFailures.push( - `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` + `[${ruleId}/${queryName}] frontend.${mismatch.field} mismatch: ` + + `expected ${JSON.stringify(mismatch.expected)}, got ` + + `${JSON.stringify(mismatch.actual)} for: ${query}` ); } - } - // Differential: the observed backend behavior must agree with the observed - // detector output through the shared contract (design §3.2, §4.3). A - // rejection-kind query the backend rejected must be one the detector flags; - // a success/advisory query the backend accepted must be one the detector - // passes. This catches drift the two halves would otherwise hide by both - // pinning to the same JSON. - if (backendReport) { - const be = backendReport.get(`${ruleId}::${queryName}`); - if (!be) { - scoringFailures.push( - `[${ruleId}/${queryName}] no backend report entry (backend did not run this query).` - ); - } else { - const backendObservation = classifyBackendReportRow(be); - if (oracleSelection.status !== 'applicable') { - // A missing or non-applicable oracle is never an acceptance claim. Keep - // any backend observation visible, but do not coerce a missing verdict - // through `!!be.rejected` or score a differential against another route. - resultEntry.backendOutcome = backendObservation.status; - } else if (backendObservation.status !== 'observed') { - resultEntry.backendOutcome = backendObservation.status; + if (oracleSelection.status === 'not-applicable') { + // Only the backend fixture is non-applicable. The detector still ran above and its + // count/severity/message assertions remain ordinary, comparable frontend evidence. + resultEntry.backendOracleReason = oracleSelection.reason; + } else if (oracleSelection.status === 'coverage-missing') { + resultEntry.outcome = 'coverage-missing'; + resultEntry.coverage = 'missing'; + resultEntry.reason = oracleSelection.reason; + resultEntry.coverageMissing = oracleSelection.reason; + if (!observeAnalytics) { scoringFailures.push( - `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + - `(outcome=${JSON.stringify(backendObservation.status)}).` + `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` + ); + } + } + + // Differential: the observed backend behavior must agree with the observed + // detector output through the shared contract (design §3.2, §4.3). A + // rejection-kind query the backend rejected must be one the detector flags; + // a success/advisory query the backend accepted must be one the detector + // passes. This catches drift the two halves would otherwise hide by both + // pinning to the same JSON. + if (backendReport) { + const be = backendReport.get(`${ruleId}::${queryName}`); + if (!be) { + scoringFailures.push( + `[${ruleId}/${queryName}] no backend report entry (backend did not run this query).` ); } else { - const backendKind = oracleSelection.oracle.kind; - const expectRejected = backendKind === 'rejection'; - const backendRejected = backendObservation.rejected; - resultEntry.backendRejected = backendRejected; - if (backendRejected !== expectRejected) { - scoringFailures.push( - `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + - `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` - ); - } - // Trigger cross-check: a trigger the detector flags must be one the engine - // ALSO objects to — but only where the contract claims the engine objects - // at all. - // - // For a `rejection` rule the two coincide: detector flags <-> engine - // rejects, and a disagreement means one side drifted. That is the original - // check and it is unchanged. - // - // An ADVISORY rule is different by design. It flags a query the engine - // runs happily: `head-without-sort` marks non-determinism, - // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via - // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that - // rule working, not drift — so pairing the detector against `be.rejected` - // failed every advisory trigger unconditionally. That, not runtime cost, - // is the structural reason those contracts could only run nightly. - // - // The contracts already carry the distinction in `backend.kind`, so this - // reads data that exists rather than adding a flag. Advisory triggers keep - // full coverage from the other two assertions: the backend-kind check above - // fires if the engine starts REJECTING a query pinned as accepted, and the - // `detectorCount` assertion fires if the detector stops flagging it. Only - // the pairing rule is scoped to the rules it makes sense for. - const detectorFlagged = actual > 0; - if (role === 'trigger' && expectRejected && detectorFlagged !== backendRejected) { - scoringFailures.push( - `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` - ); - } - // A control must pass on both sides regardless of kind: it is a valid - // query the rule has to stay quiet on. Unlike a trigger, that claim does - // not vary with `backend.kind`. - if (role === 'control' && (detectorFlagged || backendRejected)) { + const backendObservation = classifyBackendReportRow(be); + if (oracleSelection.status !== 'applicable') { + // A missing or non-applicable oracle is never an acceptance claim. Keep + // any backend observation visible, but do not coerce a missing verdict + // through `!!be.rejected` or score a differential against another route. + resultEntry.backendOutcome = backendObservation.status; + } else if (backendObservation.status !== 'observed') { + resultEntry.backendOutcome = backendObservation.status; scoringFailures.push( - `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` - ); - } - if ( - role === 'suppression-control' && - (detectorFlagged || !backendRejected) - ) { - scoringFailures.push( - `[${ruleId}/${queryName}] differential: suppression control must retain a backend ` + - `syntax rejection without a "${frontendOracle.code}" suggestion, but frontend ` + - `${detectorFlagged ? 'suggested a rewrite' : 'did not suggest a rewrite'} and ` + - `backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + + `(outcome=${JSON.stringify(backendObservation.status)}).` ); + } else { + const backendKind = oracleSelection.oracle.kind; + const expectRejected = backendKind === 'rejection'; + const backendRejected = backendObservation.rejected; + resultEntry.backendRejected = backendRejected; + if (backendRejected !== expectRejected) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Pair detector and backend rejection only for rejection rules. + // Advisory rules intentionally flag queries the backend accepts. + const detectorFlagged = actual > 0; + if ( + role === 'trigger' && + expectRejected && + detectorFlagged !== backendRejected + ) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if (role === 'control' && (detectorFlagged || backendRejected)) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if ( + role === 'suppression-control' && + (detectorFlagged || !backendRejected) + ) { + scoringFailures.push( + `[${ruleId}/${queryName}] differential: suppression control must retain a backend ` + + `syntax rejection without a "${frontendOracle.code}" suggestion, but frontend ` + + `${detectorFlagged ? 'suggested a rewrite' : 'did not suggest a rewrite'} and ` + + `backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } } } } - } - report.results.push(resultEntry); + report.results.push(resultEntry); + } catch (error) { + recordExecutionError({ + spec, + channel, + queryName, + queryDef, + expected: expectedCount, + error, + reportOnly, + scoringFailures, + }); + } } } From e3fef6808418e51d9323eff2855cc0fb09064196 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 13:37:18 -0700 Subject: [PATCH 73/78] test(ci): prove PPL lint fault isolation Signed-off-by: Hanyu Wei --- .../__tests__/run-frontend-contract.test.mjs | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs index ff0b48f9100..e8ffa5ad281 100644 --- a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -4,7 +4,12 @@ */ import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; import { assertActiveShippingContracts, @@ -14,6 +19,8 @@ import { selectManifestContractNames, } from '../run-frontend-contract.mjs'; +const SCRIPT = fileURLToPath(new URL('../run-frontend-contract.mjs', import.meta.url)); + const RANGE = { startLine: 1, startColumn: 11, @@ -161,6 +168,133 @@ test('a frontend execution error remains a complete report row', () => { ); }); +test('the runner writes later rule rows after one frontend execution error', (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-runner-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const osdRoot = path.join(root, 'osd'); + const contractDir = path.join(root, 'contracts'); + const reportPath = path.join(root, 'detector-report.json'); + const grammarPath = path.join(root, 'ppl-grammar-bundle.json'); + const targetPath = path.join(root, 'target.json'); + fs.mkdirSync( + path.join(osdRoot, 'src/plugins/data/public/antlr/opensearch_ppl'), + { recursive: true } + ); + fs.mkdirSync(path.join(osdRoot, 'packages/osd-monaco'), { recursive: true }); + fs.mkdirSync(contractDir, { recursive: true }); + + const wiring = (ruleId) => ({ + detector: ruleId, + enabled: true, + severity: 'info', + runtimeOnly: false, + needsContext: false, + needsExplain: false, + sourceScoped: false, + appliesTo: {}, + }); + const contract = (ruleId, query) => ({ + schemaVersion: 4, + ruleId, + grammarSurface: 'runtime-bundle', + schedule: 'pr', + wiring: wiring(ruleId), + index: 'test-index', + queries: { + trigger: { role: 'trigger', query }, + }, + expectations: [ + { + version: '>=0.0.0', + queries: { + trigger: { + frontend: { + count: 0, + deterministicFix: { offered: false }, + }, + backends: { + standard: { kind: 'advisory', httpStatus: 200 }, + analytics: { kind: 'advisory', httpStatus: 200 }, + }, + }, + }, + }, + ], + }); + const files = ['first-rule.spec.json', 'second-rule.spec.json']; + fs.writeFileSync( + path.join(contractDir, files[0]), + JSON.stringify(contract('first-rule', 'source={{index}} | fail')) + ); + fs.writeFileSync( + path.join(contractDir, files[1]), + JSON.stringify(contract('second-rule', 'source={{index}} | pass')) + ); + fs.writeFileSync( + path.join(contractDir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 4, + contracts: files, + defaultError: [], + requiredSyntaxFeatures: [], + }) + ); + fs.writeFileSync( + path.join(osdRoot, 'packages/osd-monaco/ppl-lint.js'), + `const wiring = (id) => ({ + id, detector: id, enabled: true, severity: 'info', runtimeOnly: false, + needsContext: false, needsExplain: false, sourceScoped: false, appliesTo: {} + }); + exports.getBundledCatalog = () => [wiring('first-rule'), wiring('second-rule')];` + ); + fs.writeFileSync( + path.join( + osdRoot, + 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint.js' + ), + `exports.deserializeBundleOrThrow = (bundle) => bundle; + exports.lintQueryWithBundle = (query) => { + if (query.includes('| fail')) throw new Error('detector crashed'); + return { diagnostics: [] }; + };` + ); + fs.writeFileSync(grammarPath, JSON.stringify({ grammarHash: 'sha256:test' })); + fs.writeFileSync( + targetPath, + JSON.stringify({ + schemaVersion: 2, + executionBackend: 'standard', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + storage: 'lucene', + shardCount: 1, + }) + ); + + const result = spawnSync(process.execPath, [SCRIPT], { + cwd: osdRoot, + encoding: 'utf8', + env: { + ...process.env, + PPL_LINT_CONTRACT_DIR: contractDir, + PPL_LINT_SCHEDULE: 'pr', + PPL_LINT_GRAMMAR_BUNDLE: grammarPath, + PPL_LINT_TARGET_MANIFEST: targetPath, + PPL_LINT_REPORT: reportPath, + }, + }); + + assert.equal(result.status, 1); + assert.match(result.stderr, /first-rule\/trigger.*frontend\.execution failed/s); + assert.match(result.stdout, /PASS second-rule\/trigger/); + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + assert.equal(report.results.length, 2); + assert.equal(report.results[0].outcome, 'error'); + assert.equal(report.results[0].error, 'detector crashed'); + assert.equal(report.results[1].ruleId, 'second-rule'); + assert.equal(report.results[1].actual, 0); +}); + test('syntax suppression checks raw parser errors outside the suggestion code filter', () => { const parserErrors = [ { From 1394b183c1d57309be83fe22134598b9002568ad Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 15:31:26 -0700 Subject: [PATCH 74/78] fix(ci): publish compatibility report before drift failure Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 561 +++--------------- ...ppl-lint-analytics-engine-ci-validation.md | 9 +- ...pl-lint-runtime-compatibility-ci-design.md | 188 ++++++ scripts/ppl-lint/README.md | 114 +--- .../__tests__/aggregate-versions.test.mjs | 177 ++++-- .../validate-pr-build-targets.test.mjs | 271 --------- scripts/ppl-lint/aggregate-versions.mjs | 239 +++----- .../ppl-lint/validate-pr-build-targets.mjs | 285 --------- 8 files changed, 504 insertions(+), 1340 deletions(-) create mode 100644 docs/dev/ppl-lint-runtime-compatibility-ci-design.md delete mode 100644 scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs delete mode 100644 scripts/ppl-lint/validate-pr-build-targets.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 294197584f4..ef169a78bb5 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -33,10 +33,9 @@ name: PPL lint multi-version validation # not export a candidate grammar bundle, so the detector half would have nothing # to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. # -# Non-enforcing on purpose, for now: it reports and uploads, and the required -# check stays the sibling workflow's `validation-result`. Promoting this to -# required needs a green baseline across the whole matrix first (a rule that has -# quietly drifted on 3.6 would otherwise block every unrelated PR on day one). +# Compatibility differences are collected without interrupting the matrix. +# The aggregate writes the complete table and JSON artifact first, then the +# final step fails this job when a declared-supported rule has drifted. on: # Nightly is the primary schedule: the matrix pulls three engine images, so it @@ -67,10 +66,6 @@ on: description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' required: false type: string - compiled_versions: - description: 'JSON array of engine versions to validate on the compiled-simplified surface, e.g. ["2.19.0"]. Use "[]" to skip them.' - required: false - type: string permissions: contents: read @@ -87,23 +82,6 @@ env: # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch # exists — that would validate an engine no user runs). ENGINE_VERSIONS: '["3.6.0","3.7.0"]' - # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. - # - # These engines cannot export a grammar bundle (GET /_plugins/_ppl/_grammar - # landed in 3.6), so the runtime surface cannot reach them at all. But the - # compiled surface has no such floor: it lints with OSD's own checked-in grammar, - # which is exactly what a user gets when no bundle is available — including every - # user on an engine below 3.6. Those legs still run the real contract queries - # against the real engine, so the backend half of the differential is genuine. - # - # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are - # reported not-applicable. Nightly only — see the `compiled_versions` input to - # run one ad hoc. - # - # Always the LATEST PATCH of each line, never `.0`. A user on 2.19 is on - # 2.19.6, so validating 2.19.0 tests an engine nobody runs and attributes any - # bug fixed in between to the whole line. - COMPILED_ENGINE_VERSIONS: '["2.19.6","3.0.0","3.5.0"]' jobs: # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a @@ -120,7 +98,6 @@ jobs: runs-on: ubuntu-latest outputs: released: ${{ steps.plan.outputs.released }} - compiled: ${{ steps.plan.outputs.compiled }} discovery_engine: ${{ steps.plan.outputs.discovery_engine }} osd_repo: ${{ steps.plan.outputs.osd_repo }} osd_ref: ${{ steps.plan.outputs.osd_ref }} @@ -130,9 +107,6 @@ jobs: env: REQUESTED_VERSIONS: ${{ inputs.engine_versions }} DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} - REQUESTED_COMPILED: ${{ inputs.compiled_versions }} - DEFAULT_COMPILED: ${{ env.COMPILED_ENGINE_VERSIONS }} - EVENT_NAME: ${{ github.event_name }} REQUESTED_REPO: ${{ inputs.osd_repo }} REQUESTED_REF: ${{ inputs.osd_ref }} VAR_REPO: ${{ vars.OSD_REPO }} @@ -151,28 +125,6 @@ jobs: " echo "released=$released" >> "$GITHUB_OUTPUT" - # Compiled-surface legs add three more engine images, so they run on the - # nightly schedule (and on an explicit dispatch), not on every PR that - # touches the corpus. An explicit input always wins, including "[]". - if [ -n "${REQUESTED_COMPILED:-}" ]; then - compiled="$REQUESTED_COMPILED" - elif [ "$EVENT_NAME" = "pull_request" ]; then - compiled='[]' - else - compiled="$DEFAULT_COMPILED" - fi - # An EMPTY list is legitimate here (unlike engine_versions): it means "skip - # the compiled surface this run". Still reject a non-list. - echo "$compiled" | python3 -c " - import json,sys - v=json.load(sys.stdin) - assert isinstance(v,list), 'compiled_versions must be a JSON array' - for item in v: - assert isinstance(item,str), 'compiled_versions entries must be strings' - " - echo "compiled=$compiled" >> "$GITHUB_OUTPUT" - echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" - # Discovery runs against ONE engine — the newest released version in the # matrix. It is a lead-generator, not a version-drift check, so paying for # a full matrix would multiply cost without adding signal: a false positive @@ -294,191 +246,6 @@ jobs: name: ppl-lint-leg-${{ matrix.version }}-logs path: integ-test/build/reports/** - # Legs for engines BELOW the _grammar endpoint floor (3.6). These cannot export a - # grammar bundle, so they are observed for backend behavior only and their - # detector pass runs on the compiled-simplified surface — which is what a real - # user on such an engine gets, since no bundle can ever load there. - # - # Identical to observe-released except that `-Dppl.lint.grammar.bundle` is - # omitted: the IT skips the export when that property is unset, so no bundle - # fetch is attempted against an engine that has no such endpoint. - observe-compiled: - name: Observe engine ${{ matrix.version }} (compiled surface) - needs: plan - # An empty compiled list means "skip this surface" (the pull_request default). - if: ${{ needs.plan.outputs.compiled != '[]' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - version: ${{ fromJSON(needs.plan.outputs.compiled) }} - services: - opensearch: - image: opensearchproject/opensearch:${{ matrix.version }} - env: - discovery.type: single-node - DISABLE_SECURITY_PLUGIN: 'true' - DISABLE_INSTALL_DEMO_CONFIG: 'true' - OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g - ports: - - 9200:9200 - options: >- - --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" - --health-interval 15s - --health-timeout 10s - --health-retries 20 - --health-start-period 60s - steps: - - name: Checkout SQL pull request - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Wait for the engine and confirm its version - id: engine - run: | - set -euo pipefail - for i in $(seq 1 40); do - if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi - echo "waiting for engine (${i}/40)..." - sleep 5 - done - cat /tmp/root.json - reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") - case "$reported" in - ${{ matrix.version }}*) ;; - *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; - esac - curl -sf http://localhost:9200/_cat/plugins | grep -i sql - - # `_cluster/health` goes GREEN before the bundled plugins finish creating - # their system indices, and the IT's first act is to wipe every non-system - # index. On 2.19 that DELETE landed while ML Commons was still initializing - # `.plugins-ml-config` and blocked until the client's 60s socket timeout, - # failing the leg before a single contract query ran. Wait for the plugin - # indices to stop appearing, so the wipe cannot race initialization. - - name: Wait for bundled plugin system indices to settle - run: | - set -euo pipefail - previous="" - stable=0 - for i in $(seq 1 30); do - current=$(curl -sf "http://localhost:9200/_cat/indices?h=index&expand_wildcards=all" \ - | sort | tr '\n' ',' || true) - if [ -n "$current" ] && [ "$current" = "$previous" ]; then - stable=$((stable + 1)) - # Three consecutive identical listings: no plugin is still creating - # indices. One match is not enough — initialization has gaps between - # an index being created and the next one starting. - if [ "$stable" -ge 3 ]; then - echo "index set stable after ${i} poll(s): $current" - exit 0 - fi - else - stable=0 - fi - previous="$current" - sleep 2 - done - # Not fatal: a slow-but-working engine should still be observed. The IT - # tolerates a wipe failure per index, and a genuinely unreachable cluster - # fails loudly in the next step anyway. - echo "::warning::plugin index set did not stabilize; continuing" - - # Probe the EXACT requests the test framework makes before any test runs. - # `OpenSearchRestTestCase.initClient` issues `GET _nodes/plugins`, and - # `wipeAllOpenSearchIndices` issues `GET _cat/indices?expand_wildcards=all`. - # A leg that dies with a bare socket timeout gives no clue which of those - # hung, so time them here where the output is readable. - - name: Probe the framework's own startup requests - run: | - set -uo pipefail - for path in "_nodes/plugins" "_cat/indices?format=json&expand_wildcards=all" "_cluster/health"; do - start=$(date +%s) - if curl -sS --max-time 30 -o /tmp/probe.out -w '%{http_code}' \ - "http://localhost:9200/${path}" > /tmp/probe.code 2>/tmp/probe.err; then - echo "OK $(($(date +%s) - start))s HTTP $(cat /tmp/probe.code) ${path} ($(wc -c < /tmp/probe.out) bytes)" - else - echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" - fi - done - start=$(date +%s) - if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ - "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then - echo "h2 probe: negotiated HTTP/$(cat /tmp/h2.out) in $(($(date +%s) - start))s" - else - echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" - fi - # Response SIZE is the last untested difference. curl streams the body and - # does not care; the test framework calls entityAsMap on it, and - # _nodes/plugins on an engine with many bundled plugins is large. Record the - # sizes so a size-dependent hang is visible rather than inferred. - for path in "_nodes/plugins" "_nodes" "_cat/plugins"; do - bytes=$(curl -sS --max-time 30 "http://localhost:9200/${path}" | wc -c) - echo "size probe: ${path} -> ${bytes} bytes" - done - - - name: Set up JDK 21 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 - with: - distribution: 'temurin' - java-version: 21 - - # The curl probes above reach this engine instantly, yet the contract IT's own - # client times out on the same endpoint before any test body runs. Everything - # curl can tell us has been exhausted, so run the probe from a JVM: raw TCP, - # then HttpURLConnection, then the real OpenSearch RestClient per endpoint. - # Whichever layer stops working is the answer. - # - # `continue-on-error` because this is a diagnostic: its findings must not be - # what decides the leg, and the contract step below is still the real check. - - name: Probe REST client connectivity from a JVM - continue-on-error: true - run: | - set -uo pipefail - ./gradlew :integ-test:integTestRemote \ - --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ - -Dtests.rest.cluster=localhost:9200 \ - -Dtests.cluster=localhost:9200 \ - -Dtests.clustername=docker-cluster \ - --info 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD|tests? completed|No tests found' || true - - - name: Run contract observation against engine ${{ matrix.version }} - run: | - set -euo pipefail - mkdir -p leg - # No -Dppl.lint.grammar.bundle: this engine predates the _grammar endpoint, - # and the IT correctly exports nothing when the property is unset. - ./gradlew :integ-test:integTestRemote \ - --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ - -Dtests.rest.cluster=localhost:9200 \ - -Dtests.cluster=localhost:9200 \ - -Dtests.clustername=docker-cluster \ - -Dppl.lint.schedule=nightly \ - -Dppl.lint.observe.only=true \ - -Dppl.lint.execution_backend=standard \ - -Dppl.lint.sql_sha="${GITHUB_SHA}" \ - -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ - -Dppl.lint.target="$(pwd)/leg/target.json" - # Mark the leg so the detect job knows to lint it on the compiled surface. - # A leg with no bundle would otherwise look like a failed export. - echo 'compiled-simplified' > leg/surface - - - name: Upload leg artifacts - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: ppl-lint-leg-${{ matrix.version }}-compiled - path: leg - if-no-files-found: error - - - name: Upload failure logs - if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true - with: - name: ppl-lint-leg-${{ matrix.version }}-compiled-logs - path: integ-test/build/reports/** - # The PR's own engine build, so the newest point in the matrix is the code under # review rather than the last release. Same oracle as the released legs; the only # difference is a Gradle-managed cluster instead of a published image, which is @@ -541,179 +308,6 @@ jobs: integ-test/build/test-results/** integ-test/build/testclusters/*/logs/* - # The PR build through the full composite/Parquet + DataFusion stack. This is - # an observation leg: route/identity/infrastructure failures are fatal, while - # backend oracles are promoted only after their captured behavior is reviewed. - observe-pr-build-analytics: - name: Observe engine pr-build (analytics) - needs: Get-CI-Image-Tag - runs-on: ubuntu-latest - timeout-minutes: 30 - env: - ANALYTICS_FEATURE_BUILD_LATEST: https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch - container: - image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} - steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - - name: Checkout SQL pull request - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Set up JDK 25 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 - with: - distribution: 'temurin' - java-version: 25 - - - name: Resolve analytics feature build - id: analytics-build - run: | - set -euo pipefail - mkdir -p leg - requested_manifest="${ANALYTICS_FEATURE_BUILD_LATEST}/manifest.yml" - resolved_manifest=$(curl --fail --silent --show-error --location \ - --retry 3 --retry-all-errors \ - --output leg/analytics-feature-manifest.yml \ - --write-out '%{url_effective}' \ - "$requested_manifest") - artifact_root="${resolved_manifest%/manifest.yml}" - plugin_base="${artifact_root}/plugins" - native_url="${artifact_root}/dist/libopensearch_native.so" - { - echo "artifact_root=$artifact_root" - echo "plugin_base=$plugin_base" - echo "native_url=$native_url" - } >> "$GITHUB_OUTPUT" - ANALYTICS_ARTIFACT_ROOT="$artifact_root" \ - ANALYTICS_PLUGIN_BASE="$plugin_base" \ - ANALYTICS_NATIVE_URL="$native_url" \ - ANALYTICS_RESOLVED_MANIFEST="$resolved_manifest" \ - python3 - <<'PY' - import hashlib - import json - import os - from pathlib import Path - - manifest = Path("leg/analytics-feature-manifest.yml") - context = { - "schemaVersion": 1, - "stage": "feature-build-resolved", - "sqlSha": os.environ["GITHUB_SHA"], - "executionBackend": "analytics", - "storage": "composite-parquet", - "artifactRoot": os.environ["ANALYTICS_ARTIFACT_ROOT"], - "pluginBase": os.environ["ANALYTICS_PLUGIN_BASE"], - "nativeLibraryUrl": os.environ["ANALYTICS_NATIVE_URL"], - "resolvedManifestUrl": os.environ["ANALYTICS_RESOLVED_MANIFEST"], - "manifestSha256": "sha256:" + hashlib.sha256(manifest.read_bytes()).hexdigest(), - } - Path("leg/analytics-bootstrap.json").write_text( - json.dumps(context, indent=2) + "\n", encoding="utf-8" - ) - PY - - - name: Run analytics contract observation against the PR build - id: analytics-observation - env: - ANALYTICS_FEATURE_BUILD_BASE: ${{ steps.analytics-build.outputs.plugin_base }} - ANALYTICS_NATIVE_LIB_URL: ${{ steps.analytics-build.outputs.native_url }} - run: | - set -euo pipefail - chown -R 1000:1000 "$(pwd)" - su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ - -PanalyticsFeatureBuildBase=${ANALYTICS_FEATURE_BUILD_BASE} \ - -PanalyticsNativeLibUrl=${ANALYTICS_NATIVE_LIB_URL} \ - -Dppl.lint.schedule=nightly \ - -Dppl.lint.observe.only=true \ - -Dppl.lint.sql_sha=${GITHUB_SHA} \ - -Dppl.lint.report=$(pwd)/leg/backend-report.json \ - -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ - -Dppl.lint.target=$(pwd)/leg/target.json" - - - name: Record analytics bootstrap provenance - if: ${{ always() }} - env: - OBSERVATION_OUTCOME: ${{ steps.analytics-observation.outcome }} - ANALYTICS_ARTIFACT_ROOT: ${{ steps.analytics-build.outputs.artifact_root }} - ANALYTICS_PLUGIN_BASE: ${{ steps.analytics-build.outputs.plugin_base }} - ANALYTICS_NATIVE_URL: ${{ steps.analytics-build.outputs.native_url }} - run: | - mkdir -p leg - python3 - <<'PY' - import hashlib - import json - import os - from pathlib import Path - - context_file = Path("leg/analytics-bootstrap.json") - if context_file.exists(): - context = json.loads(context_file.read_text(encoding="utf-8")) - else: - context = { - "schemaVersion": 1, - "sqlSha": os.environ["GITHUB_SHA"], - "executionBackend": "analytics", - "storage": "composite-parquet", - "artifactRoot": os.environ.get("ANALYTICS_ARTIFACT_ROOT") or None, - "pluginBase": os.environ.get("ANALYTICS_PLUGIN_BASE") or None, - "nativeLibraryUrl": os.environ.get("ANALYTICS_NATIVE_URL") or None, - } - - def describe(file): - digest = hashlib.sha256() - with file.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return { - "path": str(file), - "size": file.stat().st_size, - "sha256": "sha256:" + digest.hexdigest(), - } - - distributions = Path("integ-test/build/distributions") - native_libraries = sorted( - Path("integ-test/build/native").glob( - "*/release/libopensearch_native.so" - ) - ) - artifacts = ( - [describe(file) for file in sorted(distributions.glob("*.zip"))] - if distributions.is_dir() - else [] - ) - artifacts.extend(describe(file) for file in native_libraries) - context["stage"] = "observation-finished" - context["outcome"] = os.environ.get("OBSERVATION_OUTCOME") or "not-run" - context["effectiveJavaLibraryPaths"] = [ - str(file.parent) for file in native_libraries - ] - context["downloadedArtifacts"] = artifacts - context_file.write_text( - json.dumps(context, indent=2) + "\n", encoding="utf-8" - ) - PY - - - name: Upload analytics leg artifacts - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: ppl-lint-leg-pr-build-analytics - path: leg - if-no-files-found: error - - - name: Upload analytics failure logs - if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true - with: - name: ppl-lint-leg-pr-build-analytics-logs - path: | - integ-test/build/reports/** - integ-test/build/test-results/** - integ-test/build/testclusters/*/logs/* - # Lint each engine's exported grammar with the OSD detectors. Separate from the # observation legs because OSD needs a newer Node/glibc than the engine image # provides, and because one bootstrap can serve every leg. @@ -722,13 +316,11 @@ jobs: # partial matrix must be visibly partial, not silently absent. The aggregate # step fails if NO leg produced a report. detect: - name: Detect on each engine grammar + name: Aggregate rule compatibility needs: - plan - observe-released - - observe-compiled - observe-pr-build - - observe-pr-build-analytics if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 40 @@ -744,16 +336,6 @@ jobs: pattern: ppl-lint-leg-* path: legs - - name: Validate PR-build target identity - run: | - node scripts/ppl-lint/validate-pr-build-targets.mjs \ - --standard legs/ppl-lint-leg-pr-build/target.json \ - --analytics legs/ppl-lint-leg-pr-build-analytics/target.json \ - --standard-report legs/ppl-lint-leg-pr-build/backend-report.json \ - --analytics-report legs/ppl-lint-leg-pr-build-analytics/backend-report.json \ - --contracts integ-test/src/test/resources/ppl-lint/contracts \ - --schedule nightly - - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -814,26 +396,15 @@ jobs: fi for leg in "${legs[@]}"; do version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - # A leg is compiled-surface when its observe job said so. That marker is - # what distinguishes "this engine has no _grammar endpoint" from "the - # bundle export failed", which must stay a hard error. - if [ -f "$leg/surface" ] && [ "$(cat "$leg/surface")" = 'compiled-simplified' ]; then - surface_env=(PPL_LINT_SURFACE=compiled-simplified) - echo "=== detectors vs engine $version (compiled-simplified surface) ===" - elif [ -f "$leg/ppl-grammar-bundle.json" ]; then - surface_env=(PPL_LINT_SURFACE=runtime-bundle - PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json") - echo "=== detectors vs engine $version (runtime-bundle surface) ===" - else + if [ ! -f "$leg/ppl-grammar-bundle.json" ]; then # Skip the log-only artifacts an observation failure may have uploaded. - echo "skipping $leg (no grammar bundle and no compiled-surface marker)" + echo "skipping $leg (no runtime grammar bundle)" continue fi - observe_env=(PPL_LINT_OBSERVE_ONLY=1) - if [ "$(jq -r '.executionBackend // empty' "$leg/target.json")" = 'analytics' ]; then - observe_env+=(PPL_LINT_OBSERVE_ANALYTICS=1) - fi - env "${surface_env[@]}" "${observe_env[@]}" \ + echo "=== detectors vs engine $version (runtime-bundle surface) ===" + env PPL_LINT_SURFACE=runtime-bundle \ + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ + PPL_LINT_OBSERVE_ONLY=1 \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_INCLUDE_DORMANT=1 \ @@ -860,7 +431,6 @@ jobs: id: aggregate env: RELEASED: ${{ needs.plan.outputs.released }} - COMPILED: ${{ needs.plan.outputs.compiled }} run: | set -euo pipefail shopt -s nullglob @@ -881,14 +451,7 @@ jobs: # a matrix that silently lost one — the exact vacuous pass this workflow # exists to prevent. A dead leg is a failure, not a smaller matrix. missing=() - # Compiled legs are labelled "-compiled" to match their artifact - # name, so they occupy their own column even when a runtime leg validated - # the same engine version. - compiled_wanted=$(echo "$COMPILED" | python3 -c " - import json,sys - print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) - ") - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build pr-build-analytics; do + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do found=no for have in "${present[@]}"; do [ "$have" = "$want" ] && found=yes && break @@ -899,27 +462,74 @@ jobs: echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." exit 1 fi + set +e node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ --summary "$GITHUB_STEP_SUMMARY" \ --all-rules \ - --observe-analytics \ "${args[@]}" + aggregate_exit=$? + set -e + echo "exit_code=$aggregate_exit" >> "$GITHUB_OUTPUT" - name: Upload drift report if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true with: name: ppl-lint-multiversion-drift + if-no-files-found: error + path: drift-report.json + + - name: Upload compatibility evidence + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-multiversion-evidence + if-no-files-found: warn path: | - drift-report.json legs/**/detector-report.json legs/**/detector.log legs/**/target.json - legs/**/analytics-bootstrap.json - legs/**/analytics-feature-manifest.yml + + - name: Fail after publishing compatibility drift + if: ${{ always() }} + env: + AGGREGATE_EXIT: ${{ steps.aggregate.outputs.exit_code }} + RELEASED: ${{ needs.plan.outputs.released }} + run: | + set -euo pipefail + if [ -z "$AGGREGATE_EXIT" ]; then + echo "::error::aggregation did not complete" + exit 1 + fi + expected_legs=$(echo "$RELEASED" | jq 'length + 1') + expected_rules=$(jq -c \ + '[.contracts[] | sub("\\.spec\\.json$"; "")] | sort' \ + integ-test/src/test/resources/ppl-lint/contracts/manifest.json) + if [ ! -s drift-report.json ] || ! jq -e \ + --argjson expected_legs "$expected_legs" \ + --argjson expected_rules "$expected_rules" \ + ' + type == "object" and + ($expected_rules | length) == 12 and + ($expected_rules | index("command-suggestion") | not) and + (.legs | type == "array" and length == $expected_legs) and + (.matrix | type == "array") and + (.matrix | length) == (12 * $expected_legs) and + (.matrix | map(.ruleId) | unique | sort) == $expected_rules and + (.matrix | map(.legKey) | unique | length) == $expected_legs and + ([.matrix[] | [.ruleId, .legKey]] | unique | length) == + (12 * $expected_legs) + ' drift-report.json > /dev/null; then + echo "::error::aggregation did not produce the complete 12-rule compatibility matrix" + exit 1 + fi + if [ "$AGGREGATE_EXIT" -ne 0 ]; then + echo "::error::rule compatibility validation failed; see the table above and the ppl-lint-multiversion-drift artifact" + exit "$AGGREGATE_EXIT" + fi # Discovery: harvest queries from OSD's own lint tests, run both halves over them, # and report detector/engine disagreements as LEADS. @@ -1057,12 +667,8 @@ jobs: -H 'content-type: application/json' \ -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' - # Export this engine's grammar bundle so the detector pass can run on the - # RUNTIME surface. That surface matters more than the compiled one here: the - # four `runtimeOnly` rules (union/multisearch/replace arity) are SKIPPED by - # lint_runner on the compiled grammar because the productions they walk do not - # exist there — so a compiled-only discovery run cannot observe them at all, - # and three of the four ship at error severity. + # Export this engine's grammar bundle. Discovery only runs against the runtime + # grammar surface, so an unavailable endpoint skips the detector pass. - name: Export the engine grammar bundle id: bundle run: | @@ -1086,23 +692,8 @@ jobs: " echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" else - # Not fatal. Discovery is best-effort, and the compiled surface still - # covers 12 of the rules — a lead-generator that produces nothing because - # one endpoint was unavailable is worse than one with narrower coverage. - # The surface is recorded in the report, so a reader can see which ran. - echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." - echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" - python3 -c " - import json - json.dump({'schemaVersion': 2, - 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', - 'grammarHash': '', - 'grammarBundle': '', - 'executionBackend': 'standard', - 'storage': 'lucene', - 'shardCount': 1}, - open('$GITHUB_WORKSPACE/discovery-target.json','w')) - " + echo "::warning::_grammar export failed; skipping discovery detector pass." + echo "surface=unavailable" >> "$GITHUB_OUTPUT" fi - name: Run the detectors over the discovery corpus @@ -1111,21 +702,17 @@ jobs: SURFACE: ${{ steps.bundle.outputs.surface }} run: | set -uo pipefail - # Seeded with a harmless assignment rather than left empty: under `set -u`, - # expanding an empty array as "${a[@]}" is an unbound-variable error in bash - # before 4.4, which would crash the compiled-surface fallback — the very - # path that only runs when something else already went wrong. - extra=(PPL_LINT_DISCOVERY=1 - PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") - if [ "$SURFACE" = 'runtime-bundle' ]; then - extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" - ) + if [ "$SURFACE" != 'runtime-bundle' ]; then + echo "Discovery detector pass skipped: runtime grammar bundle unavailable." + exit 0 fi # A non-zero exit is EXPECTED and ignored: the generated specs carry # placeholder expectations, so the runner reports a "failure" for every # query whose real diagnostic count differs. Only the report is read. - env "${extra[@]}" \ - PPL_LINT_SURFACE="$SURFACE" \ + env PPL_LINT_DISCOVERY=1 \ + PPL_LINT_SURFACE=runtime-bundle \ + PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" \ + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md index 3777f561dc7..ba7632b2d54 100644 --- a/docs/dev/ppl-lint-analytics-engine-ci-validation.md +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -1,11 +1,18 @@ # Analytics Engine Coverage for PPL Lint CI Validation -- **Status:** Validated for phased implementation +- **Status:** Deferred; not part of PPL lint pull-request or multi-version CI - **Last updated:** 2026-07-28 - **Scope:** PPL lint contract validation in `.github/workflows/ppl-lint-rule-validation.yml` and `.github/workflows/ppl-lint-multiversion-validation.yml` +> **Decision update (2026-08-04):** Analytics-engine lint validation is +> deferred because the feature build and composite/Parquet fixture surface are +> not stable enough for this compatibility workflow. The active design is +> [PPL Lint Runtime Compatibility CI](ppl-lint-runtime-compatibility-ci-design.md), +> which covers standard runtime-bundle engines only. This document is retained +> as future design context and is not an implementation commitment. + ## 1. Summary The PPL lint CI contract currently compares OpenSearch Dashboards (OSD) diff --git a/docs/dev/ppl-lint-runtime-compatibility-ci-design.md b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md new file mode 100644 index 00000000000..eb0c7242fbe --- /dev/null +++ b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md @@ -0,0 +1,188 @@ +# PPL Lint Runtime Compatibility CI + +- **Status:** Proposed revision for the SQL PPL lint CI +- **Last updated:** 2026-08-04 +- **Scope:** `.github/workflows/ppl-lint-multiversion-validation.yml` + +## 1. Decision + +The multi-version workflow validates PPL lint compatibility only against +standard OpenSearch runtime grammar bundles: + +```text +OpenSearch 3.6 release ─┐ +OpenSearch 3.7 release ─┼─> Aggregate rule compatibility +SQL pull request build ─┘ +``` + +The workflow does not run: + +- the compiled-simplified grammar surface for pre-3.6 engines; +- the analytics engine or composite/Parquet storage; +- syntax-channel features; +- AI action tests. + +Analytics coverage is deferred until that engine and its fixtures provide a +stable CI contract. Pre-3.6 coverage is removed because those engines cannot +export the runtime grammar bundle consumed by the production lint path. + +The required single-version workflow remains responsible for proving that all +active shipping detectors agree with the standard SQL pull request build. The +multi-version workflow explains where each rule works and fails its final +aggregation job when a declared-supported version drifts. + +## 2. Rule Inventory + +The active inventory currently contains **12 detector rules**, not 13. +`command-suggestion` was removed from this effort and must not be silently +reintroduced as a lint rule. The final table is generated from +`manifest.json`. CI also asserts that the current inventory is exactly these 12 +rules, so adding a future reviewed rule requires an intentional guard and test +update. + +| Rule | Declared compatibility | +| --- | --- | +| `agg-on-text` | Calcite, OpenSearch >= 3.7 | +| `division-by-zero` | All runtime-bundle versions | +| `enabled-false-object` | Calcite, OpenSearch >= 3.7 | +| `field-validation` | All runtime-bundle versions | +| `invalid-capture-group-name` | OpenSearch >= 3.4 | +| `multisearch-min-subsearch` | OpenSearch >= 3.4 | +| `replace-wildcard-asymmetry` | Calcite, OpenSearch >= 3.4 | +| `rex-scan-cost` | All runtime-bundle versions | +| `type-mismatch-numeric` | Calcite, OpenSearch >= 3.7 | +| `union-min-datasets` | Calcite, OpenSearch >= 3.7 | +| `unsupported-window-function-in-eventstats` | OpenSearch >= 3.4 | +| `wildcard-source-zero-match` | All runtime-bundle versions | + +## 3. Workflow Shape + +### 3.1 Plan + +`Plan matrix` resolves: + +- released engines: `3.6.0` and `3.7.0`; +- the OSD repository and revision; +- the discovery engine, currently the newest released engine. + +There is no compiled-surface input or analytics target. + +### 3.2 Observe released engines + +One `Observe engine ` job runs per released engine. Each job: + +1. starts the official OpenSearch distribution containing its matching SQL + plugin; +2. runs the contract queries in observe-only mode; +3. exports that engine's runtime grammar bundle; +4. uploads `target.json`, `backend-report.json`, and + `ppl-grammar-bundle.json`. + +An expectation mismatch is observation data, not a job failure. + +### 3.3 Observe the pull request build + +`Observe engine pr-build` runs the same corpus against the standard Gradle test +cluster built from the pull request. It exports the same artifact shape as the +released legs. + +### 3.4 Aggregate rule compatibility + +`Aggregate rule compatibility` is the only fan-in job. It: + +1. waits for the released and pull request observation jobs; +2. downloads every `ppl-lint-leg-*` artifact; +3. bootstraps OSD once; +4. runs the production headless lint detector against each engine's runtime + grammar bundle; +5. compares declared compatibility with observed detector and backend results; +6. writes `drift-report.json`; +7. publishes the Markdown compatibility table in the GitHub step summary; +8. uploads the mandatory `ppl-lint-multiversion-drift` artifact and + supplemental `ppl-lint-multiversion-evidence` artifact; +9. fails if the aggregate result recorded supported-version drift. + +The job display name is intentionally explicit. A reader should not have to +infer that a job named "detect" is the final aggregation. + +## 4. Expected Versus Actual Compatibility + +The aggregate summary has one row per active rule: + +| Rule | Expected compatibility | 3.6 actual | 3.7 actual | PR build actual | +| --- | --- | --- | --- | --- | +| `agg-on-text` | Calcite, >= 3.7 | expected n/a | compatible | compatible | +| `division-by-zero` | all versions | compatible | compatible | compatible | + +Each actual cell uses one of these states: + +| State | Meaning | +| --- | --- | +| `compatible` | Detector output and backend behavior match the contract. | +| `expected n/a` | The engine is outside `wiring.appliesTo`, such as 3.6 for a rule with `minVersion: 3.7.0`. | +| `drift` | The engine is declared compatible but detector or backend behavior differs. | +| `inconclusive` | A fixture, query, or detector execution did not produce a trustworthy verdict. | + +`minVersion` is part of the expected result, not a workaround applied after +the fact. If a rule is intentionally unsupported on 3.6 and declares +`minVersion: 3.7.0`, the 3.6 cell is `expected n/a` and does not count as +drift. + +The JSON report retains query-level evidence and remediation details. The +Markdown table is the concise compatibility view, not a replacement for the +machine-readable report. + +## 5. Failure Semantics + +Compatibility aggregation is write-first and then enforcing: + +- observation jobs record detector and backend mismatches without failing; +- expected out-of-scope versions do not fail the workflow; +- an inconclusive rule produces an `inconclusive` cell and annotation; +- one rule cannot prevent results for the other rules; +- the fan-in writes the complete table and `drift-report.json`; +- the artifact upload runs even when the aggregate result is failing; +- only after those outputs exist does supported-version drift or an enforced + inconclusive result fail the final aggregation job. + +This ordering is required. A bare `Process completed with exit code 1` before +the table exists is not an actionable compatibility result. + +Structural failures remain errors because no truthful table can be produced: + +- a planned engine leg uploads no artifacts; +- JSON artifacts are malformed; +- target identity conflicts with report identity; +- the contract manifest is malformed; +- `drift-report.json` cannot be written. + +The artifact upload must require `drift-report.json`. An artifact named +`ppl-lint-multiversion-drift` that contains only raw target files is not a drift +report and must not be presented as one. + +## 6. Outputs + +Every run produces: + +- a GitHub step-summary table with expected and actual compatibility; +- `drift-report.json`; +- one detector report and detector log per engine leg; +- target manifests that identify the exact engine and grammar hash. + +The required PPL lint workflow remains the stable branch-protection signal. +The multi-version aggregation job is also red on declared-supported drift, with +the table and artifact serving as the evidence for adjusting `minVersion`, +narrowing a detector, or updating a backend oracle after review. + +## 7. Deferred Coverage + +Analytics-engine validation may return only after: + +- its feature build is immutable for the duration of a run; +- all required fixtures can be represented or explicitly scoped; +- route attestation is stable; +- a rule-specific analytics limitation cannot invalidate unrelated rules. + +Compiled-simplified coverage may return only if pre-3.6 support becomes a +shipping requirement. It must be a separate workflow because it tests OSD's +checked-in grammar rather than the SQL runtime grammar bundle. diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 63cc7797409..660a88c09af 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -1,16 +1,16 @@ # PPL lint rule validation A cross-repository GitHub Actions check that proves the OpenSearch Dashboards -(OSD) PPL lint detectors and runtime syntax validation still agree with the SQL -backend on the **same candidate runtime grammar** built by a SQL pull request. +(OSD) PPL lint detectors still agree with the SQL backend on the **same +candidate runtime grammar** built by a SQL pull request. PPL language behavior lives in SQL; PPL lint detectors live in OSD. A SQL change can silently invalidate an OSD rule (a parser refactor stops a detector matching, or a semantic change makes a flagged query valid) without touching OSD. Neither repository's own unit tests catch that. This check does. -- **Design:** `ppl-lint-ci-validation-design.md` -- **Analytics rollout:** [`docs/dev/ppl-lint-analytics-engine-ci-validation.md`](../../docs/dev/ppl-lint-analytics-engine-ci-validation.md) +- **Multi-version design:** [`docs/dev/ppl-lint-runtime-compatibility-ci-design.md`](../../docs/dev/ppl-lint-runtime-compatibility-ci-design.md) +- **Deferred analytics design:** [`docs/dev/ppl-lint-analytics-engine-ci-validation.md`](../../docs/dev/ppl-lint-analytics-engine-ci-validation.md) - **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) - **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) @@ -227,94 +227,36 @@ rule that is correct on `main` can be a false positive on 3.6 or a false negativ on 3.7, and the single-version check cannot see it. [`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) -validates every `defaultError` rule against several engine versions at once, and -reports **what to change in the linter** when one disagrees. +validates every active shipping detector against several engine versions at +once, and reports **what to change in the linter** when one disagrees. ``` -observe (matrix: released images + pr-build standard + pr-build analytics) +observe (matrix: released 3.6/3.7 images + standard pr-build) └── each leg exports the same 4 artifacts as the single-version check -detect (one OSD bootstrap, one detector pass per leg's grammar) +aggregate rule compatibility (one OSD bootstrap, one detector pass per runtime grammar) └── aggregate-versions.mjs → drift-report.json + remediation report ``` Released legs run the official `opensearchproject/opensearch:` image, which bundles the matching `opensearch-sql` plugin, so no old branch is built. The `pr-build` leg is the same Gradle test cluster the single-version check uses. The -`pr-build-analytics` leg installs the full Arrow, analytics, composite, Parquet, -Lucene-backend, and DataFusion-backend stack and fails unless fixture settings, -explain output, and a profiled canary attest the route. These legs -run the **same** contract oracle (`PplLintRuleValidationIT`) with +legs run the **same** contract oracle (`PplLintRuleValidationIT`) with `-Dppl.lint.observe.only=true`, which records real behavior instead of asserting against expectations — on an older engine a mismatch is the signal being collected, not a broken run. -**Engine floor: 3.6.0 — for the runtime-bundle surface.** +**Engine floor: 3.6.0.** `GET /_plugins/_ppl/_grammar` landed in #5162, which is an ancestor of 3.6 but not 3.5, so a 3.5 leg cannot export a grammar bundle for the detectors to lint against. -### The two grammar surfaces +This workflow intentionally excludes the compiled-simplified surface and +analytics engine. Those dimensions do not share the stable runtime-bundle +contract being compared here. -OSD ships lint on **two** surfaces, and a user gets whichever one their session -resolves to (`lintRuntimePPLQuery`): - -| Surface | When the product uses it | Engine floor | -| --- | --- | --- | -| `runtime-bundle` | the engine exported a grammar bundle and it has loaded | 3.6.0 | -| `compiled-simplified` | no bundle — no dataset selected, engine below 3.6, or bundle not yet loaded | none | - -The compiled surface is not a degraded copy of the runtime one: it runs detector -logic the runtime path does not (`field_validation`'s text-side pass keys off -`grammarSurface === 'compiled-simplified'`). It is also the surface with no engine -floor, so it is where old-engine coverage is possible at all. - -`PPL_LINT_SURFACE` selects which surface a detector run validates. It defaults to -`runtime-bundle`, so the required check is unchanged, and the compiled surface is -an **explicit opt-in** — never a silent fallback. A missing bundle on the runtime -surface stays a hard failure, because quietly linting OSD's own grammar instead of -the candidate would validate the wrong thing. - -**`runtimeOnly` rules do not run on the compiled surface.** `lint_runner` skips -them (the productions they walk are absent from the compiled grammar), so a -compiled leg reports them `not-applicable` rather than as zero diagnostics. This -distinction is load-bearing: counted as zero, a healthy rule would classify as -`detector-silent` drift and send someone to "fix" it. In the summary table those -cells read `n/a (surface)`, and a rule whose every case is inert is `n/a` — not -`agree` (it proved nothing) and not `inconclusive` (nothing went wrong, and there -is nothing to re-run). - -Two legs may share an engine version while validating different surfaces, so the -matrix is keyed on the **leg label**, grammar surface, and execution backend, not -the version alone. - -Each contract declares the surface(s) it was verified against, and a contract is -only scored on a matching leg — `"both"` opts into either. Judged on a surface it -never claimed, every verdict is meaningless: a runtime-bundle contract on a -compiled leg yields both `version-scope-too-narrow` ("the engine rejects but the -rule is scoped away") and a coverage hole, each about a surface the contract does -not describe. Contracts declaring `"both"` are what a pre-3.6 leg can actually -validate; the rest report `n/a (surface)`. - -**Compiled-surface legs run nightly** (`COMPILED_ENGINE_VERSIONS`, default -`2.19.0` / `3.0.0` / `3.5.0`) — three more engine images is too slow for every PR. -Dispatch with `compiled_versions` to run one ad hoc, or `[]` to skip. Their observe -job omits `-Dppl.lint.grammar.bundle` (the IT then exports nothing) and writes a -`surface` marker file, which is what tells the detect job to lint them on the -compiled surface rather than treating a missing bundle as a failed export. - -```bash -# A compiled-surface leg: no grammar bundle needed, so any engine version works. -PPL_LINT_SURFACE=compiled-simplified \ -PPL_LINT_CONTRACT_DIR= \ -PPL_LINT_TARGET_MANIFEST=/target.json \ -PPL_LINT_SCHEDULE=nightly \ -PPL_LINT_REPORT=/detector-report.json \ -node -r ./src/setup_node_env /scripts/ppl-lint/run-frontend-contract.mjs -``` - -This workflow is **non-enforcing for now**: it reports and uploads, while the -required check stays the single-version `validation-result`. Promoting it needs a -green baseline across the whole matrix first, so a rule that has already drifted -on 3.6 does not block every unrelated PR on day one. +Observation jobs do not fail on compatibility differences. The final +`Aggregate rule compatibility` job writes the complete expected-versus-actual +table and `drift-report.json`, uploads them, and then fails when a rule drifts on +a version declared by its `wiring.appliesTo` scope. ### What a drift report tells you @@ -422,9 +364,9 @@ remediation. Severity is not cosmetic: | Finding | Level | Why | | --- | --- | --- | -| enforced drift, coverage hole | `error` | a shipped default-error rule disagrees with a supported engine | +| enforced drift, coverage hole | `error` | an active shipping rule disagrees with a supported engine; aggregation writes the report and then fails | | non-enforced drift | `warning` | reported, but it does not block | -| `inconclusive` | `warning` | "we could not check" is a leg problem; the run is already red from the exit code, and rendering it as an error invites editing a rule because a leg timed out | +| `inconclusive` | `warning` | "we could not check" is a leg problem, not evidence that a rule is wrong | | unvalidated default-error rule | `error` (no file) | the edit goes in `manifest.json`, not a contract | A line number is emitted only when it is unambiguous. If a contract pins the same @@ -447,22 +389,20 @@ mkdir -p legs/3.7.0 -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ -Dppl.lint.target=$PWD/legs/3.7.0/target.json -# Observe the PR build through composite/Parquet + DataFusion. -mkdir -p legs/pr-build-analytics -./gradlew :integ-test:analyticsEnginePplLintIT \ - -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ - -Dppl.lint.report=$PWD/legs/pr-build-analytics/backend-report.json \ - -Dppl.lint.grammar.bundle=$PWD/legs/pr-build-analytics/ppl-grammar-bundle.json \ - -Dppl.lint.target=$PWD/legs/pr-build-analytics/target.json - # Lint each leg's grammar from an OSD checkout (writes detector-report.json), -# then compare every version at once: +# then compare every standard runtime-bundle version at once. The aggregator +# writes the table and JSON report before returning a failing drift status. node scripts/ppl-lint/aggregate-versions.mjs \ --contracts integ-test/src/test/resources/ppl-lint/contracts \ - --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 \ + --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg pr-build=legs/pr-build \ --out drift-report.json ``` +The step summary has one row per active detector. It prints the compatibility +declared by `wiring.appliesTo` next to the actual result for every engine leg. +For example, a rule with `minVersion: 3.7.0` renders `expected n/a` on 3.6 +instead of reporting drift. + The classifier is pure and has no cluster or OSD dependency, so its tests run anywhere: diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index df46ab6acd3..669b2d6bb54 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -25,6 +25,18 @@ import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const SCRIPT = path.join(HERE, '..', 'aggregate-versions.mjs'); +const REAL_CONTRACTS = path.resolve( + HERE, + '..', + '..', + '..', + 'integ-test', + 'src', + 'test', + 'resources', + 'ppl-lint', + 'contracts' +); /** Contract used by every case: a >=3.7 calcite-only rule with one trigger + one control. */ const SPEC = { @@ -225,11 +237,12 @@ function writeLeg({ return dir; } -/** Run the aggregator; returns { status, stdout, report }. */ +/** Run the aggregator; returns the process result plus its JSON and Markdown reports. */ function run({ contracts, legs, extraArgs = [] }) { const outDir = makeTmp('ppl-lint-out-'); const out = path.join(outDir, 'drift-report.json'); - const args = [SCRIPT, '--contracts', contracts, '--out', out]; + const summaryFile = path.join(outDir, 'summary.md'); + const args = [SCRIPT, '--contracts', contracts, '--out', out, '--summary', summaryFile]; const entries = Array.isArray(legs) ? legs : Object.entries(legs); for (const [version, dir] of entries) { args.push('--leg', `${version}=${dir}`); @@ -237,7 +250,14 @@ function run({ contracts, legs, extraArgs = [] }) { args.push(...extraArgs); const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); const report = fs.existsSync(out) ? JSON.parse(fs.readFileSync(out, 'utf8')) : undefined; - return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '', report }; + const summary = fs.existsSync(summaryFile) ? fs.readFileSync(summaryFile, 'utf8') : ''; + return { + status: result.status, + stdout: result.stdout || '', + stderr: result.stderr || '', + report, + summary, + }; } /** The all-agree case, reused as the base for each drift scenario. */ @@ -291,17 +311,49 @@ function writeSchema4Contracts({ includeAnalytics = true } = {}) { }); } -test('all versions agreeing exits 0 and reports no drift', () => { - const { status, report, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); +test('all versions agreeing exits 0 and reports expected versus actual compatibility', () => { + const { status, report, stdout, summary } = run({ + contracts: writeContracts(), + legs: healthyLegs(), + }); assert.equal(status, 0); assert.equal(report.result.passed, true); assert.equal(report.drifts.length, 0); assert.match(stdout, /agrees with all 2 engine version\(s\)/); + assert.match(summary, /\| Rule \| Expected compatibility \| `3\.7\.0` actual \| `3\.8\.0` actual \|/); + assert.match( + summary, + /\| `union-min-datasets` \| Calcite, >= 3\.7\.0 \| compatible \| compatible \|/ + ); // Every rule/version pair is accounted for in the matrix. assert.equal(report.matrix.length, 2); assert.ok(report.matrix.every((m) => m.status === 'agree')); }); +test('the compatibility table contains exactly the 12 active shipping detectors', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, summary } = run({ + contracts: REAL_CONTRACTS, + legs: [['pr-build', leg]], + extraArgs: ['--all-rules'], + }); + + assert.equal(status, 1, 'missing synthetic observations remain inconclusive'); + const ruleRows = summary + .split('\n') + .filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)); + assert.equal(ruleRows.length, 12); + assert.match(summary, /\| `rex-scan-cost` \| all versions \|/); + assert.doesNotMatch(summary, /command-suggestion/); +}); + test('same-version standard and analytics verdicts are classified as backend divergence', () => { const grammarHash = 'sha256:shared-runtime-grammar'; const standard = writeLeg({ @@ -360,8 +412,10 @@ test('same-version standard and analytics verdicts are classified as backend div 0, 'route differences must not be rendered as product-version drift' ); - assert.match(stdout, /`3\.8\.0`
    standard/); - assert.match(stdout, /`3\.8\.0`
    analytics/); + const tableHeader = stdout.split('\n').find((line) => line.startsWith('| Rule |')); + assert.match(tableHeader, /Expected compatibility/); + assert.match(tableHeader, /`pr-build` actual/); + assert.doesNotMatch(tableHeader, /analytics/); }); test('schema-v3 analytics has explicit backend-oracle coverage holes', () => { @@ -878,6 +932,30 @@ test('a version where only one engine relaxed is red, and names just that versio assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); }); +test('drift exits nonzero only after writing the JSON report and full Markdown table', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 0, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report, summary } = run({ contracts: writeContracts(), legs }); + + assert.equal(status, 1); + assert.equal(report.result.passed, false); + assert.equal(report.result.enforcedDriftCount, 1); + assert.match(summary, /## PPL lint multi-version validation/); + assert.match(summary, /\| Rule \| Expected compatibility \|/); + assert.match( + summary, + /\| `union-min-datasets` \| Calcite, >= 3\.7\.0 \| compatible \| \*\*drift\*\* \|/ + ); + assert.match(summary, /### Remediation/); +}); + test('a changed rejection HTTP status is semantic drift, not agreement', () => { const leg = writeLeg({ version: '3.8.0', @@ -1060,23 +1138,29 @@ test('a rule out of scope on an older engine that accepts is not drift', () => { version: '3.6.0', cases: { trigger: { detector: 0, rejected: false }, control: { detector: 0, rejected: false } }, }); - const { status, report } = run({ contracts: writeContracts(), legs }); + const { status, report, summary } = run({ contracts: writeContracts(), legs }); assert.equal(status, 0); assert.equal(report.matrix.find((m) => m.version === '3.6.0').status, 'out-of-scope'); assert.equal(report.coverageHoles.length, 0); + const row = summary + .split('\n') + .find((line) => line.startsWith('| `union-min-datasets` |')); + assert.match(row, /Calcite, >= 3\.7\.0/); + assert.match(row, /expected n\/a/); + assert.equal((row.match(/compatible/g) || []).length, 2); }); -test('an out-of-scope engine that rejects is flagged as scoped too narrowly', () => { +test('an engine below minVersion is expected n/a even when its query rejects', () => { const legs = healthyLegs(); legs['3.6.0'] = writeLeg({ version: '3.6.0', cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, }); - const { status, report } = run({ contracts: writeContracts(), legs }); - assert.equal(status, 1); - const drift = report.drifts.find((d) => d.version === '3.6.0'); - assert.equal(drift.driftClass, 'version-scope-too-narrow'); - assert.equal(drift.remediation.action, 'version-scope-rule'); + const { status, report, summary } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.drifts.filter((drift) => drift.version === '3.6.0').length, 0); + assert.equal(report.matrix.find((row) => row.version === '3.6.0').status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); }); test('an in-scope version with no expectation is a coverage hole, not silent success', () => { @@ -1406,18 +1490,19 @@ test('a reworded engine message does not mask a detector that went silent', () = assert.equal(drift.remediation.action, 'update-detector'); }); -test('a detector firing on a version its appliesTo excludes is reported', () => { - // OSD's version filter runs a rule when the cluster version is unknown, so an - // out-of-scope rule CAN reach users. Silence here would hide that false positive. +test('a detector observation below minVersion remains expected n/a', () => { const dir = writeLeg({ version: '3.6.0', // below the rule's 3.7 minVersion cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, }); - const { status, report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); - assert.equal(status, 1); - const drift = report.drifts.find((d) => d.version === '3.6.0'); - assert.equal(drift.driftClass, 'detector-noisy'); - assert.equal(drift.remediation.action, 'update-detector'); + const { status, report, summary } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + assert.equal(status, 0); + assert.equal(report.drifts.length, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); }); test('a calcite-scoped expectation is selected rather than counted twice', () => { @@ -1438,11 +1523,7 @@ test('a calcite-scoped expectation is selected rather than counted twice', () => ); }); -test('an errored trigger on an out-of-scope rule does not silently pass', () => { - // The out-of-scope path used to read `entry.rejected` directly. An errored - // observation has no such field, so it coerced to false, the - // version-scope-too-narrow check (which needs `=== true`) never fired, and a - // genuinely mis-scoped rule rendered as `out-of-scope` with exit 0. +test('an errored trigger below minVersion remains expected n/a', () => { const dir = writeLeg({ version: '3.6.0', // below the rule's 3.7 minVersion => out of scope cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, @@ -1473,21 +1554,13 @@ test('an errored trigger on an out-of-scope rule does not silently pass', () => contracts: writeContracts(), legs: { '3.6.0': dir }, }); - // The point is that an unobserved trigger yields no CLAIM either way: it must - // not be reported as a confident out-of-scope agreement... - assert.equal( - report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow').length, - 0, - 'an unobserved trigger cannot support a version-scope finding' - ); - // ...nor may it invent linter advice from a verdict that never arrived. + assert.equal(status, 0); assert.equal(report.drifts.length, 0); - assert.equal(status, 1); - assert.equal(report.matrix[0].status, 'inconclusive'); - assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.equal(report.result.enforcedInconclusive, 0); }); -test('a missing detector and backend row is inconclusive even when the rule is out of scope', () => { +test('missing rows below minVersion remain expected n/a', () => { const dir = writeLeg({ version: '3.6.0', cases: { @@ -1510,16 +1583,12 @@ test('a missing detector and backend row is inconclusive even when the rule is o legs: { '3.6.0': dir }, }); - assert.equal(status, 1); - assert.equal(report.matrix[0].status, 'inconclusive'); - assert.match(report.inconclusive[0].reasons.join(' '), /trigger \(no detector result\)/); + assert.equal(status, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.equal(report.inconclusive.length, 0); }); -test('an errored control cannot fail open into "widen appliesTo" advice', () => { - // controlAlsoRejected suppresses the version-scope finding when the command - // itself is unsupported. Reading `entry.rejected` raw made that suppression fail - // OPEN on an errored control: the run would then advise lowering minVersion, - // shipping a precise-cause diagnostic for an unknown-command failure. +test('an errored control below minVersion remains expected n/a', () => { const dir = writeLeg({ version: '3.6.0', cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: true } }, @@ -1538,19 +1607,15 @@ test('an errored control cannot fail open into "widen appliesTo" advice', () => : { ...e, outcome: 'observed' } ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); - const { status, report, stdout } = run({ + const { status, report, stdout, summary } = run({ contracts: writeContracts(), legs: { '3.6.0': dir }, }); - const scoped = report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow'); - assert.equal( - scoped.length, - 0, - 'with the control unobserved there is no evidence the command is supported, so no widening advice' - ); + assert.equal(report.drifts.length, 0); assert.ok(!/Widen "/.test(stdout)); - assert.equal(status, 1); - assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(status, 0); + assert.equal(report.matrix[0].status, 'out-of-scope'); + assert.match(summary, /expected n\/a/); }); test('a bad --leg argument is rejected', () => { diff --git a/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs b/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs deleted file mode 100644 index 430daf81797..00000000000 --- a/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { after, test } from 'node:test'; -import { fileURLToPath } from 'node:url'; - -import { - validatePrBuildArtifacts, - validatePrBuildTargetPair, -} from '../validate-pr-build-targets.mjs'; - -const HERE = path.dirname(fileURLToPath(import.meta.url)); -const SCRIPT = path.join(HERE, '..', 'validate-pr-build-targets.mjs'); -const tmpDirs = []; - -function standardTarget(overrides = {}) { - return { - schemaVersion: 2, - sqlSha: 'candidate-sql-sha', - engineVersion: '3.8.0-SNAPSHOT', - grammarHash: 'sha256:candidate-grammar', - grammarBundle: 'ppl-grammar-bundle.json', - executionBackend: 'standard', - storage: 'lucene', - shardCount: 1, - ...overrides, - }; -} - -function analyticsTarget(overrides = {}) { - return { - schemaVersion: 2, - sqlSha: 'candidate-sql-sha', - engineVersion: '3.8.0-SNAPSHOT', - grammarHash: 'sha256:candidate-grammar', - grammarBundle: 'ppl-grammar-bundle.json', - executionBackend: 'analytics', - storage: 'composite-parquet', - shardCount: 1, - analyticsStack: { source: 'https://example.test/analytics-build' }, - routeAttestation: { - pluginsVerified: true, - clusterSettingsVerified: true, - fixtureIndicesVerified: true, - explainVerified: true, - profiledExecutionVerified: true, - }, - ...overrides, - }; -} - -function backendReport(executionBackend, queryNames = ['trigger', 'control']) { - return queryNames.map((queryName) => ({ - ruleId: 'test-rule', - queryName, - role: queryName === 'control' ? 'control' : 'trigger', - query: - queryName === 'control' - ? 'source=test-index | head 1' - : 'source=test-index | head 0', - executionBackend, - rejected: queryName !== 'control', - observed: { - httpStatus: queryName === 'control' ? 200 : 400, - rejected: queryName !== 'control', - }, - })); -} - -function writeContractCorpus() { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-contracts-')); - tmpDirs.push(dir); - fs.writeFileSync( - path.join(dir, 'manifest.json'), - JSON.stringify({ schemaVersion: 3, contracts: ['test-rule.spec.json'] }) - ); - fs.writeFileSync( - path.join(dir, 'test-rule.spec.json'), - JSON.stringify({ - schemaVersion: 3, - ruleId: 'test-rule', - schedule: 'pr', - queries: { - trigger: { role: 'trigger', query: 'source={{index}} | head 0' }, - control: { role: 'control', query: 'source={{index}} | head 1' }, - }, - }) - ); - return dir; -} - -after(() => { - for (const dir of tmpDirs) { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -test('matching standard and analytics PR-build targets pass', () => { - const pair = validatePrBuildTargetPair(standardTarget(), analyticsTarget()); - assert.equal(pair.standard.executionBackend, 'standard'); - assert.equal(pair.analytics.executionBackend, 'analytics'); -}); - -test('the two PR-build target roles require their exact backend identities', () => { - assert.throws( - () => validatePrBuildTargetPair(analyticsTarget(), analyticsTarget()), - /standard PR-build target executionBackend must be "standard"/ - ); - assert.throws( - () => validatePrBuildTargetPair(standardTarget(), standardTarget()), - /analytics PR-build target executionBackend must be "analytics"/ - ); -}); - -test('both PR-build targets require the same non-empty SQL SHA', () => { - assert.throws( - () => validatePrBuildTargetPair(standardTarget({ sqlSha: '' }), analyticsTarget()), - /both report a non-empty SQL SHA/ - ); - assert.throws( - () => - validatePrBuildTargetPair( - standardTarget(), - analyticsTarget({ sqlSha: 'different-sql-sha' }) - ), - /report different values for SQL SHA/ - ); -}); - -test('both PR-build targets require the same engine version', () => { - assert.throws( - () => - validatePrBuildTargetPair( - standardTarget(), - analyticsTarget({ engineVersion: '3.9.0-SNAPSHOT' }) - ), - /report different values for engine version/ - ); -}); - -test('both PR-build targets require the same non-empty grammar hash', () => { - assert.throws( - () => validatePrBuildTargetPair(standardTarget(), analyticsTarget({ grammarHash: ' ' })), - /both report a non-empty grammar hash/ - ); - assert.throws( - () => - validatePrBuildTargetPair( - standardTarget(), - analyticsTarget({ grammarHash: 'sha256:different-grammar' }) - ), - /report different values for grammar hash/ - ); -}); - -test('target schema validation runs before pair identity comparison', () => { - assert.throws( - () => - validatePrBuildTargetPair( - standardTarget({ schemaVersion: 1 }), - analyticsTarget() - ), - /standard PR-build target is invalid: Unsupported target schemaVersion/ - ); -}); - -test('paired backend reports require exact, usable query coverage', () => { - const base = { - standardTarget: standardTarget(), - analyticsTarget: analyticsTarget(), - standardReport: backendReport('standard'), - analyticsReport: backendReport('analytics'), - contractsDir: writeContractCorpus(), - }; - const result = validatePrBuildArtifacts(base); - assert.equal(result.expectedRows, 2); - - assert.throws( - () => - validatePrBuildArtifacts({ - ...base, - analyticsReport: backendReport('analytics', ['trigger']), - }), - /analytics PR-build backend report query coverage is incomplete.*test-rule::control/ - ); - assert.throws( - () => - validatePrBuildArtifacts({ - ...base, - standardReport: [ - ...backendReport('standard'), - { ...backendReport('standard')[0] }, - ], - }), - /duplicate backend report key/ - ); - - const errored = backendReport('analytics'); - errored[0] = { ...errored[0], outcome: 'error' }; - assert.throws( - () => validatePrBuildArtifacts({ ...base, analyticsReport: errored }), - /contains rows without an engine verdict: test-rule::trigger/ - ); - - const unobservedCoverageGap = backendReport('analytics'); - unobservedCoverageGap[0] = { - ...unobservedCoverageGap[0], - outcome: 'coverage-missing', - }; - delete unobservedCoverageGap[0].rejected; - assert.throws( - () => - validatePrBuildArtifacts({ - ...base, - analyticsReport: unobservedCoverageGap, - }), - /contains rows without an engine verdict: test-rule::trigger/ - ); - - const changedQuery = backendReport('analytics'); - changedQuery[0] = { ...changedQuery[0], query: 'source=different-index | head 0' }; - assert.throws( - () => validatePrBuildArtifacts({ ...base, analyticsReport: changedQuery }), - /executed different query text for test-rule::trigger/ - ); -}); - -test('the CLI reads and validates both PR-build artifact sets', () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-pair-')); - tmpDirs.push(dir); - const standardFile = path.join(dir, 'standard.json'); - const analyticsFile = path.join(dir, 'analytics.json'); - const standardReportFile = path.join(dir, 'standard-report.json'); - const analyticsReportFile = path.join(dir, 'analytics-report.json'); - fs.writeFileSync(standardFile, JSON.stringify(standardTarget())); - fs.writeFileSync(analyticsFile, JSON.stringify(analyticsTarget())); - fs.writeFileSync(standardReportFile, JSON.stringify(backendReport('standard'))); - fs.writeFileSync(analyticsReportFile, JSON.stringify(backendReport('analytics'))); - - const result = spawnSync( - process.execPath, - [ - SCRIPT, - '--standard', - standardFile, - '--analytics', - analyticsFile, - '--standard-report', - standardReportFile, - '--analytics-report', - analyticsReportFile, - '--contracts', - writeContractCorpus(), - '--schedule', - 'nightly', - ], - { encoding: 'utf8' } - ); - - assert.equal(result.status, 0, result.stderr); - assert.match(result.stdout, /verified engine=3\.8\.0-SNAPSHOT/); - assert.match(result.stdout, /backends=standard,analytics/); - assert.match(result.stdout, /backendRows=2/); -}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 7ceb86ae6df..6f46f088a5e 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -959,75 +959,6 @@ function failedExtendedFrontendAssertions(entry, frontendOracle) { return [...failures].sort(); } -/** - * Check an out-of-scope rule for the one drift that still matters there: the - * engine rejects a trigger query, but the rule's `appliesTo` excludes this - * version, so users on it see no diagnostic for a real error. Everything else - * about an out-of-scope rule is intentional silence. - * - * The trigger queries come from the spec's own `queries` map (there is no - * expectation to read on this path), and the backend observation from this leg's - * report; `classifyDrift` decides, so the "too narrow" wording stays in one place. - */ -function classifyOutOfScope({ spec, ruleId, leg, classify, divergentCases }) { - const found = []; - const unusable = []; - const observations = new Map(); - - for (const [queryName] of Object.entries(spec.queries || {})) { - const rowKey = `${ruleId}::${queryName}`; - const backendEntry = leg.backend.get(rowKey); - const detectorResult = leg.detector.resultsByKey.get(rowKey); - const { observed, usable } = readBackendObservation(backendEntry, detectorResult); - if (!usable) { - unusable.push(`${queryName} (${unusableObservationReason(detectorResult)})`); - continue; - } - observations.set(queryName, observed); - } - - // What did this rule's CONTROL queries — valid uses of the same command — do on - // this engine? THREE states, not two, and the difference decides whether a - // rejected trigger means anything. - const controlVerdicts = Object.entries(spec.queries || {}) - .filter(([, def]) => (def.role || 'trigger') === 'control') - .map(([name]) => observations.get(name)?.backendRejected); - const controlAlsoRejected = controlVerdicts.some((v) => v === true); - // A rule with controls, none of which produced a verdict, cannot be judged here. - const controlUnknown = - controlVerdicts.length > 0 && !controlVerdicts.some((v) => typeof v === 'boolean'); - - for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { - if ((queryDef.role || 'trigger') !== 'trigger') continue; - const rowKey = `${ruleId}::${queryName}`; - const outOfScopeObserved = observations.get(queryName); - if (!outOfScopeObserved) continue; - const pairedDivergence = divergentCases.has(`${leg.key}::${rowKey}`); - if (pairedDivergence && leg.executionBackend === 'analytics') continue; - const drift = classify({ - ruleId, - version: leg.version, - queryName, - role: 'trigger', - query: queryDef.query.split('{{index}}').join(spec.index), - // Out of scope means the rule is expected to stay silent here. - expected: { detectorCount: 0 }, - observed: outOfScopeObserved, - wiring: spec.wiring, - detectorPath: spec.detectorPath, - executionBackend: leg.executionBackend, - // An unknown control verdict is treated the same as a rejected one: both - // mean "we cannot claim this engine supports the command", and staying quiet - // is the only honest option. - controlAlsoRejected: controlAlsoRejected || controlUnknown, - // Deliberately no parser-rule check here: a grammar that lacks the rule is - // expected on an engine the command predates. - }); - if (drift) found.push(drift); - } - return { drifts: found, unusable }; -} - /** * Pick the contract expectation that applies to a version, reusing the same * "exactly one must match" rule as the two single-version halves. Returns @@ -1186,6 +1117,22 @@ function main() { } const inScope = versionInAppliesTo(appliesTo, leg.version); + if (!inScope) { + notApplicable.push({ + ruleId, + ...legFields(leg), + surface: legSurface, + kind: 'applies-to', + reason: `wiring.appliesTo excludes engine ${leg.version}`, + }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'out-of-scope', + drifts: 0, + }); + continue; + } // A parser rule that vanished from the grammar is one fact about this // rule on this engine, not one per query — raise it once and move on, so @@ -1209,42 +1156,6 @@ function main() { const expectation = selectExpectation(spec, leg.version, versionMatchesRange); if (!expectation) { - if (!inScope) { - // Deliberately out of scope on this engine. Still run the classifier - // for the one case that matters — an engine that rejects a trigger the - // rule has been scoped away from (a missed diagnostic). - const outOfScope = classifyOutOfScope({ - spec, - ruleId, - leg, - classify: classifyDrift, - divergentCases, - }); - for (const drift of outOfScope.drifts) { - addDrift(drift, leg, { enforced: isEnforced, contractFile: file }); - } - if (outOfScope.unusable.length > 0) { - inconclusive.push({ - ruleId, - file, - ...legFields(leg), - enforced: isEnforced, - reasons: outOfScope.unusable, - }); - } - matrix.push({ - ruleId, - ...legFields(leg), - status: - outOfScope.unusable.length > 0 - ? 'inconclusive' - : outOfScope.drifts.length > 0 - ? 'drift' - : 'out-of-scope', - drifts: outOfScope.drifts.length, - }); - continue; - } // In scope on this engine but nothing pins its behavior there. coverageHoles.push({ ruleId, @@ -1724,10 +1635,8 @@ function main() { } const enforcedDrifts = drifts.filter((d) => d.blocking); const enforcedHoles = coverageHoles.filter((h) => h.blocking); - // `--all-rules` widens observation to the whole corpus. Semantic drift remains - // enforced only for default-error rules, but a missing detector row or backend - // verdict is an infrastructure failure for every rule we asked the run to - // observe. + // `--all-rules` makes drift, missing coverage, and inconclusive observations + // blocking for every active shipping rule in the manifest. const enforcedInconclusive = inconclusive.filter((i) => i.enforced || args.allRules); const blockingMissingContracts = missingContracts.filter((entry) => entry.blocking); for (const row of matrix) { @@ -1807,7 +1716,7 @@ function main() { workspace: process.env.GITHUB_WORKSPACE, }); - const markdown = renderMarkdown(report, drifts, coverageHoles, legs); + const markdown = renderMarkdown(report, drifts, coverageHoles, legs, specs); // eslint-disable-next-line no-console console.log(markdown); if (args.summary) { @@ -1836,8 +1745,42 @@ function main() { ); } -/** Rule × version agreement matrix followed by the grouped remediation report. */ -function renderMarkdown(report, drifts, coverageHoles, legs) { +function expectedCompatibility(spec) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const scope = []; + if (appliesTo.engine) { + scope.push( + appliesTo.engine === 'calcite' + ? 'Calcite' + : appliesTo.engine.charAt(0).toUpperCase() + appliesTo.engine.slice(1) + ); + } + if (appliesTo.minVersion && appliesTo.maxVersion) { + scope.push(`>= ${appliesTo.minVersion}, <= ${appliesTo.maxVersion}`); + } else if (appliesTo.minVersion) { + scope.push(`>= ${appliesTo.minVersion}`); + } else if (appliesTo.maxVersion) { + scope.push(`<= ${appliesTo.maxVersion}`); + } else { + scope.push('all versions'); + } + return scope.join(', '); +} + +function actualCompatibility(row) { + if (!row) return 'not evaluated'; + if (row.status === 'agree') return 'compatible'; + if (row.status === 'out-of-scope') return 'expected n/a'; + if (row.status === 'drift') return row.drifts > 1 ? `**drift** (${row.drifts})` : '**drift**'; + if (row.status === 'uncovered' || row.status === 'inconclusive') { + return '**inconclusive**'; + } + if (row.status === 'not-applicable') return 'expected n/a'; + return `**${row.status}**`; +} + +/** Expected-vs-actual compatibility table followed by the detailed remediation report. */ +function renderMarkdown(report, drifts, coverageHoles, legs, specs) { const lines = []; lines.push('## PPL lint multi-version validation'); lines.push(''); @@ -1863,56 +1806,46 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { `${report.result.blockingShippingCensusProblems} shipping census problem(s)` ); } + const compatibilityLegs = legs.filter( + (leg) => + leg.executionBackend === 'standard' && + (leg.surface || 'runtime-bundle') === 'runtime-bundle' + ); lines.push( - // Name the surface when a leg is not the default runtime-bundle one, so a - // reader knows a column speaks for OSD's compiled grammar rather than the - // engine's exported one — the two do not run the same set of rules. - `Engine versions: ${legs - .map((l) => { - const identity = - l.label === l.version ? `\`${l.version}\`` : `\`${l.label}\` → \`${l.version}\``; - return l.surface && l.surface !== 'runtime-bundle' - ? `${identity} (${l.executionBackend}, ${l.surface})` - : `${identity} (${l.executionBackend})`; - }) - .join(', ')} — ` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` + `Standard runtime-bundle engines: ${ + compatibilityLegs + .map((leg) => + leg.label === leg.version + ? `\`${leg.version}\`` + : `\`${leg.label}\` → \`${leg.version}\`` + ) + .join(', ') || 'none' + } — **${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); - // Columns use the full leg key, including execution backend and grammar surface. - // A label or engine version alone is not unique once the same candidate runs - // through both standard and analytics. - const columns = legs.map((l) => ({ - key: l.key, + const columns = compatibilityLegs.map((leg) => ({ + key: leg.key, heading: - l.surface && l.surface !== 'runtime-bundle' - ? `\`${l.version}\`
    ${l.executionBackend}
    ${l.surface}` + - (l.label === l.version ? '' : `
    ${l.label}`) - : `\`${l.version}\`
    ${l.executionBackend}` + - (l.label === l.version ? '' : `
    ${l.label}`), + leg.label === leg.version + ? `\`${leg.version}\` actual` + : `\`${leg.label}\` actual
    (\`${leg.version}\`)`, })); - const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); - lines.push(`| Rule | ${columns.map((c) => c.heading).join(' | ')} |`); - lines.push(`| ---- | ${columns.map(() => '----').join(' | ')} |`); - const cell = { - agree: 'agree', - drift: 'DRIFT', - uncovered: 'not covered', - 'out-of-scope': 'n/a (out of scope)', - 'not-applicable': 'n/a (surface)', - inconclusive: '**inconclusive**', - }; - for (const ruleId of rules) { + const rules = [...specs.entries()] + .filter(([, entry]) => contractChannel(entry.spec) === 'lint') + .sort(([left], [right]) => left.localeCompare(right)); + lines.push( + `| Rule | Expected compatibility | ${columns.map((column) => column.heading).join(' | ')} |` + ); + lines.push(`| ---- | ---- | ${columns.map(() => '----').join(' | ')} |`); + for (const [ruleId, { spec }] of rules) { const cells = columns.map((column) => { const row = report.matrix.find((m) => m.ruleId === ruleId && m.legKey === column.key); - if (!row) return '—'; - if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; - // An unmapped status must still render as something visible. A blank cell - // reads as "nothing to see here", which is the opposite of what an - // unrecognized state means. - return cell[row.status] || `**${row.status}**`; + return actualCompatibility(row); }); - lines.push(`| \`${ruleId}\` | ${cells.join(' | ')} |`); + lines.push( + `| \`${ruleId}\` | ${expectedCompatibility(spec)} | ${cells.join(' | ')} |` + ); } lines.push(''); diff --git a/scripts/ppl-lint/validate-pr-build-targets.mjs b/scripts/ppl-lint/validate-pr-build-targets.mjs deleted file mode 100644 index b7944ed76d7..00000000000 --- a/scripts/ppl-lint/validate-pr-build-targets.mjs +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import fs from 'node:fs'; -import path from 'node:path'; - -import { - assertContractSchema, - classifyBackendReportRow, - indexBackendReport, - normalizeTarget, -} from './contract-schema.mjs'; - -function normalizeLabeledTarget(target, label) { - try { - return normalizeTarget(target); - } catch (error) { - throw new Error(`${label} target is invalid: ${error.message}`); - } -} - -function requireMatchingNonEmptyField(standard, analytics, field, label) { - if ( - typeof standard[field] !== 'string' || - standard[field].trim().length === 0 || - typeof analytics[field] !== 'string' || - analytics[field].trim().length === 0 - ) { - throw new Error( - `standard and analytics targets must both report a non-empty ${label}: ` + - `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` - ); - } - if (standard[field] !== analytics[field]) { - throw new Error( - `standard and analytics targets report different values for ${label}: ` + - `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` - ); - } -} - -export function validatePrBuildTargetPair(standardRaw, analyticsRaw) { - const standard = normalizeLabeledTarget(standardRaw, 'standard PR-build'); - const analytics = normalizeLabeledTarget(analyticsRaw, 'analytics PR-build'); - - if (standard.executionBackend !== 'standard') { - throw new Error( - `standard PR-build target executionBackend must be "standard", got ` + - `${JSON.stringify(standard.executionBackend)}` - ); - } - if (analytics.executionBackend !== 'analytics') { - throw new Error( - `analytics PR-build target executionBackend must be "analytics", got ` + - `${JSON.stringify(analytics.executionBackend)}` - ); - } - - requireMatchingNonEmptyField(standard, analytics, 'engineVersion', 'engine version'); - requireMatchingNonEmptyField(standard, analytics, 'sqlSha', 'SQL SHA'); - requireMatchingNonEmptyField(standard, analytics, 'grammarHash', 'grammar hash'); - - return { standard, analytics }; -} - -function expectedBackendReportKeys(contractsDir, schedule) { - if (schedule !== 'pr' && schedule !== 'nightly') { - throw new Error(`schedule must be "pr" or "nightly", got ${JSON.stringify(schedule)}`); - } - const manifest = readJson(path.join(contractsDir, 'manifest.json'), 'contract manifest'); - if ( - manifest === null || - typeof manifest !== 'object' || - Array.isArray(manifest) || - !Array.isArray(manifest.contracts) - ) { - throw new TypeError('contract manifest.contracts must be a JSON array'); - } - - const files = new Set(); - const ruleIds = new Set(); - const expectedRows = new Map(); - for (const file of manifest.contracts) { - if (typeof file !== 'string' || file.length === 0) { - throw new TypeError('contract manifest entries must be non-empty strings'); - } - if (files.has(file)) { - throw new Error(`contract manifest contains duplicate file ${JSON.stringify(file)}`); - } - files.add(file); - - const spec = readJson(path.join(contractsDir, file), `contract ${file}`); - assertContractSchema(spec); - if (ruleIds.has(spec.ruleId)) { - throw new Error(`contract manifest contains duplicate ruleId ${JSON.stringify(spec.ruleId)}`); - } - ruleIds.add(spec.ruleId); - if (schedule === 'pr' && (spec.schedule || 'pr') !== 'pr') { - continue; - } - if ( - spec.queries === null || - typeof spec.queries !== 'object' || - Array.isArray(spec.queries) || - Object.keys(spec.queries).length === 0 - ) { - throw new TypeError(`[${spec.ruleId}] contract.queries must be a non-empty JSON object`); - } - for (const queryName of Object.keys(spec.queries)) { - const key = `${spec.ruleId}::${queryName}`; - if (expectedRows.has(key)) { - throw new Error(`contract corpus contains duplicate query key ${JSON.stringify(key)}`); - } - const query = spec.queries[queryName]; - if (query === null || typeof query !== 'object' || Array.isArray(query)) { - throw new TypeError(`[${spec.ruleId}] query ${JSON.stringify(queryName)} must be an object`); - } - expectedRows.set(key, { role: query.role || 'trigger' }); - } - } - if (expectedRows.size === 0) { - throw new Error(`contract corpus selected no queries for schedule ${JSON.stringify(schedule)}`); - } - return expectedRows; -} - -function validateBackendReport(raw, target, label, expectedRows) { - let rows; - try { - rows = indexBackendReport(raw, target); - } catch (error) { - throw new Error(`${label} backend report is invalid: ${error.message}`); - } - - const missing = [...expectedRows.keys()].filter((key) => !rows.has(key)).sort(); - const extra = [...rows.keys()].filter((key) => !expectedRows.has(key)).sort(); - if (missing.length > 0 || extra.length > 0) { - const details = []; - if (missing.length > 0) details.push(`missing: ${missing.join(', ')}`); - if (extra.length > 0) details.push(`unexpected: ${extra.join(', ')}`); - throw new Error(`${label} backend report query coverage is incomplete (${details.join('; ')})`); - } - - const unusable = []; - for (const [key, row] of rows) { - const expected = expectedRows.get(key); - if (row.role !== expected.role) { - throw new Error( - `${label} backend report row ${key}.role must be ${JSON.stringify(expected.role)}, ` + - `got ${JSON.stringify(row.role)}` - ); - } - if (typeof row.query !== 'string' || row.query.length === 0) { - throw new Error(`${label} backend report row ${key}.query must be a non-empty string`); - } - const status = classifyBackendReportRow(row).status; - if ( - status === 'error' || - (status === 'coverage-missing' && typeof row.rejected !== 'boolean') - ) { - unusable.push(key); - } - } - if (unusable.length > 0) { - throw new Error( - `${label} backend report contains rows without an engine verdict: ${unusable.sort().join(', ')}` - ); - } - return rows; -} - -export function validatePrBuildArtifacts({ - standardTarget, - analyticsTarget, - standardReport, - analyticsReport, - contractsDir, - schedule = 'nightly', -}) { - const pair = validatePrBuildTargetPair(standardTarget, analyticsTarget); - const expectedRows = expectedBackendReportKeys(contractsDir, schedule); - const standardRows = validateBackendReport( - standardReport, - pair.standard, - 'standard PR-build', - expectedRows - ); - const analyticsRows = validateBackendReport( - analyticsReport, - pair.analytics, - 'analytics PR-build', - expectedRows - ); - for (const key of expectedRows.keys()) { - const standard = standardRows.get(key); - const analytics = analyticsRows.get(key); - if (standard.query !== analytics.query) { - throw new Error( - `standard and analytics backend reports executed different query text for ${key}: ` + - `standard=${JSON.stringify(standard.query)}, analytics=${JSON.stringify(analytics.query)}` - ); - } - } - return { ...pair, expectedRows: expectedRows.size, standardRows, analyticsRows }; -} - -function parseArgs(argv) { - const options = new Map([ - ['--standard', 'standard'], - ['--analytics', 'analytics'], - ['--standard-report', 'standardReport'], - ['--analytics-report', 'analyticsReport'], - ['--contracts', 'contracts'], - ['--schedule', 'schedule'], - ]); - const args = { schedule: 'nightly' }; - const seen = new Set(); - for (let i = 0; i < argv.length; i++) { - const arg = argv[i]; - const key = options.get(arg); - if (!key) { - throw new Error(`unknown argument ${JSON.stringify(arg)}`); - } - const value = argv[++i]; - if (!value) { - throw new Error(`${arg} requires a value`); - } - if (seen.has(key)) { - throw new Error(`${arg} may be specified only once`); - } - seen.add(key); - args[key] = value; - } - for (const key of [ - 'standard', - 'analytics', - 'standardReport', - 'analyticsReport', - 'contracts', - ]) { - if (!args[key]) { - throw new Error(`${[...options].find(([, value]) => value === key)[0]} is required`); - } - } - return args; -} - -function readJson(file, label) { - try { - return JSON.parse(fs.readFileSync(file, 'utf8')); - } catch (error) { - throw new Error(`could not read ${label} ${file}: ${error.message}`); - } -} - -function main() { - const args = parseArgs(process.argv.slice(2)); - const { standard, analytics, expectedRows } = validatePrBuildArtifacts({ - standardTarget: readJson(args.standard, 'standard PR-build target'), - analyticsTarget: readJson(args.analytics, 'analytics PR-build target'), - standardReport: readJson(args.standardReport, 'standard PR-build backend report'), - analyticsReport: readJson(args.analyticsReport, 'analytics PR-build backend report'), - contractsDir: args.contracts, - schedule: args.schedule, - }); - // eslint-disable-next-line no-console - console.log( - `[ppl-lint-target-pair] verified engine=${standard.engineVersion} ` + - `sqlSha=${standard.sqlSha} grammarHash=${standard.grammarHash} ` + - `backends=${standard.executionBackend},${analytics.executionBackend} ` + - `backendRows=${expectedRows}` - ); -} - -if (process.argv[1] && process.argv[1].endsWith('validate-pr-build-targets.mjs')) { - try { - main(); - } catch (error) { - // eslint-disable-next-line no-console - console.error(`[ppl-lint-target-pair] FATAL: ${error.message}`); - process.exitCode = 2; - } -} From f59c89d9594620cf4b744eb46389ac5cd1d300d0 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 15:33:00 -0700 Subject: [PATCH 75/78] fix(ci): verify compatibility matrix leg identities Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-multiversion-validation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index ef169a78bb5..af2aaac8264 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -520,6 +520,8 @@ jobs: (.matrix | length) == (12 * $expected_legs) and (.matrix | map(.ruleId) | unique | sort) == $expected_rules and (.matrix | map(.legKey) | unique | length) == $expected_legs and + (.matrix | map(.legKey) | unique | sort) == + (.legs | map(.key) | unique | sort) and ([.matrix[] | [.ruleId, .legKey]] | unique | length) == (12 * $expected_legs) ' drift-report.json > /dev/null; then From f697b39305df19cfb32f271f9f81bad6f54d739c Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 16:35:07 -0700 Subject: [PATCH 76/78] feat(ci): validate PPL lint across grammar surfaces Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 820 +++++---------- scripts/ppl-lint/README.md | 208 ++-- .../aggregate-compatibility.test.mjs | 500 +++++++++ .../__tests__/plan-compatibility.test.mjs | 100 ++ .../__tests__/run-frontend-contract.test.mjs | 49 + scripts/ppl-lint/aggregate-compatibility.mjs | 961 ++++++++++++++++++ scripts/ppl-lint/plan-compatibility.mjs | 187 ++++ scripts/ppl-lint/run-frontend-contract.mjs | 75 +- 8 files changed, 2230 insertions(+), 670 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs create mode 100644 scripts/ppl-lint/__tests__/plan-compatibility.test.mjs create mode 100644 scripts/ppl-lint/aggregate-compatibility.mjs create mode 100644 scripts/ppl-lint/plan-compatibility.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index af2aaac8264..f68f53993d0 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -1,69 +1,27 @@ -name: PPL lint multi-version validation - -# Multi-version companion to ppl-lint-rule-validation.yml. -# -# The sibling workflow answers "do the OSD PPL lint detectors and THIS engine -# build agree?". It validates one engine: the one built from the PR. That leaves -# the failure mode that actually reaches users unguarded — a lint rule ships to -# everyone, but each user runs it against whatever engine version their cluster -# happens to be. A rule that is correct on main can be a false positive on 3.6 or -# a false negative on 3.7, and nothing notices. -# -# This workflow validates all 12 active detector contracts against released -# engine versions plus the PR build. The aggregate still records the exact -# default-error census separately, but --all-rules makes warning/info omissions -# visible too. -# -# Shape — a per-version matrix of observation legs, then one aggregation: -# -# observe (matrix: 3.6.0, 3.7.0, pr-build) ──▶ aggregate ──▶ drift report -# -# Each leg produces the SAME four artifacts the single-version workflow already -# defines (ppl-grammar-bundle.json, target.json, backend-report.json, -# detector-report.json), so this workflow adds no new producer format — only the -# per-version fan-out and the cross-version comparison. -# -# Released legs run the official distribution image, which bundles the matching -# opensearch-sql plugin (verified against opensearch-build's release manifests), -# so no old branch has to be built. The `pr-build` leg is the same Gradle test -# cluster the sibling workflow uses. -# -# Engine floor: 3.6.0. GET /_plugins/_ppl/_grammar landed in #5162 (`fe95703b5`), -# which is an ancestor of the 3.6 release branch but NOT of 3.5 — a 3.5 leg could -# not export a candidate grammar bundle, so the detector half would have nothing -# to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. -# -# Compatibility differences are collected without interrupting the matrix. -# The aggregate writes the complete table and JSON artifact first, then the -# final step fails this job when a declared-supported rule has drifted. +name: PPL lint multi-surface compatibility + +# Validate the 12 active PPL lint rules on the checked-in OSD fallback grammar +# and on runtime grammar bundles exported by the latest eligible GA engine and +# this SQL pull request. Compatibility differences are data until the aggregate +# job has published the complete 12 x 3 matrix. on: - # Nightly is the primary schedule: the matrix pulls three engine images, so it - # is too slow to sit on every push. - schedule: - - cron: '30 10 * * *' - # Run on PRs that touch the contract corpus or this machinery, where the whole - # point is to see the multi-version effect of the change. pull_request: paths: + - 'build.gradle' - 'integ-test/build.gradle' - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' - 'integ-test/src/test/resources/ppl-lint/**' - - 'scripts/ppl-lint-rule-validation.sh' - 'scripts/ppl-lint/**' - '.github/workflows/ppl-lint-multiversion-validation.yml' workflow_dispatch: inputs: osd_repo: - description: OSD repository to check out. Defaults to opensearch-project/OpenSearch-Dashboards. + description: OSD repository containing the detector implementation. required: false type: string osd_ref: - description: OSD commit or branch whose detectors are validated. - required: false - type: string - engine_versions: - description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' + description: OSD branch or commit containing the detector implementation. required: false type: string @@ -71,102 +29,90 @@ permissions: contents: read concurrency: - group: ppl-lint-multiversion-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ppl-lint-multi-surface-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must - # be >= 3.6.0 (the _grammar endpoint floor) and must have a published - # distribution image. - # Latest patch of each line (3.6.0 and 3.7.0 ARE the latest patches today; bump - # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch - # exists — that would validate an engine no user runs). - ENGINE_VERSIONS: '["3.6.0","3.7.0"]' - jobs: - # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a - # dependabot bump moves one set of action versions rather than two. Get-CI-Image-Tag: uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch - # Resolve the matrix and the OSD target once, so every leg and the aggregate - # step agree on exactly what is being validated. plan: - name: Plan matrix + name: Plan compatibility matrix runs-on: ubuntu-latest outputs: - released: ${{ steps.plan.outputs.released }} - discovery_engine: ${{ steps.plan.outputs.discovery_engine }} - osd_repo: ${{ steps.plan.outputs.osd_repo }} - osd_ref: ${{ steps.plan.outputs.osd_ref }} + released_targets: ${{ steps.outputs.outputs.released_targets }} + osd_repo: ${{ steps.outputs.outputs.osd_repo }} + osd_ref: ${{ steps.outputs.outputs.osd_ref }} + pr_target: ${{ steps.outputs.outputs.pr_target }} steps: - - name: Resolve engine versions and OSD target - id: plan + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Read official OpenSearch GA tags + run: | + set -euo pipefail + git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > "$RUNNER_TEMP/opensearch-release-tags.txt" + + - name: Resolve target versions and configurations env: - REQUESTED_VERSIONS: ${{ inputs.engine_versions }} - DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} - REQUESTED_REPO: ${{ inputs.osd_repo }} - REQUESTED_REF: ${{ inputs.osd_ref }} - VAR_REPO: ${{ vars.OSD_REPO }} - VAR_REF: ${{ vars.OSD_REF }} + REQUESTED_OSD_REPO: ${{ inputs.osd_repo }} + REQUESTED_OSD_REF: ${{ inputs.osd_ref }} + VARIABLE_OSD_REPO: ${{ vars.OSD_REPO }} + VARIABLE_OSD_REF: ${{ vars.OSD_REF }} run: | set -euo pipefail - released="${REQUESTED_VERSIONS:-$DEFAULT_VERSIONS}" - # Fail loudly on a malformed override rather than silently validating - # an empty matrix (which would look like a pass). - echo "$released" | python3 -c " - import json,sys - v=json.load(sys.stdin) - assert isinstance(v,list) and v, 'engine_versions must be a non-empty JSON array' - for item in v: - assert isinstance(item,str), 'engine_versions entries must be strings' - " - echo "released=$released" >> "$GITHUB_OUTPUT" - - # Discovery runs against ONE engine — the newest released version in the - # matrix. It is a lead-generator, not a version-drift check, so paying for - # a full matrix would multiply cost without adding signal: a false positive - # found on the newest engine is the one users hit soonest, and per-version - # differences are already the enforced corpus's job. - discovery_engine=$(echo "$released" | python3 -c " - import json,sys - v=json.load(sys.stdin) - # Newest by semver, not list order, so a reordered matrix cannot silently - # point discovery at an old engine. - def key(s): - parts=[int(p) for p in s.split('-')[0].split('.') if p.isdigit()] - return parts + [0]*(3-len(parts)) - print(sorted(v,key=key)[-1]) - ") - echo "discovery_engine=$discovery_engine" >> "$GITHUB_OUTPUT" - echo "Discovery engine: \`$discovery_engine\`" >> "$GITHUB_STEP_SUMMARY" - # Same precedence as the sibling workflow: dispatch input, then repo - # variable, then the canonical upstream default. - echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" - echo "osd_ref=${REQUESTED_REF:-${VAR_REF:-main}}" >> "$GITHUB_OUTPUT" - - # One leg per released engine version: run the contract queries against the - # official distribution image (which bundles the matching sql plugin) and - # export that engine's grammar bundle. + osd_repo="${REQUESTED_OSD_REPO:-${VARIABLE_OSD_REPO:-opensearch-project/OpenSearch-Dashboards}}" + osd_ref="${REQUESTED_OSD_REF:-${VARIABLE_OSD_REF:-main}}" + node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags "$RUNNER_TEMP/opensearch-release-tags.txt" \ + --compiled-version 2.19.6 \ + --sql-sha "$GITHUB_SHA" \ + --osd-repository "$osd_repo" \ + --osd-ref "$osd_ref" \ + --out compatibility-plan.json + + - name: Publish plan outputs + id: outputs + run: | + set -euo pipefail + { + echo "released_targets=$(jq -c '.releasedTargets' compatibility-plan.json)" + echo "osd_repo=$(jq -r '.osd.repository' compatibility-plan.json)" + echo "osd_ref=$(jq -r '.osd.ref' compatibility-plan.json)" + echo "pr_target=$(jq -r '.prTargetVersion' compatibility-plan.json)" + } >> "$GITHUB_OUTPUT" + { + echo '## PPL lint compatibility plan' + echo + echo "- PR target: \`$(jq -r '.prTargetVersion' compatibility-plan.json)\`" + echo "- Latest eligible GA: \`$(jq -r '.latestEligibleGa' compatibility-plan.json)\`" + echo "- OSD: \`$(jq -r '.osd.repository + " @ " + .osd.ref' compatibility-plan.json)\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload compatibility plan + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-compatibility-plan + path: compatibility-plan.json + if-no-files-found: error + observe-released: - name: Observe engine ${{ matrix.version }} + name: Observe engine ${{ matrix.version }} (${{ matrix.label }}) needs: plan runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 strategy: fail-fast: false - matrix: - version: ${{ fromJSON(needs.plan.outputs.released) }} + matrix: ${{ fromJSON(needs.plan.outputs.released_targets) }} services: opensearch: image: opensearchproject/opensearch:${{ matrix.version }} env: discovery.type: single-node - # The lint contract only needs the PPL query and grammar endpoints, so - # run without the security plugin: no TLS or credentials to manage, and - # the observed error bodies are the engine's own rather than a proxy's. DISABLE_SECURITY_PLUGIN: 'true' DISABLE_INSTALL_DEMO_CONFIG: 'true' OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g @@ -182,41 +128,53 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Wait for the engine and confirm its version - id: engine + - name: Confirm released engine identity run: | set -euo pipefail - for i in $(seq 1 40); do - if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi - echo "waiting for engine (${i}/40)..." + for attempt in $(seq 1 40); do + curl -sf http://localhost:9200 > "$RUNNER_TEMP/engine-root.json" && break + echo "waiting for engine (${attempt}/40)..." sleep 5 done - cat /tmp/root.json - reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") - echo "reported=$reported" >> "$GITHUB_OUTPUT" - # A leg mislabeled as another version would attribute drift to the wrong - # engine, so require the image to be what the matrix asked for. + reported=$(jq -r '.version.number' "$RUNNER_TEMP/engine-root.json") case "$reported" in ${{ matrix.version }}*) ;; - *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + *) echo "::error::engine reported $reported; expected ${{ matrix.version }}"; exit 1 ;; esac - # The PPL plugin must actually be present, or every query would "pass" - # by failing identically. curl -sf http://localhost:9200/_cat/plugins | grep -i sql - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: - distribution: 'temurin' + distribution: temurin java-version: 21 - # The same contract oracle the single-version workflow runs, pointed at an - # external cluster instead of a Gradle-managed one. One oracle, many - # engines: a per-version copy would be free to drift from the real check. - - name: Run contract observation against engine ${{ matrix.version }} + - name: Observe backend contracts + env: + EXPORT_RUNTIME_BUNDLE: ${{ matrix.export_runtime_bundle }} run: | set -euo pipefail mkdir -p leg + bundle_args=() + if [ "$EXPORT_RUNTIME_BUNDLE" = 'true' ]; then + bundle_args+=("-Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json") + fi + { + printf './gradlew :integ-test:integTestRemote ' + printf '%q ' \ + '--tests' 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' \ + '-Dtests.rest.cluster=localhost:9200' \ + '-Dtests.cluster=localhost:9200' \ + '-Dtests.clustername=docker-cluster' \ + '-Dppl.lint.schedule=nightly' \ + '-Dppl.lint.observe.only=true' \ + '-Dppl.lint.execution_backend=standard' \ + "-Dppl.lint.sql_sha=$GITHUB_SHA" \ + "-Dppl.lint.report=$(pwd)/leg/backend-report.json" \ + "-Dppl.lint.target=$(pwd)/leg/target.json" \ + "${bundle_args[@]}" + echo + } > leg/backend-command.txt ./gradlew :integ-test:integTestRemote \ --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dtests.rest.cluster=localhost:9200 \ @@ -225,36 +183,37 @@ jobs: -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ -Dppl.lint.execution_backend=standard \ - -Dppl.lint.sql_sha="${GITHUB_SHA}" \ + -Dppl.lint.sql_sha="$GITHUB_SHA" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ - -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ - -Dppl.lint.target="$(pwd)/leg/target.json" + -Dppl.lint.target="$(pwd)/leg/target.json" \ + "${bundle_args[@]}" - - name: Upload leg artifacts + - name: Upload released observation if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-${{ matrix.version }} + name: ${{ matrix.artifact_name }} path: leg - if-no-files-found: error + if-no-files-found: warn - - name: Upload failure logs + - name: Upload released observation logs if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-${{ matrix.version }}-logs - path: integ-test/build/reports/** + name: ${{ matrix.artifact_name }}-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + if-no-files-found: warn - # The PR's own engine build, so the newest point in the matrix is the code under - # review rather than the last release. Same oracle as the released legs; the only - # difference is a Gradle-managed cluster instead of a published image, which is - # why it cannot just be another matrix entry. observe-pr-build: - name: Observe engine pr-build - needs: Get-CI-Image-Tag + name: Observe engine pr-build (runtime) + needs: + - Get-CI-Image-Tag + - plan runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -268,75 +227,67 @@ jobs: - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: - distribution: 'temurin' + distribution: temurin java-version: 21 - # Observe-only here too, so this leg reports what the PR engine does rather - # than duplicating the sibling workflow's assertions. The sibling workflow - # remains the enforcing single-version check. - - name: Run contract observation against the PR build + - name: Observe backend contracts run: | set -euo pipefail mkdir -p leg chown -R 1000:1000 "$(pwd)" - su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ - --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ - -Dppl.lint.schedule=nightly \ - -Dppl.lint.observe.only=true \ - -Dppl.lint.execution_backend=standard \ - -Dppl.lint.sql_sha=${GITHUB_SHA} \ - -Dppl.lint.report=$(pwd)/leg/backend-report.json \ - -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ - -Dppl.lint.target=$(pwd)/leg/target.json" + command="./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true -Dppl.lint.execution_backend=standard -Dppl.lint.sql_sha=$GITHUB_SHA -Dppl.lint.report=$(pwd)/leg/backend-report.json -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json -Dppl.lint.target=$(pwd)/leg/target.json" + printf '%s\n' "$command" > leg/backend-command.txt + su "$(id -un 1000)" -c "$command" - - name: Upload leg artifacts + - name: Upload PR-build observation if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-pr-build + name: ppl-lint-observation-pr-build-runtime path: leg - if-no-files-found: error + if-no-files-found: warn - - name: Upload failure logs + - name: Upload PR-build observation logs if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-pr-build-logs + name: ppl-lint-observation-pr-build-runtime-logs path: | integ-test/build/reports/** integ-test/build/test-results/** integ-test/build/testclusters/*/logs/* + if-no-files-found: warn - # Lint each engine's exported grammar with the OSD detectors. Separate from the - # observation legs because OSD needs a newer Node/glibc than the engine image - # provides, and because one bootstrap can serve every leg. - # - # `always()` so a single broken leg still yields a report for the others: a - # partial matrix must be visibly partial, not silently absent. The aggregate - # step fails if NO leg produced a report. - detect: + aggregate: name: Aggregate rule compatibility + if: ${{ always() && needs.plan.result == 'success' }} needs: - plan - observe-released - observe-pr-build - if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest - timeout-minutes: 40 - outputs: - osd_sha: ${{ steps.osd-rev.outputs.sha }} + timeout-minutes: 45 steps: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Download all leg artifacts + - name: Download compatibility plan + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-compatibility-plan + path: plan + + - name: Download available observations + continue-on-error: true uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: - pattern: ppl-lint-leg-* + pattern: ppl-lint-observation-* path: legs - name: Checkout OpenSearch-Dashboards + id: osd-checkout + continue-on-error: true uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ${{ needs.plan.outputs.osd_repo }} @@ -344,18 +295,27 @@ jobs: path: .ci/OpenSearch-Dashboards - name: Record OSD revision - id: osd-rev + id: osd-revision + if: ${{ always() }} run: | - sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) - echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "OSD revision: \`$sha\` (${{ needs.plan.outputs.osd_repo }} @ \`${{ needs.plan.outputs.osd_ref }}\`)" >> "$GITHUB_STEP_SUMMARY" + if sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD 2>/dev/null); then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" + fi - name: Set up Node from OSD .nvmrc + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - name: Pin Yarn from OSD engines + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true working-directory: .ci/OpenSearch-Dashboards run: | yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") @@ -363,123 +323,123 @@ jobs: npm install -g "yarn@${yarn_version}" - name: Cache OSD Yarn dependencies + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 with: - path: | - ~/.cache/yarn + path: ~/.cache/yarn key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-osd-yarn- + restore-keys: ${{ runner.os }}-osd-yarn- - - name: Bootstrap OpenSearch-Dashboards + - name: Bootstrap OpenSearch-Dashboards once + id: osd-bootstrap + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true working-directory: .ci/OpenSearch-Dashboards run: | - for i in 1 2 3; do + for attempt in 1 2 3; do yarn osd bootstrap && exit 0 - echo "Bootstrap attempt $i failed, retrying in 10s..." + echo "Bootstrap attempt $attempt failed; retrying in 10 seconds." sleep 10 done exit 1 - # One detector pass per leg, each against THAT engine's grammar bundle. The - # runner is the same SQL-owned script the single-version workflow uses, so - # the detector half cannot drift between the two checks. - - name: Run detectors against every engine grammar - working-directory: .ci/OpenSearch-Dashboards + - name: Run applicable detector passes + if: ${{ steps.osd-bootstrap.outcome == 'success' }} + continue-on-error: true run: | - set -euo pipefail - shopt -s nullglob - legs=("$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*) - if [ ${#legs[@]} -eq 0 ]; then - echo "::error::no leg artifacts were downloaded; nothing to validate." - exit 1 - fi - for leg in "${legs[@]}"; do - version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - if [ ! -f "$leg/ppl-grammar-bundle.json" ]; then - # Skip the log-only artifacts an observation failure may have uploaded. - echo "skipping $leg (no runtime grammar bundle)" + set -uo pipefail + while IFS= read -r configuration; do + id=$(jq -r '.id' <<< "$configuration") + artifact=$(jq -r '.artifactName' <<< "$configuration") + surface=$(jq -r '.surface' <<< "$configuration") + engine_mode=$(jq -r '.engineMode' <<< "$configuration") + leg="$GITHUB_WORKSPACE/legs/$artifact" + mkdir -p "$leg" + if [ ! -s "$leg/target.json" ] || [ ! -s "$leg/backend-report.json" ]; then + echo "Skipping detector pass for $id: backend evidence is incomplete." continue fi - echo "=== detectors vs engine $version (runtime-bundle surface) ===" - env PPL_LINT_SURFACE=runtime-bundle \ - PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ - PPL_LINT_OBSERVE_ONLY=1 \ - PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ - PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_INCLUDE_DORMANT=1 \ - PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ - PPL_LINT_REPORT="$leg/detector-report.json" \ - node -r ./src/setup_node_env \ - "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ - > "$leg/detector.log" 2>&1 || true - # A per-leg non-zero exit is EXPECTED when that engine disagrees with - # the pinned expectation — that is the drift this workflow exists to - # report, and the aggregate step below is what classifies it. Only a - # missing report means the runner itself broke. - if [ ! -f "$leg/detector-report.json" ]; then - echo "::error::detector runner produced no report for engine $version" - tail -50 "$leg/detector.log" || true - exit 1 + + if [ "$surface" = 'compiled-simplified' ]; then + grammar_root=".ci/OpenSearch-Dashboards/packages/osd-antlr-grammar/src/opensearch_ppl_simplified" + grammar_hash=$( + find "$grammar_root" -type f -print0 | + sort -z | + xargs -0 sha256sum | + sha256sum | + awk '{print "sha256:" $1}' + ) + jq --arg hash "$grammar_hash" \ + '.grammarHash = $hash | .grammarBundle = ""' \ + "$leg/target.json" > "$leg/detector-target.json" + bundle='' + else + cp "$leg/target.json" "$leg/detector-target.json" + bundle="$leg/ppl-grammar-bundle.json" + if [ ! -s "$bundle" ]; then + echo "Skipping detector pass for $id: runtime grammar bundle is missing." + continue + fi fi - tail -5 "$leg/detector.log" || true - done - # Compare every engine version against every other and against the pinned - # contracts, then print the remediation report. - - name: Aggregate drift across engine versions - id: aggregate - env: - RELEASED: ${{ needs.plan.outputs.released }} + { + printf 'cd %q && env ' "$GITHUB_WORKSPACE/.ci/OpenSearch-Dashboards" + printf 'PPL_LINT_SURFACE=%q ' "$surface" + printf 'PPL_LINT_ENGINE_MODE=%q ' "$engine_mode" + printf 'PPL_LINT_APPLICABLE_ONLY=1 ' + printf 'PPL_LINT_GRAMMAR_BUNDLE=%q ' "$bundle" + printf 'PPL_LINT_TARGET_MANIFEST=%q ' "$leg/detector-target.json" + printf 'PPL_LINT_BACKEND_REPORT=%q ' "$leg/backend-report.json" + printf 'PPL_LINT_CONTRACT_DIR=%q ' "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" + printf 'PPL_LINT_SCHEDULE=nightly PPL_LINT_OBSERVE_ONLY=1 ' + printf 'PPL_LINT_REPORT=%q ' "$leg/detector-report.json" + printf 'node -r ./src/setup_node_env %q\n' \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + } > "$leg/detector-command.txt" + + ( + cd .ci/OpenSearch-Dashboards + env \ + PPL_LINT_SURFACE="$surface" \ + PPL_LINT_ENGINE_MODE="$engine_mode" \ + PPL_LINT_APPLICABLE_ONLY=1 \ + PPL_LINT_GRAMMAR_BUNDLE="$bundle" \ + PPL_LINT_TARGET_MANIFEST="$leg/detector-target.json" \ + PPL_LINT_BACKEND_REPORT="$leg/backend-report.json" \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_OBSERVE_ONLY=1 \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + ) > "$leg/detector.log" 2>&1 || true + tail -10 "$leg/detector.log" || true + done < <(jq -c '.configurations[]' plan/compatibility-plan.json) + + - name: Aggregate every planned rule and configuration + id: compatibility + if: ${{ always() }} run: | - set -euo pipefail - shopt -s nullglob - args=() - present=() - for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do - [ -f "$leg/detector-report.json" ] || continue - version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - args+=(--leg "$version=$leg") - present+=("$version") - done - if [ ${#args[@]} -eq 0 ]; then - echo "::error::no complete legs to aggregate." - exit 1 - fi - # Every version the plan asked for must have produced a leg. Aggregating - # only the survivors would report "PASS: agrees with all N versions" over - # a matrix that silently lost one — the exact vacuous pass this workflow - # exists to prevent. A dead leg is a failure, not a smaller matrix. - missing=() - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do - found=no - for have in "${present[@]}"; do - [ "$have" = "$want" ] && found=yes && break - done - [ "$found" = yes ] || missing+=("$want") - done - if [ ${#missing[@]} -gt 0 ]; then - echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." - exit 1 - fi set +e - node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ - --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ - --out "$GITHUB_WORKSPACE/drift-report.json" \ - --summary "$GITHUB_STEP_SUMMARY" \ - --all-rules \ - "${args[@]}" - aggregate_exit=$? - set -e - echo "exit_code=$aggregate_exit" >> "$GITHUB_OUTPUT" + node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan plan/compatibility-plan.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --artifacts legs \ + --osd-sha "${{ steps.osd-revision.outputs.sha }}" \ + --out drift-report.json \ + --summary "$GITHUB_STEP_SUMMARY" + result=$? + echo "exit_code=$result" >> "$GITHUB_OUTPUT" + exit 0 - name: Upload drift report if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ppl-lint-multiversion-drift - if-no-files-found: error path: drift-report.json + if-no-files-found: error - name: Upload compatibility evidence if: ${{ always() }} @@ -487,277 +447,45 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ppl-lint-multiversion-evidence - if-no-files-found: warn path: | + plan/compatibility-plan.json + legs/**/backend-report.json legs/**/detector-report.json - legs/**/detector.log legs/**/target.json + legs/**/detector-target.json + legs/**/*-command.txt + legs/**/detector.log + if-no-files-found: warn - - name: Fail after publishing compatibility drift + - name: Fail after publishing compatibility results if: ${{ always() }} env: - AGGREGATE_EXIT: ${{ steps.aggregate.outputs.exit_code }} - RELEASED: ${{ needs.plan.outputs.released }} + AGGREGATE_EXIT: ${{ steps.compatibility.outputs.exit_code }} run: | set -euo pipefail - if [ -z "$AGGREGATE_EXIT" ]; then - echo "::error::aggregation did not complete" + if [ ! -s drift-report.json ]; then + echo "::error::drift-report.json was not published" exit 1 fi - expected_legs=$(echo "$RELEASED" | jq 'length + 1') - expected_rules=$(jq -c \ - '[.contracts[] | sub("\\.spec\\.json$"; "")] | sort' \ - integ-test/src/test/resources/ppl-lint/contracts/manifest.json) - if [ ! -s drift-report.json ] || ! jq -e \ - --argjson expected_legs "$expected_legs" \ - --argjson expected_rules "$expected_rules" \ - ' - type == "object" and - ($expected_rules | length) == 12 and - ($expected_rules | index("command-suggestion") | not) and - (.legs | type == "array" and length == $expected_legs) and - (.matrix | type == "array") and - (.matrix | length) == (12 * $expected_legs) and - (.matrix | map(.ruleId) | unique | sort) == $expected_rules and - (.matrix | map(.legKey) | unique | length) == $expected_legs and - (.matrix | map(.legKey) | unique | sort) == - (.legs | map(.key) | unique | sort) and - ([.matrix[] | [.ruleId, .legKey]] | unique | length) == - (12 * $expected_legs) - ' drift-report.json > /dev/null; then - echo "::error::aggregation did not produce the complete 12-rule compatibility matrix" + jq -e ' + .schemaVersion == 3 and + .inventory.ruleCount == 12 and + (.inventory.ruleIds | length) == 12 and + (.configurations | length) == 3 and + (.matrix | length) == 36 and + .result.cellCount == 36 and + ( + .result.compatible + .result.notApplicable + + .result.drift + .result.inconclusive + ) == 36 + ' drift-report.json > /dev/null + if [ -z "$AGGREGATE_EXIT" ]; then + echo "::error::compatibility aggregation did not complete" exit 1 fi if [ "$AGGREGATE_EXIT" -ne 0 ]; then - echo "::error::rule compatibility validation failed; see the table above and the ppl-lint-multiversion-drift artifact" + drift=$(jq -r '.result.drift' drift-report.json) + inconclusive=$(jq -r '.result.inconclusive' drift-report.json) + echo "::error::rule compatibility validation failed after artifact publication: $drift drift, $inconclusive inconclusive" exit "$AGGREGATE_EXIT" fi - - # Discovery: harvest queries from OSD's own lint tests, run both halves over them, - # and report detector/engine disagreements as LEADS. - # - # Why this is separate from `detect`, and why it can never fail the build: - # - # The enforced corpus is hand-pinned — every expectation is a reviewed claim, which - # is what lets a mismatch red the build. That corpus is also small (about one - # trigger per rule), and `classifyRelaxationScope` needs SEVERAL triggers per rule - # to tell a FULL engine fix (version-scope the rule away) from a PARTIAL one - # (narrow the detector). Those need opposite actions, so with one trigger the - # advice can be confidently wrong. - # - # This job supplies that trigger variety from queries OSD's own detector authors - # already wrote. It pins NOTHING: roles are derived from real detector output and - # the engine supplies the other half, so no expectation is ever auto-generated. - # An auto-derived expectation could only confirm current behavior — locking in - # whatever the detector does today, bugs included. - # - # `continue-on-error` AND a zero exit from the labeler: a finding here is a lead to - # investigate, not a proven defect, and blocking unrelated PRs on an auto-generated - # guess would poison the whole check's credibility. - discovery: - name: Discovery corpus (harvested, not enforced) - # Only `plan`, for the OSD target and the engine version. Deliberately NOT the - # observe legs: discovery runs its own engine and harvests its own queries, so - # depending on them would idle this job behind ~30 minutes of matrix work it - # never reads, and a failed leg would block a report that does not need it. - needs: plan - continue-on-error: true - runs-on: ubuntu-latest - timeout-minutes: 40 - services: - opensearch: - image: opensearchproject/opensearch:${{ needs.plan.outputs.discovery_engine }} - env: - discovery.type: single-node - DISABLE_SECURITY_PLUGIN: 'true' - DISABLE_INSTALL_DEMO_CONFIG: 'true' - OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g - ports: - - 9200:9200 - options: >- - --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" - --health-interval 15s - --health-timeout 10s - --health-retries 20 - --health-start-period 60s - steps: - - name: Checkout SQL pull request - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Checkout OpenSearch-Dashboards - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - repository: ${{ needs.plan.outputs.osd_repo }} - ref: ${{ needs.plan.outputs.osd_ref }} - path: .ci/OpenSearch-Dashboards - - - name: Set up Node from OSD .nvmrc - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 - with: - node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - - - name: Pin Yarn from OSD engines - working-directory: .ci/OpenSearch-Dashboards - run: | - yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") - yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') - npm install -g "yarn@${yarn_version}" - - - name: Cache OSD Yarn dependencies - uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 - with: - path: | - ~/.cache/yarn - key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-osd-yarn- - - - name: Bootstrap OpenSearch-Dashboards - working-directory: .ci/OpenSearch-Dashboards - run: | - for i in 1 2 3; do - yarn osd bootstrap && exit 0 - echo "Bootstrap attempt $i failed, retrying in 10s..." - sleep 10 - done - exit 1 - - # The rule id list comes from the OSD catalog being validated, not a hardcoded - # copy: attribution keys off `describe('')` titles, so a stale list - # would silently stop harvesting queries for any newly added rule. - - name: Harvest the discovery corpus from OSD's lint tests - run: | - set -euo pipefail - node -e " - const c = require('./.ci/OpenSearch-Dashboards/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); - process.stdout.write(JSON.stringify(c.map((r) => r.id))); - " > /tmp/catalog-rules.json - node scripts/ppl-lint/harvest-queries.mjs \ - --osd .ci/OpenSearch-Dashboards \ - --catalog-rules @/tmp/catalog-rules.json \ - --index opensearch-sql_test_index_account \ - --out "$GITHUB_WORKSPACE/discovery-corpus.json" \ - --specs-out "$GITHUB_WORKSPACE/discovery-specs" - - # Seed the one index every harvested query was rewritten onto. Without it the - # engine rejects everything with IndexNotFoundException — which the labeler - # would correctly suppress as uninformative, yielding a run that reports - # nothing at all. - - name: Seed the fixture index - run: | - set -euo pipefail - for i in $(seq 1 40); do - curl -sf http://localhost:9200 > /dev/null && break - echo "waiting for engine (${i}/40)..." - sleep 5 - done - curl -sf -X PUT "http://localhost:9200/opensearch-sql_test_index_account" \ - -H 'content-type: application/json' -d '{ - "mappings": { "properties": { - "account_number": { "type": "long" }, - "balance": { "type": "long" }, - "age": { "type": "integer" }, - "status": { "type": "keyword" }, - "firstname": { "type": "text" }, - "lastname": { "type": "text" }, - "msg": { "type": "text" }, - "body": { "type": "text" }, - "raw": { "type": "object", "enabled": false } - } } - }' - curl -sf -X POST "http://localhost:9200/opensearch-sql_test_index_account/_doc?refresh=true" \ - -H 'content-type: application/json' \ - -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' - - # Export this engine's grammar bundle. Discovery only runs against the runtime - # grammar surface, so an unavailable endpoint skips the detector pass. - - name: Export the engine grammar bundle - id: bundle - run: | - set -uo pipefail - if curl -sf --max-time 60 "http://localhost:9200/_plugins/_ppl/_grammar" \ - -o "$GITHUB_WORKSPACE/discovery-bundle.json"; then - hash=$(python3 -c " - import json - print(json.load(open('$GITHUB_WORKSPACE/discovery-bundle.json')).get('grammarHash','')) - ") - python3 -c " - import json - json.dump({'schemaVersion': 2, - 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', - 'grammarHash': '$hash', - 'grammarBundle': 'discovery-bundle.json', - 'executionBackend': 'standard', - 'storage': 'lucene', - 'shardCount': 1}, - open('$GITHUB_WORKSPACE/discovery-target.json','w')) - " - echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" - else - echo "::warning::_grammar export failed; skipping discovery detector pass." - echo "surface=unavailable" >> "$GITHUB_OUTPUT" - fi - - - name: Run the detectors over the discovery corpus - working-directory: .ci/OpenSearch-Dashboards - env: - SURFACE: ${{ steps.bundle.outputs.surface }} - run: | - set -uo pipefail - if [ "$SURFACE" != 'runtime-bundle' ]; then - echo "Discovery detector pass skipped: runtime grammar bundle unavailable." - exit 0 - fi - # A non-zero exit is EXPECTED and ignored: the generated specs carry - # placeholder expectations, so the runner reports a "failure" for every - # query whose real diagnostic count differs. Only the report is read. - env PPL_LINT_DISCOVERY=1 \ - PPL_LINT_SURFACE=runtime-bundle \ - PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" \ - PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json" \ - PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ - PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ - node -r ./src/setup_node_env \ - "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ - > "$GITHUB_WORKSPACE/discovery-detector.log" 2>&1 || true - if [ ! -f "$GITHUB_WORKSPACE/discovery-detector-report.json" ]; then - echo "::warning::the detector runner produced no discovery report; skipping." - tail -50 "$GITHUB_WORKSPACE/discovery-detector.log" || true - fi - - - name: Probe the engine with the discovery corpus - run: | - set -euo pipefail - node scripts/ppl-lint/probe-discovery-backend.mjs \ - --corpus discovery-corpus.json \ - --endpoint http://localhost:9200 \ - --out discovery-backend-report.json - - - name: Label and report - run: | - set -euo pipefail - if [ ! -f discovery-detector-report.json ]; then - echo "::warning::no detector report; nothing to label." - exit 0 - fi - node scripts/ppl-lint/label-discovery.mjs \ - --corpus discovery-corpus.json \ - --detector discovery-detector-report.json \ - --backend discovery-backend-report.json \ - --version "${{ needs.plan.outputs.discovery_engine }}" \ - --out discovery-findings.json \ - --summary "$GITHUB_STEP_SUMMARY" - - - name: Upload discovery artifacts - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true - with: - name: ppl-lint-discovery - path: | - discovery-corpus.json - discovery-findings.json - discovery-detector-report.json - discovery-backend-report.json - discovery-detector.log diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 660a88c09af..769435da88b 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -114,7 +114,10 @@ writes `detector-report.json`. | --- | --- | | `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | | `PPL_LINT_SCHEDULE` | `pr` or `nightly` | -| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | +| `PPL_LINT_SURFACE` | `runtime-bundle` (default) or explicit `compiled-simplified` | +| `PPL_LINT_ENGINE_MODE` | optional `calcite` or `legacy` identity for compatibility filtering/context | +| `PPL_LINT_APPLICABLE_ONLY` | `1` omits contracts excluded by surface, version, or engine mode | +| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required on the runtime surface) | | `PPL_LINT_TARGET_MANIFEST` | schema-v2 `target.json` (engine, grammar, execution backend, and storage identity) | | `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | | `PPL_LINT_REPORT` | where to write `detector-report.json` | @@ -223,40 +226,49 @@ status, not blocking behavior**. The check above validates **one** engine: the build from the PR. But a lint rule ships to every user, and each user's cluster is on whatever version they run. A -rule that is correct on `main` can be a false positive on 3.6 or a false negative -on 3.7, and the single-version check cannot see it. +rule that is correct on `main` can be a false positive or false negative on a +released cluster, and the single-version check cannot see it. [`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) -validates every active shipping detector against several engine versions at -once, and reports **what to change in the linter** when one disagrees. +validates every active shipping detector against three planned configurations +and reports **what to change** when one disagrees: + +- the OSD compiled-simplified fallback grammar against OpenSearch 2.19.6; +- the runtime grammar exported by the highest official GA release at or below + the normalized SQL PR target; +- the runtime grammar exported by the SQL PR build. ``` -observe (matrix: released 3.6/3.7 images + standard pr-build) - └── each leg exports the same 4 artifacts as the single-version check -aggregate rule compatibility (one OSD bootstrap, one detector pass per runtime grammar) - └── aggregate-versions.mjs → drift-report.json + remediation report +plan configurations + ├── observe 2.19.6 backend (compiled comparison) + ├── observe latest eligible GA + export runtime grammar + └── observe PR build + export runtime grammar +aggregate rule compatibility (one OSD bootstrap, three detector passes) + └── aggregate-compatibility.mjs → 36-cell schema-v3 report ``` -Released legs run the official `opensearchproject/opensearch:` image, -which bundles the matching `opensearch-sql` plugin, so no old branch is built. The -`pr-build` leg is the same Gradle test cluster the single-version check uses. The -legs run the **same** contract oracle (`PplLintRuleValidationIT`) with +Released legs run official `opensearchproject/opensearch:` images, +which bundle the matching `opensearch-sql` plugin, so no old branch is built. +The plan reads the default `opensearch.version` from `build.gradle`, removes its +prerelease/build suffix, and selects the highest exact-semver OpenSearch tag at +or below it. The `pr-build` leg uses a Gradle test cluster. All legs run the +**same** contract oracle (`PplLintRuleValidationIT`) with `-Dppl.lint.observe.only=true`, which records real behavior instead of asserting against expectations — on an older engine a mismatch is the signal being collected, not a broken run. -**Engine floor: 3.6.0.** -`GET /_plugins/_ppl/_grammar` landed in #5162, which is an ancestor of 3.6 but not -3.5, so a 3.5 leg cannot export a grammar bundle for the detectors to lint against. - -This workflow intentionally excludes the compiled-simplified surface and -analytics engine. Those dimensions do not share the stable runtime-bundle -contract being compared here. +The 2.19.6 leg does not request a runtime bundle. Its backend observations are +joined to a detector pass over OSD's checked-in simplified grammar. Rules with +`grammarSurface: runtime-bundle` are `n/a (surface)` there; rules outside their +`wiring.appliesTo` version range are `n/a (version)`. Surface takes precedence +when both exclusions apply. Analytics, syntax-channel, dormant-rule, discovery, +and AI action tests are not part of this workflow. -Observation jobs do not fail on compatibility differences. The final +Observation jobs do not fail on compatibility differences. A failed or missing +observation remains a planned column and becomes `inconclusive`. The final `Aggregate rule compatibility` job writes the complete expected-versus-actual -table and `drift-report.json`, uploads them, and then fails when a rule drifts on -a version declared by its `wiring.appliesTo` scope. +table and `drift-report.json`, uploads both report and evidence, and only then +fails for drift or inconclusive in-scope cells. ### What a drift report tells you @@ -264,20 +276,16 @@ Every finding names a drift class, the evidence, and one remediation action: | Action | When | What you change | | --- | --- | --- | -| `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | +| `scope-rule-version` | every contracted trigger is now accepted and controls prove support | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` | +| `narrow-detector` | only some contracted triggers are now accepted | keep the version in scope and narrow the detector to invalid forms | | `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | | `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | -| `align-execution-backends` | standard and analytics disagree for the same SQL version and grammar | reconcile the detector with both routes or add a reliable backend signal to OSD | - -Drift classes: `grammar-rule-missing` (a parser rule the detector walks was -renamed or removed — the finding names the closest current rule names), -`engine-relaxed` / `engine-partially-relaxed` / `engine-tightened` (the engine's -verdict flipped), `engine-message-changed` (same verdict, reworded error), -`detector-silent` / `detector-noisy` (false negative / false positive), -`version-scope-too-narrow` (the engine rejects but the rule is scoped away from -that version, so users see no diagnostic), `execution-backend-divergence` (same -version, different route verdict), and `severity-mismatch`. Backend divergence -never recommends changing a version range. +| `fix-test-leg` | a detector/backend row or target identity is missing or errored | repair or rerun the test leg before changing product behavior | + +The schema-v3 report consolidates query symptoms into four rule/configuration +classifications: `detector-regression`, `full-engine-relaxation`, +`partial-engine-relaxation`, and `contract-drift`. Missing or errored evidence +is `inconclusive`, not a compatibility classification. #### Full vs partial relaxation: scope the rule, or narrow the detector? @@ -285,10 +293,12 @@ When an engine starts accepting a query a rule flags, the fix depends on a quest a single query cannot answer: is the behavior **fully** gone on that version, or only **partially**? -- **Every trigger relaxed** → `engine-relaxed`, action `version-scope-rule`. Nothing +- **Every trigger relaxed** → `full-engine-relaxation`, action + `scope-rule-version`. Nothing the rule claims is still true on that engine, so bound it with `maxVersion`. -- **Some triggers relaxed, others still rejected** → `engine-partially-relaxed`, - action `update-detector`. The engine fixed *part* of the condition. Scoping the +- **Some triggers relaxed, others still rejected** → + `partial-engine-relaxation`, action `narrow-detector`. The engine fixed *part* + of the condition. Scoping the rule away here would drop the diagnostics that are still correct, converting a partial engine fix into a shipped **false negative**. Narrow the detector so it stops matching the now-valid shapes while still flagging the rest. @@ -299,24 +309,16 @@ supersedes the per-query ones. A trigger with no verdict is counted as neither treating it as "still rejects" would let a timed-out leg masquerade as a partial fix and send someone to narrow a healthy detector. -The evidence always states the tally (`2 of 3 observed trigger(s) relaxed`), and a -rule with only one pinned trigger gets an explicit warning that a "fully relaxed" -verdict rests on a single observation. That is the gap the discovery corpus below -closes. - -Three hard guards keep the check from passing vacuously. The shipping census is -also recorded, but remains report-only until the paired OSD default-alignment -change lands: - -- A rule that is default-error in OSD's catalog but has no contract file is - reported in the shipping census. The detector runner records the catalog's - default-error census in `detector-report.json`, and the aggregate step compares - it against `manifest.defaultError`. This becomes blocking when census - enforcement is enabled after OSD defaults are aligned. -- A leg whose artifacts are missing is a hard failure, never a silently dropped - version. The aggregate step also checks that every version the plan asked for - produced a report, so a dead observe job cannot shrink the matrix into a green - "agrees with all N versions". +The evidence always states the contracted, accepted, rejected, and missing +trigger tally. A one-trigger rule can be fully relaxed when that trigger and its +controls produce complete evidence. + +Four hard guards keep the check from passing vacuously: + +- The active manifest must contain exactly the approved 12 rule IDs. +- A leg whose artifacts are missing remains in the matrix as a complete + `inconclusive` column. A dead observe job cannot shrink the matrix into a green + result. - A case with no engine verdict (a transport failure, recorded by the IT as `outcome: "error"`) is **not** read as acceptance. Coercing it would report a timeout as an engine that now accepts the query — and advise disabling a @@ -327,81 +329,46 @@ change lands: — it proved nothing. Inconclusive findings say "check that leg's logs and re-run", never "edit the rule", because the linter is not what went wrong. -A rule that is out of scope on an engine (`appliesTo` excludes it) and that the -engine also accepts is reported as `n/a (out of scope)`, not as drift — that is -the version window working. But if the engine *rejects* the trigger there, it is -`version-scope-too-narrow`. +A rule that is out of scope for a surface, version, or engine mode is not +executed or compared. Its cell is `n/a` with the corresponding reason and never +blocks the job. ### Where a failure shows up in the GitHub UI -Every finding is emitted twice, because the run page and the diff are two -different places a developer looks: - -1. **Annotations** (top of the run page, and inline on the file in *Files - changed* when the contract is part of the PR's diff). Each carries the drift - class, the rule, the engine version, and the one-line action. An - `update-contract` finding anchors on the exact `expectations[]` entry whose - `version` range produced it — not the top of the file — so the drift appears on - the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on - the contract's `ruleId` instead. -2. **The job summary** — the rule × version table plus the full grouped - remediation report, which stays the authoritative account. - -The required single-version lane follows the same rule: frontend and backend -failures with a `[rule/query]` identity anchor on that contract's `ruleId`. -An individual detector/query execution error is recorded as an `error` row and -does not stop the remaining contracts from running or prevent -`detector-report.json` from being uploaded. The required check still fails after -the complete report is written, with the failing rule/query named directly. -Shipping-census findings anchor on `manifest.json` and remain report-only until -the paired OSD default-alignment change lands. Artifact and job failures without -a trustworthy repository location remain file-less rather than pointing at a -guessed line. - -Without the annotations the only thing above the summary is `Process completed -with exit code 1`, so the natural next click lands in raw job logs rather than the -remediation. Severity is not cosmetic: - -| Finding | Level | Why | -| --- | --- | --- | -| enforced drift, coverage hole | `error` | an active shipping rule disagrees with a supported engine; aggregation writes the report and then fails | -| non-enforced drift | `warning` | reported, but it does not block | -| `inconclusive` | `warning` | "we could not check" is a leg problem, not evidence that a rule is wrong | -| unvalidated default-error rule | `error` (no file) | the edit goes in `manifest.json`, not a contract | - -A line number is emitted only when it is unambiguous. If a contract pins the same -version range twice, or the range cannot be found, the annotation carries the file -and no line — a wrong line sends the reader to edit the wrong expectation, which -is worse than making them find it. +The `Aggregate rule compatibility` step summary is the primary interface. It +always contains all 12 rows and all three columns, followed by blocking findings +and remediation. `ppl-lint-multiversion-drift/drift-report.json` carries the +complete schema-v3 matrix and query cases; `ppl-lint-multiversion-evidence` +carries target identities, detector/backend reports, logs, and reproduction +commands. The final step fails only after both uploads have run. ### Running the multi-version check locally -Each leg needs a reachable cluster. Point the observe step at any running engine: +Use the planner with an exact-semver tag list, then point the aggregator at the +three artifact directories produced by backend and detector runs: ```bash -# Observe one engine (repeat per version into its own leg dir). -mkdir -p legs/3.7.0 -./gradlew :integ-test:integTestRemote \ - --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ - -Dtests.rest.cluster=localhost:9200 \ - -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ - -Dppl.lint.report=$PWD/legs/3.7.0/backend-report.json \ - -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ - -Dppl.lint.target=$PWD/legs/3.7.0/target.json - -# Lint each leg's grammar from an OSD checkout (writes detector-report.json), -# then compare every standard runtime-bundle version at once. The aggregator -# writes the table and JSON report before returning a failing drift status. -node scripts/ppl-lint/aggregate-versions.mjs \ +git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > /tmp/opensearch-release-tags.txt +node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags /tmp/opensearch-release-tags.txt \ + --compiled-version 2.19.6 \ + --sql-sha "$(git rev-parse HEAD)" \ + --osd-repository opensearch-project/OpenSearch-Dashboards \ + --osd-ref main \ + --out compatibility-plan.json + +node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan compatibility-plan.json \ --contracts integ-test/src/test/resources/ppl-lint/contracts \ - --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg pr-build=legs/pr-build \ + --artifacts legs \ + --osd-sha "" \ --out drift-report.json ``` The step summary has one row per active detector. It prints the compatibility -declared by `wiring.appliesTo` next to the actual result for every engine leg. -For example, a rule with `minVersion: 3.7.0` renders `expected n/a` on 3.6 -instead of reporting drift. +declared by `wiring.appliesTo` and `grammarSurface` next to each actual result. The classifier is pure and has no cluster or OSD dependency, so its tests run anywhere: @@ -412,9 +379,8 @@ node --test "scripts/ppl-lint/__tests__/*.test.mjs" ## Discovery corpus (harvested, never enforced) -The required corpus is hand-pinned, which is what lets a mismatch red the build. -The `discovery` job builds a larger unpinned corpus to distinguish full engine -fixes from partial behavior changes. +The discovery scripts remain available as standalone investigation tooling. +They are not invoked by the multi-surface compatibility workflow. ``` harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report diff --git a/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs new file mode 100644 index 00000000000..13df6796325 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs @@ -0,0 +1,500 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { resolveBackendOracle } from '../contract-schema.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-compatibility.mjs'); +const REPOSITORY = path.resolve(HERE, '..', '..', '..'); +const CONTRACTS = path.join( + REPOSITORY, + 'integ-test', + 'src', + 'test', + 'resources', + 'ppl-lint', + 'contracts' +); +const WORKFLOW = path.join( + REPOSITORY, + '.github', + 'workflows', + 'ppl-lint-multiversion-validation.yml' +); +const RULE_IDS = JSON.parse( + fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8') +).contracts.map((file) => file.replace(/\.spec\.json$/, '')); +const CONFIGURATIONS = [ + { + id: '2.19.6-compiled', + label: '2.19.6 compiled', + engineVersion: '2.19.6', + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: 'ppl-lint-observation-2.19.6-compiled', + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: 'Latest release (3.8.0) runtime', + engineVersion: '3.8.0', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: '3.8.0-SNAPSHOT', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, +]; +const temporaryDirectories = []; + +function temporaryDirectory(prefix) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function version(value) { + const match = /^(\d+)\.(\d+)\.(\d+)/.exec(value); + return match.slice(1, 4).map(Number); +} + +function compare(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +function applicable(spec, configuration) { + const appliesTo = spec.wiring.appliesTo || {}; + const surfaces = + spec.grammarSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [spec.grammarSurface || 'runtime-bundle']; + if (!surfaces.includes(configuration.surface)) return false; + const actual = version(configuration.engineVersion); + if (appliesTo.minVersion && compare(actual, version(appliesTo.minVersion)) < 0) return false; + if (appliesTo.maxVersion && compare(actual, version(appliesTo.maxVersion)) > 0) return false; + return !appliesTo.engine || appliesTo.engine === configuration.engineMode; +} + +function rangeMatches(range, engineVersion) { + const actual = version(engineVersion); + return String(range || '') + .trim() + .split(/\s+/) + .filter(Boolean) + .every((token) => { + const match = /^(>=|<=|>|<|=)?(.+)$/.exec(token); + const comparison = compare(actual, version(match[2])); + return ( + (!match[1] && comparison === 0) || + (match[1] === '=' && comparison === 0) || + (match[1] === '>=' && comparison >= 0) || + (match[1] === '<=' && comparison <= 0) || + (match[1] === '>' && comparison > 0) || + (match[1] === '<' && comparison < 0) + ); + }); +} + +function loadSpecs() { + return new Map( + JSON.parse(fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8')).contracts.map( + (file) => { + const spec = JSON.parse(fs.readFileSync(path.join(CONTRACTS, file), 'utf8')); + return [spec.ruleId, spec]; + } + ) + ); +} + +function selectedExpectation(spec, configuration) { + return spec.expectations.find( + (expectation) => + rangeMatches(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); +} + +function target(configuration, grammarHash) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: configuration.engineVersion, + grammarHash, + grammarBundle: + configuration.surface === 'runtime-bundle' ? 'ppl-grammar-bundle.json' : '', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; +} + +function writeHealthyArtifacts(root) { + const specs = loadSpecs(); + for (const configuration of CONFIGURATIONS) { + const directory = path.join(root, configuration.artifactName); + fs.mkdirSync(directory, { recursive: true }); + const grammarHash = + configuration.surface === 'runtime-bundle' + ? `sha256:${configuration.id}` + : 'sha256:compiled-grammar'; + const backendTarget = target( + configuration, + configuration.surface === 'runtime-bundle' ? grammarHash : '' + ); + const detectorTarget = target(configuration, grammarHash); + const detectorResults = []; + const backendResults = []; + + for (const spec of specs.values()) { + if (!applicable(spec, configuration)) continue; + const expectation = selectedExpectation(spec, configuration); + assert.ok(expectation, `fixture expectation for ${spec.ruleId} on ${configuration.id}`); + for (const [queryName, queryDefinition] of Object.entries(spec.queries)) { + const resolved = resolveBackendOracle( + spec, + expectation.queries[queryName], + 'standard' + ); + assert.equal(resolved.status, 'applicable'); + const rejected = resolved.oracle.kind === 'rejection'; + const error = resolved.oracle.body && resolved.oracle.body.error; + detectorResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + expected: resolved.detector.count, + actual: resolved.detector.count, + severities: + resolved.detector.count > 0 && resolved.detector.severity + ? [resolved.detector.severity] + : [], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }); + backendResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + rejected, + executionBackend: 'standard', + observed: { + rejected, + httpStatus: resolved.oracle.httpStatus, + ...(error && error.type ? { type: error.type } : {}), + ...(error && error.reason ? { reason: error.reason } : {}), + }, + }); + } + } + + fs.writeFileSync(path.join(directory, 'target.json'), JSON.stringify(backendTarget)); + fs.writeFileSync( + path.join(directory, 'detector-target.json'), + JSON.stringify(detectorTarget) + ); + fs.writeFileSync( + path.join(directory, 'backend-report.json'), + JSON.stringify(backendResults) + ); + fs.writeFileSync( + path.join(directory, 'detector-report.json'), + JSON.stringify({ + schemaVersion: 2, + engineVersion: configuration.engineVersion, + grammarHash, + executionBackend: 'standard', + surface: configuration.surface, + results: detectorResults, + }) + ); + if (configuration.surface === 'runtime-bundle') { + fs.writeFileSync( + path.join(directory, 'ppl-grammar-bundle.json'), + JSON.stringify({ grammarHash }) + ); + } + fs.writeFileSync(path.join(directory, 'backend-command.txt'), 'backend command\n'); + fs.writeFileSync(path.join(directory, 'detector-command.txt'), 'detector command\n'); + } +} + +function createFixture() { + const directory = temporaryDirectory('ppl-lint-compatibility-'); + const artifacts = path.join(directory, 'legs'); + fs.mkdirSync(artifacts); + writeHealthyArtifacts(artifacts); + const plan = { + schemaVersion: 1, + sqlSha: 'candidate-sql-sha', + prTargetVersion: '3.8.0-SNAPSHOT', + normalizedPrTarget: '3.8.0', + latestEligibleGa: '3.8.0', + osd: { repository: 'example/osd', ref: 'main' }, + configurations: CONFIGURATIONS, + }; + const planFile = path.join(directory, 'compatibility-plan.json'); + fs.writeFileSync(planFile, JSON.stringify(plan)); + return { directory, artifacts, planFile }; +} + +function run(fixture) { + const reportFile = path.join(fixture.directory, 'drift-report.json'); + const summaryFile = path.join(fixture.directory, 'summary.md'); + const result = spawnSync( + process.execPath, + [ + SCRIPT, + '--plan', + fixture.planFile, + '--contracts', + CONTRACTS, + '--artifacts', + fixture.artifacts, + '--osd-sha', + 'osd-sha', + '--out', + reportFile, + '--summary', + summaryFile, + ], + { encoding: 'utf8' } + ); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + report: fs.existsSync(reportFile) + ? JSON.parse(fs.readFileSync(reportFile, 'utf8')) + : undefined, + summary: fs.existsSync(summaryFile) ? fs.readFileSync(summaryFile, 'utf8') : '', + }; +} + +function editBackend(fixture, configurationId, ruleId, queryName, patch) { + const configuration = CONFIGURATIONS.find((entry) => entry.id === configurationId); + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'backend-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + const row = report.find( + (entry) => entry.ruleId === ruleId && entry.queryName === queryName + ); + Object.assign(row, patch); + Object.assign(row.observed, patch.observed || {}); + fs.writeFileSync(file, JSON.stringify(report)); +} + +test('emits exactly 12 rules, 3 configurations, and 36 complete cells', () => { + const result = run(createFixture()); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.report.schemaVersion, 3); + assert.equal(result.report.inventory.ruleCount, 12); + assert.deepEqual(result.report.inventory.ruleIds, [...RULE_IDS].sort()); + assert.equal(result.report.configurations.length, 3); + assert.equal(result.report.matrix.length, 36); + assert.deepEqual(result.report.result, { + status: 'pass', + cellCount: 36, + compatible: 28, + notApplicable: 8, + drift: 0, + inconclusive: 0, + exitCode: 0, + }); + assert.equal( + result.summary.split('\n').filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)) + .length, + 12 + ); +}); + +test('uses surface before version when a compiled pair has multiple exclusions', () => { + const { report } = run(createFixture()); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'invalid-capture-group-name' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'surface' + ); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'agg-on-text' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'version' + ); +}); + +test('classifies a one-trigger full engine relaxation and keeps the complete table', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'wildcard-source-zero-match', + 'missing-wildcard-source', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const cell = result.report.matrix.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(cell.classification, 'full-engine-relaxation'); + assert.deepEqual(cell.triggerSummary, { + contracted: 1, + acceptedByBackend: 1, + rejectedByBackend: 0, + missing: 0, + }); + assert.equal( + result.report.findings.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ).remediation.action, + 'scope-rule-version' + ); +}); + +test('classifies partial relaxation separately and never recommends version scoping', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'union-min-datasets', + 'union-single-dataset', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + const finding = result.report.findings.find( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(finding.classification, 'partial-engine-relaxation'); + assert.equal(finding.remediation.action, 'narrow-detector'); + assert.ok( + !result.report.findings.some( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.remediation.action === 'scope-rule-version' + ) + ); +}); + +test('a detector regression writes JSON and every summary row before exiting nonzero', () => { + const fixture = createFixture(); + const configuration = CONFIGURATIONS[1]; + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'detector-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + report.results.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && entry.queryName === 'parse-text-field' + ).actual = 0; + fs.writeFileSync(file, JSON.stringify(report)); + + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + assert.equal( + result.report.matrix.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && + entry.configurationId === 'latest-release-runtime' + ).classification, + 'detector-regression' + ); + assert.equal( + result.summary + .split('### Blocking findings')[0] + .split('\n') + .filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)).length, + 12 + ); + assert.match(result.stderr, /after writing the complete report/); +}); + +test('a missing observation preserves the full column as inconclusive', () => { + const fixture = createFixture(); + fs.rmSync( + path.join( + fixture.artifacts, + CONFIGURATIONS[1].artifactName + ), + { recursive: true } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const column = result.report.matrix.filter( + (entry) => entry.configurationId === 'latest-release-runtime' + ); + assert.equal(column.length, 12); + assert.ok(column.every((entry) => entry.status === 'inconclusive')); + assert.equal(result.report.result.inconclusive, 12); +}); + +test('workflow uploads the mandatory report before the only enforcement step', () => { + const workflow = fs.readFileSync(WORKFLOW, 'utf8'); + const upload = workflow.indexOf('- name: Upload drift report'); + const evidence = workflow.indexOf('- name: Upload compatibility evidence'); + const enforce = workflow.indexOf('- name: Fail after publishing compatibility results'); + assert.ok(upload > 0 && evidence > upload && enforce > evidence); + const reportStep = workflow.slice(upload, evidence); + assert.match(reportStep, /path: drift-report\.json/); + assert.match(reportStep, /if-no-files-found: error/); + assert.equal( + (workflow.match(/- name: Fail after publishing compatibility results/g) || []) + .length, + 1 + ); +}); diff --git a/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs new file mode 100644 index 00000000000..78e6dc45a68 --- /dev/null +++ b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; + +import { + createPlan, + parseVersion, + releaseVersions, + selectLatestGaAtOrBelow, +} from '../plan-compatibility.mjs'; + +const temporaryDirectories = []; + +function temporaryDirectory() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-plan-')); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('normalizes prerelease and build suffixes', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); + assert.deepEqual(parseVersion('3.8.0+build.42'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); +}); + +test('only exact semantic version tags count as official GA candidates', () => { + const tags = [ + 'a refs/tags/3.7.0', + 'b refs/tags/3.8.0-alpha1', + 'c refs/tags/3.7.1', + 'd refs/tags/v3.8.0', + 'e refs/tags/3.8.0', + ].join('\n'); + assert.deepEqual( + releaseVersions(tags).map((entry) => entry.version), + ['3.7.0', '3.7.1', '3.8.0'] + ); +}); + +test('selects the highest GA at or below the normalized PR target', () => { + const tags = ['3.6.0', '3.7.0', '3.7.2', '3.8.0', '3.9.0'].join('\n'); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.8.0-SNAPSHOT')), + '3.8.0' + ); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.7.5-SNAPSHOT')), + '3.7.2' + ); +}); + +test('plans one compiled and two runtime configurations', () => { + const directory = temporaryDirectory(); + const buildFile = path.join(directory, 'build.gradle'); + fs.writeFileSync( + buildFile, + 'opensearch_version = System.getProperty("opensearch.version", "3.8.0-SNAPSHOT")\n' + ); + const plan = createPlan({ + buildFile, + releaseTags: ['a refs/tags/3.7.0', 'b refs/tags/3.8.0'].join('\n'), + compiledVersion: '2.19.6', + sqlSha: 'sql-sha', + osdRepository: 'example/OpenSearch-Dashboards', + osdRef: 'feature', + }); + + assert.equal(plan.latestEligibleGa, '3.8.0'); + assert.deepEqual( + plan.configurations.map((configuration) => [ + configuration.id, + configuration.surface, + configuration.engineVersion, + ]), + [ + ['2.19.6-compiled', 'compiled-simplified', '2.19.6'], + ['latest-release-runtime', 'runtime-bundle', '3.8.0'], + ['pr-build-runtime', 'runtime-bundle', '3.8.0-SNAPSHOT'], + ] + ); + assert.equal(plan.releasedTargets.include.length, 2); +}); diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs index e8ffa5ad281..42248ce3adc 100644 --- a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -15,6 +15,7 @@ import { assertActiveShippingContracts, buildCensus, buildFrontendExecutionError, + compatibilityExclusion, evaluateFrontendAssertions, selectManifestContractNames, } from '../run-frontend-contract.mjs'; @@ -28,6 +29,54 @@ const RANGE = { endColumn: 14, }; +test('compatibility exclusion uses surface, version, then engine precedence', () => { + const runtimeCalciteRule = { + grammarSurface: 'runtime-bundle', + wiring: { + appliesTo: { minVersion: '3.4.0', engine: 'calcite' }, + }, + }; + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'surface' + ); + assert.equal( + compatibilityExclusion( + { ...runtimeCalciteRule, grammarSurface: 'both' }, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'version' + ); + assert.equal( + compatibilityExclusion( + { + grammarSurface: 'both', + wiring: { appliesTo: { engine: 'calcite' } }, + }, + '3.8.0', + 'runtime-bundle', + 'legacy' + ).reason, + 'engine' + ); + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '3.8.0-SNAPSHOT', + 'runtime-bundle', + 'calcite' + ), + undefined + ); +}); + test('exact lint assertions materialize the effective deterministic edit', () => { const result = evaluateFrontendAssertions({ channel: 'lint', diff --git a/scripts/ppl-lint/aggregate-compatibility.mjs b/scripts/ppl-lint/aggregate-compatibility.mjs new file mode 100644 index 00000000000..e46a872d93f --- /dev/null +++ b/scripts/ppl-lint/aggregate-compatibility.mjs @@ -0,0 +1,961 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + +const ACTIVE_RULE_IDS = [ + 'agg-on-text', + 'division-by-zero', + 'enabled-false-object', + 'field-validation', + 'invalid-capture-group-name', + 'multisearch-min-subsearch', + 'replace-wildcard-asymmetry', + 'rex-scan-cost', + 'type-mismatch-numeric', + 'union-min-datasets', + 'unsupported-window-function-in-eventstats', + 'wildcard-source-zero-match', +]; + +function fatal(message) { + process.stderr.write(`[ppl-lint-compatibility] FATAL: ${message}\n`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { out: 'drift-report.json', summary: '', osdSha: '' }; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[++index]; + if (value === undefined) fatal(`${key} requires a value`); + if (key === '--plan') args.plan = value; + else if (key === '--contracts') args.contracts = value; + else if (key === '--artifacts') args.artifacts = value; + else if (key === '--osd-sha') args.osdSha = value; + else if (key === '--out') args.out = value; + else if (key === '--summary') args.summary = value; + else fatal(`unknown argument ${JSON.stringify(key)}`); + } + for (const field of ['plan', 'contracts', 'artifacts']) { + if (!args[field]) fatal(`--${field} is required`); + } + return args; +} + +function readRequiredJson(file) { + if (!fs.existsSync(file)) fatal(`required JSON file not found: ${file}`); + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + fatal(`could not parse ${file}: ${error.message}`); + } +} + +function readOptionalJson(file) { + if (!fs.existsSync(file)) { + return { value: undefined, error: `missing ${path.basename(file)}` }; + } + try { + return { value: JSON.parse(fs.readFileSync(file, 'utf8')), error: undefined }; + } catch (error) { + return { value: undefined, error: `invalid ${path.basename(file)}: ${error.message}` }; + } +} + +function readOptionalText(file) { + try { + return fs.readFileSync(file, 'utf8').trim(); + } catch { + return ''; + } +} + +function parseVersion(value) { + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value || '')); + return match ? match.slice(1, 4).map((part) => Number(part || 0)) : undefined; +} + +function compareVersion(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] < right[index] ? -1 : 1; + } + return 0; +} + +function inVersionRange(version, minVersion, maxVersion) { + const actual = parseVersion(version); + if (!actual) return false; + const min = parseVersion(minVersion); + const max = parseVersion(maxVersion); + if (min && compareVersion(actual, min) < 0) return false; + if (max && compareVersion(actual, max) > 0) return false; + return true; +} + +function matchesExpectationRange(range, version) { + if (!range || !String(range).trim()) return true; + const actual = parseVersion(version); + if (!actual) return false; + for (const token of String(range).trim().split(/\s+/)) { + const match = /^(>=|<=|>|<|=)?(\d+(?:\.\d+){0,2})$/.exec(token); + if (!match) return false; + const expected = parseVersion(match[2]); + const comparison = compareVersion(actual, expected); + const operator = match[1] || '='; + if ( + !( + (operator === '>=' && comparison >= 0) || + (operator === '<=' && comparison <= 0) || + (operator === '>' && comparison > 0) || + (operator === '<' && comparison < 0) || + (operator === '=' && comparison === 0) + ) + ) { + return false; + } + } + return true; +} + +function validatePlan(plan) { + if (!plan || plan.schemaVersion !== 1 || !Array.isArray(plan.configurations)) { + fatal('compatibility plan must be schemaVersion 1 with a configurations array'); + } + if (plan.configurations.length !== 3) { + fatal(`compatibility plan must contain exactly 3 configurations, found ${plan.configurations.length}`); + } + const ids = new Set(); + for (const configuration of plan.configurations) { + for (const field of [ + 'id', + 'label', + 'engineVersion', + 'surface', + 'executionBackend', + 'engineMode', + 'artifactName', + ]) { + if (typeof configuration[field] !== 'string' || !configuration[field]) { + fatal(`plan configuration ${JSON.stringify(configuration.id)} has invalid ${field}`); + } + } + if (!['compiled-simplified', 'runtime-bundle'].includes(configuration.surface)) { + fatal(`plan configuration ${configuration.id} has unknown surface ${configuration.surface}`); + } + if (!['calcite', 'legacy'].includes(configuration.engineMode)) { + fatal( + `plan configuration ${configuration.id} has unknown engine mode ` + + configuration.engineMode + ); + } + if (ids.has(configuration.id)) fatal(`duplicate plan configuration ${configuration.id}`); + ids.add(configuration.id); + } +} + +function loadContracts(dir) { + const manifestPath = path.join(dir, 'manifest.json'); + const manifest = readRequiredJson(manifestPath); + if (!Array.isArray(manifest.contracts)) fatal(`${manifestPath} contracts must be an array`); + const contracts = new Map(); + for (const file of manifest.contracts) { + const specPath = path.join(dir, file); + const spec = readRequiredJson(specPath); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new Error('expectations must be a non-empty array'); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + } + } catch (error) { + fatal(`invalid ${specPath}: ${error.message}`); + } + if (contracts.has(spec.ruleId)) fatal(`duplicate active rule id ${spec.ruleId}`); + contracts.set(spec.ruleId, { file, spec }); + } + const actual = [...contracts.keys()].sort(); + if (JSON.stringify(actual) !== JSON.stringify(ACTIVE_RULE_IDS)) { + fatal( + `active manifest must contain exactly the approved 12 rules; expected ` + + `${JSON.stringify(ACTIVE_RULE_IDS)}, got ${JSON.stringify(actual)}` + ); + } + return contracts; +} + +function expectedScope(spec, configuration) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const declaredSurface = spec.grammarSurface || 'runtime-bundle'; + const surfaces = + declaredSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [declaredSurface]; + const expected = { + applicable: true, + engine: appliesTo.engine || 'any', + minVersion: appliesTo.minVersion || null, + maxVersion: appliesTo.maxVersion || null, + surfaces, + }; + + if (!surfaces.includes(configuration.surface)) { + return { ...expected, applicable: false, reason: 'surface' }; + } + if ( + !inVersionRange( + configuration.engineVersion, + appliesTo.minVersion, + appliesTo.maxVersion + ) + ) { + return { ...expected, applicable: false, reason: 'version' }; + } + if (appliesTo.engine && appliesTo.engine !== configuration.engineMode) { + return { ...expected, applicable: false, reason: 'engine' }; + } + return expected; +} + +function selectExpectation(spec, configuration) { + const matches = spec.expectations.filter( + (expectation) => + matchesExpectationRange(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); + return matches.length === 1 ? matches[0] : undefined; +} + +function indexDetectorReport(report) { + if (!report || typeof report !== 'object' || !Array.isArray(report.results)) { + throw new Error('detector-report.json must contain a results array'); + } + const rows = new Map(); + for (const row of report.results) { + if (!row || typeof row.ruleId !== 'string' || typeof row.queryName !== 'string') { + throw new Error('detector report rows require ruleId and queryName'); + } + const key = `${row.ruleId}::${row.queryName}`; + if (rows.has(key)) throw new Error(`duplicate detector row ${key}`); + rows.set(key, row); + } + return rows; +} + +function loadEvidence(configuration, artifactsRoot, sqlSha) { + const dir = path.join(artifactsRoot, configuration.artifactName); + const errors = []; + const backendTargetRead = readOptionalJson(path.join(dir, 'target.json')); + const detectorTargetRead = readOptionalJson(path.join(dir, 'detector-target.json')); + const detectorRead = readOptionalJson(path.join(dir, 'detector-report.json')); + const backendRead = readOptionalJson(path.join(dir, 'backend-report.json')); + const bundleRead = + configuration.surface === 'runtime-bundle' + ? readOptionalJson(path.join(dir, 'ppl-grammar-bundle.json')) + : { value: undefined, error: undefined }; + + let backendTarget; + let detectorTarget; + let detectorRows = new Map(); + let backendRows = new Map(); + + for (const item of [backendTargetRead, detectorTargetRead, detectorRead, backendRead, bundleRead]) { + if (item.error) errors.push(item.error); + } + if (backendTargetRead.value) { + try { + backendTarget = normalizeTarget(backendTargetRead.value); + } catch (error) { + errors.push(`invalid target.json: ${error.message}`); + } + } + if (detectorTargetRead.value) { + try { + detectorTarget = normalizeTarget(detectorTargetRead.value); + } catch (error) { + errors.push(`invalid detector-target.json: ${error.message}`); + } + } + if (detectorRead.value) { + try { + detectorRows = indexDetectorReport(detectorRead.value); + } catch (error) { + errors.push(error.message); + } + } + if (backendRead.value && backendTarget) { + try { + backendRows = indexBackendReport(backendRead.value, backendTarget); + } catch (error) { + errors.push(`invalid backend-report.json: ${error.message}`); + } + } + + if (backendTarget) { + if ( + !backendTarget.engineVersion.startsWith( + configuration.engineVersion.replace(/[-+].*$/, '') + ) + ) { + errors.push( + `target engine ${backendTarget.engineVersion} does not match planned ` + + configuration.engineVersion + ); + } + if (backendTarget.executionBackend !== configuration.executionBackend) { + errors.push( + `target backend ${backendTarget.executionBackend} does not match planned ` + + configuration.executionBackend + ); + } + if (sqlSha && backendTarget.sqlSha && backendTarget.sqlSha !== sqlSha) { + errors.push(`target SQL SHA ${backendTarget.sqlSha} does not match planned ${sqlSha}`); + } + } + if (detectorTarget && detectorRead.value) { + for (const field of ['engineVersion', 'grammarHash', 'executionBackend']) { + if (detectorRead.value[field] !== detectorTarget[field]) { + errors.push( + `detector report ${field} ${JSON.stringify(detectorRead.value[field])} does not match ` + + `detector target ${JSON.stringify(detectorTarget[field])}` + ); + } + } + if (detectorRead.value.surface !== configuration.surface) { + errors.push( + `detector surface ${JSON.stringify(detectorRead.value.surface)} does not match planned ` + + configuration.surface + ); + } + } + if ( + bundleRead.value && + detectorTarget && + bundleRead.value.grammarHash !== detectorTarget.grammarHash + ) { + errors.push('runtime grammar bundle hash does not match detector target'); + } + + return { + dir, + errors: [...new Set(errors)], + backendTarget, + detectorTarget, + detectorReport: detectorRead.value, + detectorRows, + backendRows, + backendCommand: readOptionalText(path.join(dir, 'backend-command.txt')), + detectorCommand: readOptionalText(path.join(dir, 'detector-command.txt')), + }; +} + +function detectorActual(row, evidence) { + if (evidence.errors.length > 0 && !evidence.detectorReport) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + if (row.outcome === 'error') { + return { outcome: 'error', error: row.error || 'frontend execution failed' }; + } + if (row.outcome === 'not-applicable' || row.notApplicable) { + return { outcome: 'error', error: row.notApplicable || 'unexpected not-applicable row' }; + } + return { + outcome: 'observed', + count: row.actual, + severities: row.severities || [], + diagnostics: row.diagnostics || [], + }; +} + +function backendActual(row, evidence) { + if (evidence.errors.length > 0 && evidence.backendRows.size === 0) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + const state = classifyBackendReportRow(row); + if (state.status !== 'observed') { + return { + outcome: state.status === 'error' ? 'error' : 'error', + error: row.error || `backend outcome was ${state.status}`, + }; + } + const observed = row.observed || {}; + return { + outcome: 'observed', + rejected: state.rejected, + httpStatus: observed.httpStatus, + errorType: observed.type, + errorReason: observed.reason, + }; +} + +function expectedCase(spec, queryExpectation) { + const resolved = resolveBackendOracle(spec, queryExpectation, 'standard'); + if (resolved.status !== 'applicable') { + throw new Error(resolved.reason || 'standard backend oracle is not applicable'); + } + const expectedBackend = { + kind: resolved.oracle.kind, + httpStatus: resolved.oracle.httpStatus, + }; + const error = resolved.oracle.body && resolved.oracle.body.error; + if (error && error.type) expectedBackend.errorType = error.type; + if (error && error.reason) expectedBackend.errorReason = error.reason; + return { + detector: { + count: resolved.detector.count, + ...(resolved.detector.severity ? { severity: resolved.detector.severity } : {}), + ...(resolved.detector.matchMessage + ? { message: resolved.detector.matchMessage } + : {}), + ...(resolved.detector.messageEquals + ? { message: resolved.detector.messageEquals } + : {}), + }, + backend: expectedBackend, + frontend: resolved.frontend, + }; +} + +function compareDetector(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + if (actual.count !== expected.count) differences.push('detector.count'); + if (expected.severity && row.severityMatched === false) differences.push('detector.severity'); + if (expected.message && row.messageMatched === false) differences.push('detector.message'); + for (const field of [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (row[field] === false) differences.push(`detector.${field}`); + } + for (const [field, matched] of Object.entries(row.assertions || {})) { + if (matched === false && !['count', 'severity'].includes(field)) { + differences.push(`detector.${field}`); + } + } + return [...new Set(differences)]; +} + +function compareBackend(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + const expectedRejected = expected.kind === 'rejection'; + if (actual.rejected !== expectedRejected) differences.push('backend.rejected'); + if ( + Number.isInteger(expected.httpStatus) && + actual.httpStatus !== expected.httpStatus + ) { + differences.push('backend.httpStatus'); + } + if (expected.errorType && actual.errorType !== expected.errorType) { + differences.push('backend.errorType'); + } + if (expected.errorReason && actual.errorReason !== expected.errorReason) { + differences.push('backend.errorReason'); + } + if (row.outcome === 'observed-mismatch' || row.error) { + differences.push('backend.result'); + } + return [...new Set(differences)]; +} + +function aggregateActual(cases, side) { + const actuals = cases.map((entry) => entry.actual[side]); + const errored = actuals.find((actual) => actual.outcome === 'error'); + if (errored) return { outcome: 'error', error: errored.error }; + if (actuals.some((actual) => actual.outcome === 'missing')) return { outcome: 'missing' }; + if (side === 'detector') { + return { + outcome: 'observed', + diagnosticCount: actuals.reduce((sum, actual) => sum + (actual.count || 0), 0), + }; + } + const rejected = actuals.filter((actual) => actual.rejected === true).length; + return { outcome: 'observed', rejected, observedCases: actuals.length }; +} + +function reasonForIncomplete(cases, evidence) { + for (const entry of cases) { + if (entry.actual.detector.outcome !== 'observed') { + return { + code: + entry.actual.detector.outcome === 'missing' + ? 'missing-detector-row' + : 'detector-error', + message: + entry.actual.detector.error || + `No detector result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + if (entry.actual.backend.outcome !== 'observed') { + return { + code: + entry.actual.backend.outcome === 'missing' + ? 'missing-backend-row' + : 'backend-error', + message: + entry.actual.backend.error || + `No backend result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + } + return { + code: 'invalid-leg-identity', + message: evidence.errors.join('; ') || 'The configuration did not produce trustworthy evidence.', + }; +} + +function classifyCell(cases) { + const triggerCases = cases.filter((entry) => entry.role === 'trigger'); + const rejectionTriggers = triggerCases.filter( + (entry) => entry.expected.backend.kind === 'rejection' + ); + const accepted = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === false + ); + const rejected = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === true + ); + const controls = cases.filter((entry) => entry.role !== 'trigger'); + const controlsProveSupport = controls.every( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.differences.every((difference) => !difference.startsWith('backend.')) + ); + const detectorDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('detector.')) + ); + const backendDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('backend.')) + ); + + const triggerSummary = { + contracted: rejectionTriggers.length, + acceptedByBackend: accepted.length, + rejectedByBackend: rejected.length, + missing: rejectionTriggers.filter( + (entry) => entry.actual.backend.outcome !== 'observed' + ).length, + }; + if ( + rejectionTriggers.length > 0 && + accepted.length === rejectionTriggers.length && + controlsProveSupport + ) { + return { classification: 'full-engine-relaxation', triggerSummary }; + } + if (accepted.length > 0 && rejected.length > 0) { + return { classification: 'partial-engine-relaxation', triggerSummary }; + } + if (backendDifferences.length === 0 && detectorDifferences.length > 0) { + return { classification: 'detector-regression', triggerSummary }; + } + if (backendDifferences.length > 0) { + return { classification: 'contract-drift', triggerSummary }; + } + return { classification: undefined, triggerSummary }; +} + +function remediation(classification) { + if (classification === 'detector-regression') { + return { + action: 'update-detector', + scope: 'detector-only', + detail: 'Backend behavior is unchanged; keep appliesTo unchanged.', + }; + } + if (classification === 'full-engine-relaxation') { + return { + action: 'scope-rule-version', + scope: 'appliesTo', + detail: + 'Every contracted trigger is accepted and controls remain supported; stop applying ' + + 'this rule to this version range.', + }; + } + if (classification === 'partial-engine-relaxation') { + return { + action: 'narrow-detector', + scope: 'detector-only', + detail: 'Keep the rule active for this version and narrow it to the forms the backend still rejects.', + }; + } + return { + action: 'update-contract', + scope: 'oracle', + detail: 'Confirm the backend behavior change is intentional before updating the pinned contract.', + }; +} + +function expectedDescription(spec) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const parts = []; + if (appliesTo.engine) { + parts.push(appliesTo.engine === 'calcite' ? 'Calcite' : appliesTo.engine); + } + if (appliesTo.minVersion && appliesTo.maxVersion) { + parts.push(`>= ${appliesTo.minVersion}, <= ${appliesTo.maxVersion}`); + } else if (appliesTo.minVersion) { + parts.push(`>= ${appliesTo.minVersion}`); + } else if (appliesTo.maxVersion) { + parts.push(`<= ${appliesTo.maxVersion}`); + } else { + parts.push('all versions'); + } + parts.push((spec.grammarSurface || 'runtime-bundle') === 'both' ? 'both' : 'runtime only'); + return parts.join('; '); +} + +function markdownCell(row) { + if (row.status === 'n/a') return `n/a (${row.expected.reason})`; + if (row.status === 'drift') return '**drift**'; + if (row.status === 'inconclusive') return '**inconclusive**'; + return 'compatible'; +} + +function renderMarkdown(report, contracts) { + const lines = [ + `## PPL lint compatibility: ${report.result.status.toUpperCase()}`, + '', + `SQL: \`${report.candidate.sqlSha.slice(0, 9)}\` `, + `OSD: \`${report.candidate.osd.repository} @ ${report.candidate.osd.sha || report.candidate.osd.ref}\` `, + `Rules: ${report.inventory.ruleCount} `, + `Configurations: ${report.configurations.length} `, + `Blocking results: ${report.result.drift} drift, ${report.result.inconclusive} inconclusive`, + '', + `| Rule | Expected compatibility | ${report.configurations + .map((configuration) => configuration.label) + .join(' | ')} |`, + `| --- | --- | ${report.configurations.map(() => '---').join(' | ')} |`, + ]; + for (const ruleId of ACTIVE_RULE_IDS) { + const spec = contracts.get(ruleId).spec; + const cells = report.configurations.map((configuration) => + markdownCell( + report.matrix.find( + (entry) => + entry.ruleId === ruleId && entry.configurationId === configuration.id + ) + ) + ); + lines.push( + `| \`${ruleId}\` | ${expectedDescription(spec)} | ${cells.join(' | ')} |` + ); + } + lines.push(''); + + const blocking = report.findings.filter((finding) => finding.blocking); + if (blocking.length > 0) { + lines.push('### Blocking findings', ''); + lines.push('| Rule | Configuration | Classification | Evidence | Action |'); + lines.push('| --- | --- | --- | --- | --- |'); + for (const finding of blocking) { + const evidence = + finding.reason?.message || + `${finding.evidence.triggerSummary.acceptedByBackend}/` + + `${finding.evidence.triggerSummary.contracted} contracted triggers accepted`; + lines.push( + `| \`${finding.ruleId}\` | ${finding.configurationLabel} | ` + + `${finding.classification || 'inconclusive'} | ${evidence.replace(/\|/g, '\\|')} | ` + + `${finding.remediation.detail.replace(/\|/g, '\\|')} |` + ); + } + lines.push(''); + } + lines.push('### Published evidence', ''); + lines.push('| Output | Location |'); + lines.push('| --- | --- |'); + lines.push('| Full table | `Aggregate rule compatibility` step summary |'); + lines.push('| Machine-readable report | `ppl-lint-multiversion-drift/drift-report.json` |'); + lines.push('| Detector logs and target identities | `ppl-lint-multiversion-evidence` |'); + return lines.join('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const plan = readRequiredJson(args.plan); + validatePlan(plan); + const contracts = loadContracts(args.contracts); + const evidenceByConfiguration = new Map( + plan.configurations.map((configuration) => [ + configuration.id, + loadEvidence(configuration, args.artifacts, plan.sqlSha), + ]) + ); + const configurations = plan.configurations.map((configuration) => { + const evidence = evidenceByConfiguration.get(configuration.id); + return { + id: configuration.id, + label: configuration.label, + engineVersion: + (evidence.backendTarget && evidence.backendTarget.engineVersion) || + configuration.engineVersion, + surface: configuration.surface, + executionBackend: configuration.executionBackend, + engineMode: configuration.engineMode, + grammar: { + source: + configuration.surface === 'runtime-bundle' + ? 'engine-runtime-bundle' + : 'osd-compiled', + hash: (evidence.detectorTarget && evidence.detectorTarget.grammarHash) || null, + }, + }; + }); + + const matrix = []; + const cases = []; + const findings = []; + + for (const ruleId of ACTIVE_RULE_IDS) { + const { spec } = contracts.get(ruleId); + for (const configuration of plan.configurations) { + const expected = expectedScope(spec, configuration); + if (!expected.applicable) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'n/a', + expected, + actual: null, + }); + continue; + } + + const evidence = evidenceByConfiguration.get(configuration.id); + const expectation = selectExpectation(spec, configuration); + if (!expectation) { + const reason = { + code: 'missing-contract-expectation', + message: + `No unique expectation covers ${ruleId} on ${configuration.engineVersion} ` + + `(${configuration.engineMode}).`, + }; + const cell = { + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual: { + detector: { outcome: 'missing' }, + backend: { outcome: 'missing' }, + }, + reason, + }; + matrix.push(cell); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + remediation: { + action: 'fix-test-leg', + scope: 'contract', + detail: 'Add or correct the reviewed expectation, then rerun compatibility validation.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const cellCases = []; + for (const [queryName, queryDefinition] of Object.entries(spec.queries || {})) { + const key = `${ruleId}::${queryName}`; + const detectorRow = evidence.detectorRows.get(key); + const backendRow = evidence.backendRows.get(key); + let expectedEvidence; + try { + expectedEvidence = expectedCase(spec, expectation.queries[queryName]); + } catch (error) { + fatal(`invalid ${ruleId}::${queryName} expectation: ${error.message}`); + } + const actual = { + detector: detectorActual(detectorRow, evidence), + backend: backendActual(backendRow, evidence), + }; + const differences = [ + ...compareDetector(expectedEvidence.detector, actual.detector, detectorRow || {}), + ...compareBackend(expectedEvidence.backend, actual.backend, backendRow || {}), + ]; + const entry = { + key: `${configuration.id}::${ruleId}::${queryName}`, + ruleId, + configurationId: configuration.id, + queryName, + role: queryDefinition.role || 'trigger', + query: String(queryDefinition.query || '').split('{{index}}').join(spec.index), + expected: { + detector: expectedEvidence.detector, + backend: expectedEvidence.backend, + }, + actual, + differences, + }; + cases.push(entry); + cellCases.push(entry); + } + + const actual = { + detector: aggregateActual(cellCases, 'detector'), + backend: aggregateActual(cellCases, 'backend'), + }; + const incomplete = + evidence.errors.length > 0 || + cellCases.some( + (entry) => + entry.actual.detector.outcome !== 'observed' || + entry.actual.backend.outcome !== 'observed' + ); + if (incomplete) { + const reason = reasonForIncomplete(cellCases, evidence); + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual, + reason, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + evidence: { caseKeys: cellCases.map((entry) => entry.key) }, + remediation: { + action: 'fix-test-leg', + scope: 'test-leg', + detail: 'Fix or rerun this test leg before recommending a product change.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const classification = classifyCell(cellCases); + if (classification.classification) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'drift', + expected, + actual, + classification: classification.classification, + triggerSummary: classification.triggerSummary, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + classification: classification.classification, + blocking: true, + evidence: { + caseKeys: cellCases.map((entry) => entry.key), + triggerSummary: classification.triggerSummary, + }, + remediation: remediation(classification.classification), + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + } else { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'compatible', + expected, + actual, + caseKeys: cellCases.map((entry) => entry.key), + }); + } + } + } + + const compatible = matrix.filter((entry) => entry.status === 'compatible').length; + const notApplicable = matrix.filter((entry) => entry.status === 'n/a').length; + const drift = matrix.filter((entry) => entry.status === 'drift').length; + const inconclusive = matrix.filter((entry) => entry.status === 'inconclusive').length; + const cellCount = matrix.length; + if ( + cellCount !== ACTIVE_RULE_IDS.length * configurations.length || + compatible + notApplicable + drift + inconclusive !== cellCount + ) { + fatal('internal matrix accounting invariant failed'); + } + + const report = { + schemaVersion: 3, + candidate: { + sqlSha: plan.sqlSha, + osd: { + repository: plan.osd.repository, + ref: plan.osd.ref, + sha: args.osdSha, + }, + }, + inventory: { + ruleCount: ACTIVE_RULE_IDS.length, + ruleIds: ACTIVE_RULE_IDS, + }, + configurations, + matrix, + cases, + findings, + result: { + status: drift + inconclusive === 0 ? 'pass' : 'fail', + cellCount, + compatible, + notApplicable, + drift, + inconclusive, + exitCode: drift + inconclusive === 0 ? 0 : 1, + }, + }; + + fs.writeFileSync(args.out, `${JSON.stringify(report, null, 2)}\n`); + const markdown = renderMarkdown(report, contracts); + process.stdout.write(`${markdown}\n`); + if (args.summary) fs.appendFileSync(args.summary, `${markdown}\n`); + if (report.result.exitCode !== 0) { + process.stderr.write( + `Rule compatibility validation failed after writing the complete report: ` + + `${drift} drift, ${inconclusive} inconclusive.\n` + ); + process.exitCode = report.result.exitCode; + } +} + +main(); diff --git a/scripts/ppl-lint/plan-compatibility.mjs b/scripts/ppl-lint/plan-compatibility.mjs new file mode 100644 index 00000000000..bdd77e06581 --- /dev/null +++ b/scripts/ppl-lint/plan-compatibility.mjs @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +function fail(message) { + throw new Error(message); +} + +export function parseVersion(value) { + const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(String(value || '').trim()); + if (!match) return undefined; + return { + normalized: `${match[1]}.${match[2]}.${match[3]}`, + parts: match.slice(1, 4).map(Number), + }; +} + +function compareVersions(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +export function readPrTarget(buildFile) { + const source = fs.readFileSync(buildFile, 'utf8'); + const match = + /opensearch_version\s*=\s*System\.getProperty\(\s*["']opensearch\.version["']\s*,\s*["']([^"']+)["']\s*\)/.exec( + source + ); + if (!match) { + fail(`could not resolve the default opensearch.version from ${buildFile}`); + } + const parsed = parseVersion(match[1]); + if (!parsed) { + fail(`default opensearch.version ${JSON.stringify(match[1])} is not semantic version X.Y.Z`); + } + return { raw: match[1], normalized: parsed.normalized, parts: parsed.parts }; +} + +export function releaseVersions(text) { + const versions = new Map(); + for (const line of String(text || '').split(/\r?\n/)) { + const refMatch = /refs\/tags\/([^\s^]+)$/.exec(line.trim()); + const candidate = refMatch ? refMatch[1] : line.trim(); + if (!/^\d+\.\d+\.\d+$/.test(candidate)) continue; + const parsed = parseVersion(candidate); + versions.set(parsed.normalized, parsed.parts); + } + return [...versions.entries()] + .map(([version, parts]) => ({ version, parts })) + .sort((left, right) => compareVersions(left.parts, right.parts)); +} + +export function selectLatestGaAtOrBelow(tags, target) { + const eligible = releaseVersions(tags).filter( + (release) => compareVersions(release.parts, target.parts) <= 0 + ); + if (eligible.length === 0) { + fail(`no official GA release tag exists at or below ${target.normalized}`); + } + return eligible.at(-1).version; +} + +function parseArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) fail(`unexpected argument ${JSON.stringify(key)}`); + const value = argv[++index]; + if (value === undefined) fail(`${key} requires a value`); + args[key.slice(2)] = value; + } + for (const required of [ + 'build-file', + 'release-tags', + 'compiled-version', + 'sql-sha', + 'osd-repository', + 'osd-ref', + 'out', + ]) { + if (!args[required]) fail(`--${required} is required`); + } + return args; +} + +export function createPlan({ + buildFile, + releaseTags, + compiledVersion, + sqlSha, + osdRepository, + osdRef, +}) { + const target = readPrTarget(buildFile); + const compiled = parseVersion(compiledVersion); + if (!compiled || compiled.normalized !== compiledVersion) { + fail(`compiled version ${JSON.stringify(compiledVersion)} must be exact semantic version X.Y.Z`); + } + const latestGa = selectLatestGaAtOrBelow(releaseTags, target); + + const configurations = [ + { + id: `${compiledVersion}-compiled`, + label: `${compiledVersion} compiled`, + engineVersion: compiledVersion, + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: `ppl-lint-observation-${compiledVersion}-compiled`, + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: `Latest release (${latestGa}) runtime`, + engineVersion: latestGa, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: target.raw, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, + ]; + + return { + schemaVersion: 1, + sqlSha, + prTargetVersion: target.raw, + normalizedPrTarget: target.normalized, + latestEligibleGa: latestGa, + osd: { + repository: osdRepository, + ref: osdRef, + }, + configurations, + releasedTargets: { + include: configurations.slice(0, 2).map((configuration) => ({ + version: configuration.engineVersion, + configuration_id: configuration.id, + surface: configuration.surface, + label: configuration.id.endsWith('-compiled') ? 'compiled' : 'runtime', + export_runtime_bundle: configuration.exportRuntimeBundle, + artifact_name: configuration.artifactName, + })), + }, + }; +} + +function main() { + try { + const args = parseArgs(process.argv.slice(2)); + const plan = createPlan({ + buildFile: args['build-file'], + releaseTags: fs.readFileSync(args['release-tags'], 'utf8'), + compiledVersion: args['compiled-version'], + sqlSha: args['sql-sha'], + osdRepository: args['osd-repository'], + osdRef: args['osd-ref'], + }); + fs.mkdirSync(path.dirname(path.resolve(args.out)), { recursive: true }); + fs.writeFileSync(args.out, `${JSON.stringify(plan, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(plan)}\n`); + } catch (error) { + process.stderr.write(`[ppl-lint-plan] ${error.message}\n`); + process.exitCode = 2; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 0875f902338..628bd707ff4 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -130,6 +130,31 @@ const SURFACE = (() => { return requested; })(); +const APPLICABLE_ONLY = process.env.PPL_LINT_APPLICABLE_ONLY === '1'; +const ENGINE_MODE = (() => { + const requested = process.env.PPL_LINT_ENGINE_MODE; + if (requested === undefined || requested === '') { + if (APPLICABLE_ONLY) { + // eslint-disable-next-line no-console + console.error( + '[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE is required when ' + + 'PPL_LINT_APPLICABLE_ONLY=1.' + ); + process.exit(2); + } + return undefined; + } + if (!['calcite', 'legacy'].includes(requested)) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE must be "calcite" or "legacy", ` + + `got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + function log(message) { // eslint-disable-next-line no-console console.log(`[ppl-lint-detector-contract] ${message}`); @@ -533,6 +558,37 @@ function versionMatchesRange(range, version) { return true; } +export function compatibilityExclusion(spec, version, surface, engineMode) { + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + return { + reason: 'surface', + detail: `grammarSurface=${contractSurface}, running ${surface}`, + }; + } + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const have = parseVersion(version); + const min = parseVersion(appliesTo.minVersion); + const max = parseVersion(appliesTo.maxVersion); + if ( + have && + ((min && compareVersion(have, min) < 0) || + (max && compareVersion(have, max) > 0)) + ) { + return { + reason: 'version', + detail: `wiring.appliesTo excludes ${version || 'unknown version'}`, + }; + } + if (appliesTo.engine && appliesTo.engine !== engineMode) { + return { + reason: 'engine', + detail: `wiring.appliesTo.engine=${appliesTo.engine}, running ${engineMode}`, + }; + } + return undefined; +} + /** * Select the single expectation that applies to the candidate version + engine. * Exactly one must match (design §5.3): zero means the rule test does not cover @@ -615,10 +671,10 @@ function checkWiring(spec, catalog, getDetector, failures) { * candidate backend version so version filtering matches the backend, and sets * an enable override for default-off rules that declare `forceEnable`. */ -function buildContext(spec, engineVersion) { +function buildContext(spec, engineVersion, engineMode) { const fc = spec.frontendContext || {}; const context = { - isCalcite: fc.isCalcite !== false, + isCalcite: engineMode ? engineMode === 'calcite' : fc.isCalcite !== false, dataSourceVersion: engineVersion || undefined, // Pin the "latest verified engine" to the candidate version rather than the // hardcoded OSD_KNOWN_VERSION ('3.7.0'), which can mis-filter rules near a @@ -1131,6 +1187,19 @@ function main() { const index = spec.index; const channel = contractChannel(spec); const scoringFailures = reportOnly ? reportOnlyFailures : failures; + if (APPLICABLE_ONLY) { + const exclusion = compatibilityExclusion(spec, engineVersion, surface, ENGINE_MODE); + if (exclusion) { + log(`SKIP ${ruleId} (${exclusion.detail}) — ${path.basename(file)}`); + if (exclusion.reason === 'surface') { + skippedForSurface.push({ + ruleId, + contractSurface: spec.grammarSurface || 'runtime-bundle', + }); + } + continue; + } + } const entry = checkWiring(spec, catalog, getDetector, scoringFailures); if (!entry) { for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { @@ -1187,7 +1256,7 @@ function main() { continue; } - const context = buildContext(spec, engineVersion); + const context = buildContext(spec, engineVersion, ENGINE_MODE); const expectation = selectExpectation(spec, engineVersion, context.isCalcite, scoringFailures, { allowMissing: observeOnly, }); From f09ce716c3d4569b75baed5eb93239bd3c5232b1 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 20:37:17 -0700 Subject: [PATCH 77/78] chore(ci): label PPL workflows as linter checks Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-multiversion-validation.yml | 2 +- .github/workflows/ppl-lint-rule-validation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f68f53993d0..4ced31d611a 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -1,4 +1,4 @@ -name: PPL lint multi-surface compatibility +name: "[Linter] PPL multi-surface compatibility" # Validate the 12 active PPL lint rules on the checked-in OSD fallback grammar # and on runtime grammar bundles exported by the latest eligible GA engine and diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index c218b7fbda3..d68aaf4158d 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,4 +1,4 @@ -name: PPL lint rule validation +name: "[Linter] PPL rule validation" permissions: contents: read From 4381c284fbff50b5d1cd6f4d275f0eebfe9a7b15 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 4 Aug 2026 21:26:49 -0700 Subject: [PATCH 78/78] docs(ci): align PPL compatibility design with workflow Signed-off-by: Hanyu Wei --- ...pl-lint-runtime-compatibility-ci-design.md | 208 +++++++++--------- 1 file changed, 110 insertions(+), 98 deletions(-) diff --git a/docs/dev/ppl-lint-runtime-compatibility-ci-design.md b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md index eb0c7242fbe..e2b3375f25b 100644 --- a/docs/dev/ppl-lint-runtime-compatibility-ci-design.md +++ b/docs/dev/ppl-lint-runtime-compatibility-ci-design.md @@ -1,132 +1,146 @@ # PPL Lint Runtime Compatibility CI -- **Status:** Proposed revision for the SQL PPL lint CI +- **Status:** Draft implementation design - **Last updated:** 2026-08-04 - **Scope:** `.github/workflows/ppl-lint-multiversion-validation.yml` ## 1. Decision -The multi-version workflow validates PPL lint compatibility only against -standard OpenSearch runtime grammar bundles: +The multi-surface workflow validates the 12 active PPL lint detectors against +exactly three standard-engine configurations: ```text -OpenSearch 3.6 release ─┐ -OpenSearch 3.7 release ─┼─> Aggregate rule compatibility -SQL pull request build ─┘ +OpenSearch 2.19.6 + OSD compiled-simplified fallback grammar --+ +Latest eligible GA + its exported runtime grammar -------------+--> Aggregate rule compatibility +SQL pull request build + its exported runtime grammar ----------+ ``` +The fixed `2.19.6` leg covers the checked-in grammar OSD uses when an engine +cannot export a runtime grammar bundle. The planner reads the SQL pull request's +default `opensearch.version`, normalizes prerelease/build suffixes, and selects +the highest exact-semver OpenSearch release tag at or below that target for the +GA runtime leg. The PR leg validates the candidate runtime grammar built by the +change under review. + The workflow does not run: -- the compiled-simplified grammar surface for pre-3.6 engines; - the analytics engine or composite/Parquet storage; - syntax-channel features; - AI action tests. Analytics coverage is deferred until that engine and its fixtures provide a -stable CI contract. Pre-3.6 coverage is removed because those engines cannot -export the runtime grammar bundle consumed by the production lint path. - -The required single-version workflow remains responsible for proving that all -active shipping detectors agree with the standard SQL pull request build. The -multi-version workflow explains where each rule works and fails its final -aggregation job when a declared-supported version drifts. +stable CI contract. The required single-version workflow remains responsible +for proving that all active detectors agree with the standard SQL pull request +build. The multi-surface workflow explains compatibility across shipping grammar +surfaces and fails its final aggregation job when declared support drifts. ## 2. Rule Inventory -The active inventory currently contains **12 detector rules**, not 13. -`command-suggestion` was removed from this effort and must not be silently -reintroduced as a lint rule. The final table is generated from -`manifest.json`. CI also asserts that the current inventory is exactly these 12 -rules, so adding a future reviewed rule requires an intentional guard and test -update. - -| Rule | Declared compatibility | -| --- | --- | -| `agg-on-text` | Calcite, OpenSearch >= 3.7 | -| `division-by-zero` | All runtime-bundle versions | -| `enabled-false-object` | Calcite, OpenSearch >= 3.7 | -| `field-validation` | All runtime-bundle versions | -| `invalid-capture-group-name` | OpenSearch >= 3.4 | -| `multisearch-min-subsearch` | OpenSearch >= 3.4 | -| `replace-wildcard-asymmetry` | Calcite, OpenSearch >= 3.4 | -| `rex-scan-cost` | All runtime-bundle versions | -| `type-mismatch-numeric` | Calcite, OpenSearch >= 3.7 | -| `union-min-datasets` | Calcite, OpenSearch >= 3.7 | -| `unsupported-window-function-in-eventstats` | OpenSearch >= 3.4 | -| `wildcard-source-zero-match` | All runtime-bundle versions | +The active inventory contains **12 detector rules**, not 13. +`command-suggestion` is not a lint rule and must not be silently reintroduced. +The final table is generated from `manifest.json`, and CI asserts the exact +inventory so adding a future reviewed rule requires an intentional guard and +test update. + +| Rule | Grammar surface | Declared scope | +| --- | --- | --- | +| `agg-on-text` | Both | Calcite, OpenSearch >= 3.7 | +| `division-by-zero` | Both | All versions and engine modes | +| `enabled-false-object` | Both | Calcite, OpenSearch >= 3.7 | +| `field-validation` | Both | All versions and engine modes | +| `invalid-capture-group-name` | Runtime bundle | OpenSearch >= 3.4 | +| `multisearch-min-subsearch` | Runtime bundle | OpenSearch >= 3.4 | +| `replace-wildcard-asymmetry` | Runtime bundle | Calcite, OpenSearch >= 3.4 | +| `rex-scan-cost` | Both | All versions and engine modes | +| `type-mismatch-numeric` | Both | Calcite, OpenSearch >= 3.7 | +| `union-min-datasets` | Runtime bundle | Calcite, OpenSearch >= 3.7 | +| `unsupported-window-function-in-eventstats` | Both | OpenSearch >= 3.4 | +| `wildcard-source-zero-match` | Both | All versions and engine modes | + +Four preserved default-off contracts remain in `dormantContracts`. They do not +count toward the 12-rule active inventory. ## 3. Workflow Shape -### 3.1 Plan +### 3.1 Plan configurations -`Plan matrix` resolves: +`Plan compatibility matrix` resolves: -- released engines: `3.6.0` and `3.7.0`; +- the fixed compiled-fallback target, `2.19.6`; +- the highest official GA release at or below the normalized PR target; +- the raw and normalized PR target from `build.gradle`; - the OSD repository and revision; -- the discovery engine, currently the newest released engine. +- immutable configuration IDs, surfaces, engine modes, and artifact names. -There is no compiled-surface input or analytics target. +Only exact `X.Y.Z` release tags are eligible. The plan is uploaded as +`compatibility-plan.json` and drives the released-engine matrix. -### 3.2 Observe released engines +### 3.2 Observe released configurations -One `Observe engine ` job runs per released engine. Each job: +One `Observe engine ()` matrix job runs for each released +configuration: -1. starts the official OpenSearch distribution containing its matching SQL +1. start the matching official OpenSearch distribution, which includes its SQL plugin; -2. runs the contract queries in observe-only mode; -3. exports that engine's runtime grammar bundle; -4. uploads `target.json`, `backend-report.json`, and - `ppl-grammar-bundle.json`. +2. run the same contract corpus in observe-only mode; +3. record `target.json` and `backend-report.json`; +4. export `ppl-grammar-bundle.json` only for the runtime-bundle configuration; +5. upload the observation even when a semantic mismatch is found. -An expectation mismatch is observation data, not a job failure. +The `2.19.6` configuration records backend behavior but intentionally has no +runtime bundle. Its detector pass uses OSD's compiled-simplified fallback +grammar. The selected GA configuration exports and uses that release's runtime +bundle. ### 3.3 Observe the pull request build -`Observe engine pr-build` runs the same corpus against the standard Gradle test -cluster built from the pull request. It exports the same artifact shape as the -released legs. +`Observe engine pr-build (runtime)` runs the same corpus against the standard +Gradle test cluster built from the pull request. It exports the candidate +runtime grammar and the same target/backend artifact shape as the GA runtime +leg. ### 3.4 Aggregate rule compatibility `Aggregate rule compatibility` is the only fan-in job. It: -1. waits for the released and pull request observation jobs; -2. downloads every `ppl-lint-leg-*` artifact; -3. bootstraps OSD once; -4. runs the production headless lint detector against each engine's runtime - grammar bundle; -5. compares declared compatibility with observed detector and backend results; -6. writes `drift-report.json`; -7. publishes the Markdown compatibility table in the GitHub step summary; -8. uploads the mandatory `ppl-lint-multiversion-drift` artifact and - supplemental `ppl-lint-multiversion-evidence` artifact; -9. fails if the aggregate result recorded supported-version drift. - -The job display name is intentionally explicit. A reader should not have to -infer that a job named "detect" is the final aggregation. +1. waits for the plan and all three backend observations; +2. downloads every `ppl-lint-observation-*` artifact; +3. bootstraps OSD once at the resolved revision; +4. runs production headless lint against the compiled fallback or each runtime + bundle, as specified by the plan; +5. applies surface, version, then engine-mode exclusions before detector + execution; +6. compares declared compatibility with detector and backend evidence; +7. writes the complete 12 x 3 `drift-report.json`; +8. publishes the Markdown compatibility table and file-aware annotations; +9. uploads the mandatory report and supplemental evidence before enforcement; +10. fails if the recorded result contains supported-configuration drift or an + enforced inconclusive cell. + +The display name is intentionally explicit. A reader should not have to infer +that this fan-in is the final compatibility decision. ## 4. Expected Versus Actual Compatibility The aggregate summary has one row per active rule: -| Rule | Expected compatibility | 3.6 actual | 3.7 actual | PR build actual | +| Rule | Expected compatibility | 2.19.6 compiled | Latest GA runtime | PR runtime | | --- | --- | --- | --- | --- | -| `agg-on-text` | Calcite, >= 3.7 | expected n/a | compatible | compatible | -| `division-by-zero` | all versions | compatible | compatible | compatible | +| `agg-on-text` | Both surfaces, Calcite >= 3.7 | expected n/a | compatible | compatible | +| `division-by-zero` | Both surfaces, all versions | compatible | compatible | compatible | Each actual cell uses one of these states: | State | Meaning | | --- | --- | | `compatible` | Detector output and backend behavior match the contract. | -| `expected n/a` | The engine is outside `wiring.appliesTo`, such as 3.6 for a rule with `minVersion: 3.7.0`. | -| `drift` | The engine is declared compatible but detector or backend behavior differs. | -| `inconclusive` | A fixture, query, or detector execution did not produce a trustworthy verdict. | +| `expected n/a` | Surface, version, or engine mode is outside `wiring.appliesTo`. | +| `drift` | The configuration is declared compatible but observed behavior differs. | +| `inconclusive` | A fixture, query, artifact, or detector execution did not produce a trustworthy verdict. | -`minVersion` is part of the expected result, not a workaround applied after -the fact. If a rule is intentionally unsupported on 3.6 and declares -`minVersion: 3.7.0`, the 3.6 cell is `expected n/a` and does not count as -drift. +Applicability is part of the expected result, not a workaround applied after +observation. For example, a Calcite-only rule is expected n/a on the legacy +compiled configuration and does not count as drift there. The JSON report retains query-level evidence and remediation details. The Markdown table is the concise compatibility view, not a replacement for the @@ -137,42 +151,41 @@ machine-readable report. Compatibility aggregation is write-first and then enforcing: - observation jobs record detector and backend mismatches without failing; -- expected out-of-scope versions do not fail the workflow; -- an inconclusive rule produces an `inconclusive` cell and annotation; -- one rule cannot prevent results for the other rules; +- expected out-of-scope configurations do not fail the workflow; +- one rule cannot prevent results for the remaining rules; +- detector execution errors become complete inconclusive cells; - the fan-in writes the complete table and `drift-report.json`; -- the artifact upload runs even when the aggregate result is failing; -- only after those outputs exist does supported-version drift or an enforced - inconclusive result fail the final aggregation job. - -This ordering is required. A bare `Process completed with exit code 1` before -the table exists is not an actionable compatibility result. +- artifact upload runs before the enforcement step; +- only after those outputs exist does supported drift or an enforced + inconclusive result fail the job. Structural failures remain errors because no truthful table can be produced: -- a planned engine leg uploads no artifacts; +- a planned observation uploads no usable artifacts; - JSON artifacts are malformed; -- target identity conflicts with report identity; +- target and report identities conflict; - the contract manifest is malformed; +- a runtime configuration has no grammar bundle; - `drift-report.json` cannot be written. -The artifact upload must require `drift-report.json`. An artifact named -`ppl-lint-multiversion-drift` that contains only raw target files is not a drift -report and must not be presented as one. +An artifact named `ppl-lint-multiversion-drift` must contain +`drift-report.json`; raw target files alone are not a compatibility report. ## 6. Outputs Every run produces: +- `compatibility-plan.json`; - a GitHub step-summary table with expected and actual compatibility; - `drift-report.json`; -- one detector report and detector log per engine leg; -- target manifests that identify the exact engine and grammar hash. +- one detector report and detector log per configuration; +- target manifests that identify exact SQL, engine, surface, backend, and + grammar identities. -The required PPL lint workflow remains the stable branch-protection signal. -The multi-version aggregation job is also red on declared-supported drift, with -the table and artifact serving as the evidence for adjusting `minVersion`, -narrowing a detector, or updating a backend oracle after review. +The required PPL lint workflow remains the stable branch-protection signal. The +multi-surface aggregation is also red on declared-supported drift, with the +table and artifacts providing evidence for adjusting applicability, narrowing a +detector, or updating a backend oracle after review. ## 7. Deferred Coverage @@ -183,6 +196,5 @@ Analytics-engine validation may return only after: - route attestation is stable; - a rule-specific analytics limitation cannot invalidate unrelated rules. -Compiled-simplified coverage may return only if pre-3.6 support becomes a -shipping requirement. It must be a separate workflow because it tests OSD's -checked-in grammar rather than the SQL runtime grammar bundle. +Syntax-channel and AI-action behavior require separate contracts and are not +part of this detector compatibility matrix.