diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml new file mode 100644 index 00000000000..543139e3ebb --- /dev/null +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -0,0 +1,374 @@ +name: PPL lint rule validation + +# Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and +# the SQL backend must agree on the SAME candidate runtime grammar. A shared, +# reviewed corpus of contract files pins each rule's OSD detector diagnostic +# count to the live SQL engine's behavior, so neither side can drift unilaterally +# without a red build. +# +# The workflow is a linear three-job pipeline (design §3.1): +# +# backend-validation ──(artifacts)──▶ detector-validation ──▶ validation-result +# +# 1. backend-validation (Amazon Linux CI container): builds the SQL PR, starts the +# Gradle test cluster, runs the contract trigger/control queries against the +# live /_plugins/_ppl endpoint, and — while the cluster is alive — exports the +# candidate runtime grammar bundle (GET /_plugins/_ppl/_grammar) plus a target +# manifest and the observed backend report. Those three files are the ONLY +# bridge to the next job; the test cluster is never passed between jobs. +# 2. detector-validation (ubuntu-latest): checks out and bootstraps OSD as a Node +# code dependency (no OSD server, no Monaco, no browser), deserializes the +# candidate bundle through OSD's production headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. OSD needs a newer Node/glibc than the CI container provides, +# hence a separate Ubuntu job. +# 3. validation-result: the single stable required check. Fails unless BOTH +# validation jobs succeeded (an always() result job so a skipped detector +# cannot mask a backend failure), writes the compact per-rule PR summary, and +# uploads the run manifest recording the exact SQL SHA, OSD SHA, mode, backend +# version, and grammar hash. +# +# Modes (design §3.4, §4.1.1): +# - pull_request: SQL PR validation against the resolved OSD target. The ONLY +# enforcing mode; this is what branch protection pins to. Runs the fast +# schedule:pr subset. The committed default is `main` on the canonical repo; +# it can be overridden by the OSD_REPO/OSD_REF repo variables — see the +# "Resolve OSD ref" step. TEMPORARY: those repo variables are currently set to +# the unmerged paired OSD branch that ships the headless lint API this job +# needs; deleting them reverts to opensearch-project/...@main once that OSD PR +# merges. +# - workflow_dispatch (osd_repo + osd_ref): pre-merge evidence for an unmerged +# OSD branch, optionally on a fork (osd_repo). Records the resolved immutable +# OSD commit SHA but CANNOT satisfy branch protection — only the pull_request +# run does. +# - schedule (nightly): the full corpus + a coverage assertion. +# +# Workflow shape deliberately mirrors the sibling SQL Java workflows so a +# maintainer sees one pattern, not a bespoke one: +# - sql-test-and-build-workflow.yml : the Get-CI-Image-Tag reusable workflow, +# the OpenSearch CI container + ci-image-start-command, and the +# `chown 1000:1000` + `su` non-root Gradle invocation (backend-validation). +# - integ-tests-with-security.yml : the report-upload-on-always() shape. +# Action SHAs are pinned to the same versions those siblings use, so dependabot +# bumps one set, not two drifting ones. +# +# CI cost (measured 2026-07-22, ~13 min wall clock): backend-validation ~5 min +# (container init ~2 min + backend IT/export ~2m50s); detector-validation ~3 min, +# of which OSD `yarn osd bootstrap` is ~2m13s and the actual lint is ~2s. The +# bootstrap dominates and is CPU-bound (it was ~2m13s even with a warm yarn +# cache), so it is NOT sharded into a per-contract matrix (that would multiply +# the 2m13s, not the 2s). Overlapping bootstrap with the backend job is a tracked +# follow-up, not done here: it would require transferring the bootstrapped OSD +# tree (multi-GB, 30+ workspace symlinks, plus built target/) between runners, +# which OSD's own CI deliberately avoids. See ~/ppl-lint-ci-fixes-impl-plan.md. + +on: + pull_request: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository to check out (a fork, for pre-merge evidence). Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string + osd_ref: + description: OSD commit or branch to validate instead of main (pre-merge evidence only) + required: false + type: string + schedule: + description: Contract schedule to run (pr or nightly) + required: false + default: pr + type: string + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + backend-validation: + name: Backend validation (live /_plugins/_ppl + grammar export) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # OpenSearch refuses to start as root, so run Gradle as a non-root user. The + # IT exports the candidate grammar bundle + target manifest while the cluster + # is alive; those become the artifacts the detector job lints against. + - name: Run backend integration test and export candidate grammar + run: | + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.report=$(pwd)/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/target.json" + + - name: Upload backend artifacts (bundle + target + report) + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: | + backend-report.json + ppl-grammar-bundle.json + target.json + + - name: Upload backend failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-backend-logs + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + + detector-validation: + name: Detector validation (OSD headless lint on candidate bundle) + needs: backend-validation + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + osd_repo: ${{ steps.osd-ref.outputs.repo }} + osd_ref: ${{ steps.osd-ref.outputs.ref }} + osd_sha: ${{ steps.osd-rev.outputs.sha }} + schedule: ${{ steps.schedule.outputs.value }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + # Resolve which OSD checkout the detectors run against, in precedence order: + # 1. workflow_dispatch input (osd_repo / osd_ref) — explicit manual run + # 2. repo variable (vars.OSD_REPO / vars.OSD_REF) — the override + # point; set/cleared in repo settings with no workflow edit + # 3. canonical default opensearch-project/OpenSearch-Dashboards@main + # + # The committed default is intentionally the canonical repo + `main`, so the + # file always declares that the required check validates against upstream. + # TEMPORARY OVERRIDE: the headless lint API this job imports + # (src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint) is not yet + # on OSD `main`; it lives on the paired branch + # Hanyu-W/OpenSearch-Dashboards@ppl-lint-headless-api. Until that OSD PR + # merges, the OSD_REPO/OSD_REF repo variables are set to that branch so the + # required check validates against the OSD ref that actually ships the API. + # Deleting those two repo variables (no code change) reverts to `main`. + - name: Resolve OSD ref + id: osd-ref + env: + REQUESTED_REF: ${{ inputs.osd_ref }} + REQUESTED_REPO: ${{ inputs.osd_repo }} + VAR_REF: ${{ vars.OSD_REF }} + VAR_REPO: ${{ vars.OSD_REPO }} + run: | + ref="${REQUESTED_REF:-${VAR_REF:-main}}" + repo="${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "repo=$repo" >> "$GITHUB_OUTPUT" + + - name: Resolve contract schedule + id: schedule + env: + REQUESTED_SCHEDULE: ${{ inputs.schedule }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ -n "$REQUESTED_SCHEDULE" ]; then + value="$REQUESTED_SCHEDULE" + elif [ "$EVENT_NAME" = "schedule" ]; then + value="nightly" + else + value="pr" + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ steps.osd-ref.outputs.repo }} + ref: ${{ steps.osd-ref.outputs.ref }} + path: .ci/OpenSearch-Dashboards + + # Resolve the (possibly mutable) ref to the immutable commit SHA actually + # tested, so the run manifest pins exactly what ran (design §4.1.1, T11). + - name: Record OSD revision + id: osd-rev + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (repo: ${{ steps.osd-ref.outputs.repo }}, ref: ${{ steps.osd-ref.outputs.ref }})" >> "$GITHUB_STEP_SUMMARY" + + # Read the Node/Yarn toolchain from the OSD checkout rather than hardcoding + # it, so an OSD toolchain bump does not silently drift this job. + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + # Take the lower bound of the engines.yarn range (e.g. "^1.22.10" -> "1.22.10"). + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + # Retry-with-backoff mirrors the OSD build workflow's bootstrap step; + # `yarn osd bootstrap` occasionally fails on a transient registry hiccup. + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # Downloaded after bootstrap (not before): the ~2m13s bootstrap does not + # need the backend artifact — only the lint step below does — so a flaky + # artifact download cannot waste a completed bootstrap. + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-backend + path: artifacts + + - name: Run detector validation against the candidate bundle + working-directory: .ci/OpenSearch-Dashboards + env: + PPL_LINT_CONTRACT_DIR: ${{ github.workspace }}/integ-test/src/test/resources/ppl-lint/contracts + PPL_LINT_SCHEDULE: ${{ steps.schedule.outputs.value }} + PPL_LINT_GRAMMAR_BUNDLE: ${{ github.workspace }}/artifacts/ppl-grammar-bundle.json + PPL_LINT_TARGET_MANIFEST: ${{ github.workspace }}/artifacts/target.json + PPL_LINT_BACKEND_REPORT: ${{ github.workspace }}/artifacts/backend-report.json + PPL_LINT_REPORT: ${{ github.workspace }}/detector-report.json + run: | + # pipefail so the runner's non-zero exit propagates through `tee` — + # otherwise the pipeline takes tee's (success) status and a real + # detector failure would go green (a vacuous pass). + set -o pipefail + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + | tee "$GITHUB_WORKSPACE/detector-contract.log" + + - name: Upload detector report and corpus + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-detector + path: | + detector-contract.log + detector-report.json + integ-test/src/test/resources/ppl-lint/contracts + + validation-result: + name: validation-result + if: ${{ always() }} + needs: + - backend-validation + - detector-validation + runs-on: ubuntu-latest + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download backend artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true + with: + name: ppl-lint-backend + path: artifacts + + - name: Download detector artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + continue-on-error: true + with: + name: ppl-lint-detector + path: artifacts + + # Assemble the run manifest and the compact per-rule PR summary from the + # reports both jobs uploaded. The manifest records the immutable SQL + OSD + # SHAs so any run is exactly reproducible (design §3.3, §4.4). + - name: Assemble run manifest and summary + env: + SQL_SHA: ${{ github.sha }} + OSD_REPO: ${{ needs.detector-validation.outputs.osd_repo }} + OSD_REF: ${{ needs.detector-validation.outputs.osd_ref }} + OSD_SHA: ${{ needs.detector-validation.outputs.osd_sha }} + EVENT_NAME: ${{ github.event_name }} + SCHEDULE: ${{ needs.detector-validation.outputs.schedule }} + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: node "$GITHUB_WORKSPACE/scripts/ppl-lint/assemble-run-manifest.mjs" + + - name: Upload run manifest + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-run-manifest + path: | + run-manifest.json + + # The sole branch-protection check: red unless BOTH validation jobs + # succeeded. Because this job runs with always(), a skipped detector job + # (e.g. backend failed first) still reds the result instead of appearing + # green (design §4.4). A workflow_dispatch run is pre-merge evidence and is + # intentionally not what repo admins pin to branch protection. + - name: Require both validation jobs to have succeeded + env: + BACKEND_RESULT: ${{ needs.backend-validation.result }} + DETECTOR_RESULT: ${{ needs.detector-validation.result }} + run: | + echo "backend-validation: $BACKEND_RESULT" + echo "detector-validation: $DETECTOR_RESULT" + if [ "$BACKEND_RESULT" != "success" ] || [ "$DETECTOR_RESULT" != "success" ]; then + echo "::error::PPL lint rule validation failed (backend=$BACKEND_RESULT detector=$DETECTOR_RESULT)." + exit 1 + fi + echo "PPL lint rule validation passed: backend and detector agree on the candidate grammar." diff --git a/.gitignore b/.gitignore index bf9002f999d..00db1869c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,12 @@ http-client.env.json !.claude/harness/ .claude/settings.local.json .clinerules -memory-bank \ No newline at end of file +memory-bank +# PPL lint rule validation contract run artifacts (uploaded in CI, not committed) +backend-report.json +backend-report-nightly.json +detector-report.json +detector-contract.log +ppl-grammar-bundle.json +target.json +run-manifest.json diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c18fa6e37f6..1d8f3af45f4 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -168,6 +168,21 @@ tasks.withType(licenseHeaders.class) { additionalLicense 'AL ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")' } +// Forward the PPL lint rule validation contract knobs to every integ test JVM +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), an +// optional path to write the observed backend report, and — while the cluster is +// alive — optional paths to export the candidate runtime grammar bundle and its +// target manifest for the detector-validation job. Applied globally so every +// RestIntegTestTask that runs the class picks it up without per-task edits. +tasks.withType(Test).configureEach { + systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") + ["ppl.lint.report", "ppl.lint.grammar.bundle", "ppl.lint.target"].each { prop -> + if (System.getProperty(prop) != null) { + systemProperty prop, System.getProperty(prop) + } + } +} + validateNebulaPom.enabled = false loggerUsageCheck.enabled = false diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java new file mode 100644 index 00000000000..4185511ab04 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -0,0 +1,723 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Backend half of the schema-v3 PPL lint rule validation contract. + * + *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from + * the current checkout. For every contract (see {@code + * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} + * entry that matches the candidate backend version (exactly one must match, or the contract fails + * before any query runs), applies the contract's cluster settings, and asserts, per query's {@code + * backend.kind}: + * + *

+ * + *

The contract files are shared verbatim with the SQL-owned OSD detector runner ({@code + * scripts/ppl-lint/run-frontend-contract.mjs}) so the same reviewed cases pin both the OSD analyzer + * diagnostic count and the SQL backend behavior; neither side can drift without a red build. The + * rejection-body parsing mirrors {@link + * org.opensearch.sql.calcite.remote.CalciteErrorReportStageIT}; the Calcite setup follows {@link + * org.opensearch.sql.calcite.remote.CalcitePPLEventstatsIT}. + * + *

While the ephemeral cluster is alive, the test also exports the candidate runtime grammar + * bundle it built ({@code GET /_plugins/_ppl/_grammar}) and a small target manifest pairing the + * bundle with the backend version and grammar hash. These become workflow artifacts that the + * detector-validation job injects into OSD's headless lint API, so both halves validate against the + * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code + * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. + * + *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): PR runs only the + * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus. + */ +public class PplLintRuleValidationIT extends PPLIntegTestCase { + + private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; + private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; + private static final String GRAMMAR_API_ENDPOINT = "/_plugins/_ppl/_grammar"; + + /** Which contracts to run this session; PR is the fast blocking subset. */ + private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + + private int[] clusterVersion; + private String engineVersionRaw; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + // Seed the union of every index every scheduled contract needs, once. + for (String indexEnum : requiredIndexEnums()) { + loadIndex(Index.valueOf(indexEnum)); + } + clusterVersion = fetchClusterVersion(); + } + + @Test + public void testValidatesLintRuleContracts() throws IOException { + List contracts = loadScheduledContracts(); + List failures = new ArrayList<>(); + JSONArray report = new JSONArray(); + + // Export the candidate grammar bundle + target manifest while the cluster is + // alive. Runs before the contract loop so the artifacts are emitted even if a + // contract later fails. + exportGrammarArtifacts(failures); + + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + runContract(contract, ruleId, failures, report); + } + + writeReport(report); + + if (!failures.isEmpty()) { + fail( + "PPL lint backend contract failures (" + + failures.size() + + "):\n- " + + String.join("\n- ", failures)); + } + } + + private void runContract( + JSONObject contract, String ruleId, List failures, JSONArray report) + throws IOException { + String index = contract.getString("index"); + JSONObject queries = contract.getJSONObject("queries"); + JSONArray expectations = contract.getJSONArray("expectations"); + JSONObject fixture = contract.optJSONObject("backendFixture"); + boolean calciteOn = fixtureCalciteEnabled(fixture); + + List applied = applyClusterSettings(fixture); + try { + JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); + if (selected == null) { + return; // no/ambiguous version expectation — failure already recorded. + } + JSONObject expectedQueries = selected.getJSONObject("queries"); + for (String queryName : expectedQueries.keySet()) { + if (!queries.has(queryName)) { + failures.add( + "[" + + ruleId + + "] expectation references unknown query \"" + + queryName + + "\" (not in the top-level queries map)"); + continue; + } + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = expectedQueries.getJSONObject(queryName); + JSONObject backend = expected.getJSONObject("backend"); + String kind = backend.getString("kind"); + + JSONObject entry = reportEntry(ruleId, queryName, role, query, kind); + try { + verifyCase(kind, queryName, query, backend, entry); + entry.put("outcome", "pass"); + log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); + } catch (AssertionError | RuntimeException e) { + entry.put("outcome", "fail").put("error", String.valueOf(e.getMessage())); + failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); + log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + } + report.put(entry); + } + } finally { + resetClusterSettings(applied); + } + } + + /** + * Select the single {@code expectations[]} entry that applies to the candidate backend version + * and engine. Exactly one must match: zero means the rule test does not cover this version + * (design §9), and more than one means overlapping ranges — both fail before execution (§5.3). + */ + private JSONObject selectExpectation( + String ruleId, JSONArray expectations, boolean calciteOn, List failures) { + List matches = new ArrayList<>(); + for (int i = 0; i < expectations.length(); i++) { + JSONObject exp = expectations.getJSONObject(i); + if (!versionMatchesRange(exp.optString("version", null))) { + continue; + } + String engine = exp.optString("engine", ""); + if ("calcite".equals(engine) && !calciteOn) { + continue; + } + matches.add(exp); + } + String versionLabel = engineVersionRaw == null ? "unknown" : engineVersionRaw; + if (matches.size() == 1) { + return matches.get(0); + } + if (matches.isEmpty()) { + failures.add( + "[" + ruleId + "] no version expectation matches backend version " + versionLabel); + } else { + failures.add( + "[" + + ruleId + + "] " + + matches.size() + + " expectations match backend version " + + versionLabel + + " (exactly one required)"); + } + return null; + } + + private void verifyCase( + String kind, String queryName, String query, JSONObject backend, JSONObject entry) + throws IOException { + BackendObservation obs = observeBackend(query); + entry.put("rejected", obs.rejected); + entry.put("observed", obs.toJson()); + switch (kind) { + case "rejection": + assertRejection( + queryName, query, obs, backend.getInt("httpStatus"), backend.getJSONObject("body")); + break; + case "result-shape": + assertResultShape(queryName, query, obs, backend.optJSONObject("expect")); + break; + case "advisory": + assertAdvisory(queryName, query, obs); + break; + default: + throw new IllegalArgumentException( + "case \"" + queryName + "\": unknown backend.kind \"" + kind + "\""); + } + } + + /** + * Run the query once and categorize the observed backend behavior independently of the + * expectation, so the report carries the true behavior even when a case fails (e.g. a trigger the + * backend unexpectedly accepted). A non-2xx surfaces as a {@link ResponseException} from the REST + * client, which is the rejection signal. + */ + private BackendObservation observeBackend(String query) throws IOException { + try { + JSONObject response = runPplQuery(query); + return BackendObservation.accepted(response); + } catch (ResponseException e) { + int status = e.getResponse().getStatusLine().getStatusCode(); + JSONObject body; + try { + body = new JSONObject(getResponseBody(e.getResponse(), true)); + } catch (IOException ioe) { + throw new RuntimeException( + "failed to read rejection response body for query: " + query, ioe); + } + return BackendObservation.rejected(status, body); + } + } + + /** A rejected query must have thrown with the contracted status and structured error fields. */ + private void assertRejection( + String queryName, + String query, + BackendObservation obs, + int expectedStatus, + JSONObject expectedBody) { + assertTrue( + "case \"" + + queryName + + "\": expected the backend to REJECT the query but it was accepted: " + + query, + obs.rejected); + assertEquals( + "case \"" + queryName + "\": unexpected HTTP status for query: " + query, + expectedStatus, + obs.status); + assertEquals( + "case \"" + queryName + "\": unexpected top-level status field for query: " + query, + expectedBody.getInt("status"), + obs.body.getInt("status")); + + JSONObject expectedError = expectedBody.getJSONObject("error"); + JSONObject actualError = obs.body.getJSONObject("error"); + assertEquals( + "case \"" + queryName + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + if (expectedError.has("reason")) { + assertEquals( + "case \"" + queryName + "\": unexpected error.reason for query: " + query, + expectedError.getString("reason"), + actualError.getString("reason")); + } + } + + /** A result-shape case returns 200 whose datarows match the declared expectations. */ + private void assertResultShape( + String queryName, String query, BackendObservation obs, JSONObject expect) { + assertTrue( + "case \"" + + queryName + + "\": expected a 200 result but the backend rejected the query: " + + query, + !obs.rejected); + JSONObject response = obs.response; + assertTrue( + "case \"" + + queryName + + "\": expected a datarows array in the 200 response for query: " + + query, + response.has("datarows")); + if (expect == null) { + return; + } + JSONArray datarows = response.getJSONArray("datarows"); + + if (expect.optBoolean("datarowsNonEmpty", false)) { + assertTrue( + "case \"" + queryName + "\": expected non-empty datarows for query: " + query, + datarows.length() > 0); + } + if (expect.has("datarowsCount")) { + assertEquals( + "case \"" + queryName + "\": unexpected datarows count for query: " + query, + expect.getInt("datarowsCount"), + datarows.length()); + } + if (expect.has("columnAllNull")) { + String column = expect.getString("columnAllNull"); + int columnIndex = schemaColumnIndex(response, column); + assertTrue( + "case \"" + + queryName + + "\": column \"" + + column + + "\" not found in schema for query: " + + query, + columnIndex >= 0); + assertTrue( + "case \"" + + queryName + + "\": expected non-empty datarows to check null column for query: " + + query, + datarows.length() > 0); + for (int r = 0; r < datarows.length(); r++) { + JSONArray row = datarows.getJSONArray(r); + assertTrue( + "case \"" + + queryName + + "\": expected column \"" + + column + + "\" to be null in every row but row " + + r + + " was " + + row.get(columnIndex) + + " for query: " + + query, + row.isNull(columnIndex)); + } + } + } + + /** An advisory case only requires the query to be accepted (HTTP 200 with data). */ + private void assertAdvisory(String queryName, String query, BackendObservation obs) { + assertTrue( + "case \"" + + queryName + + "\": expected the query to be accepted (advisory) but it was " + + "rejected: " + + query, + !obs.rejected); + assertTrue( + "case \"" + + queryName + + "\": expected a datarows array in the 200 response for query: " + + query, + obs.response.has("datarows")); + } + + /** + * POST a PPL query to {@code /_plugins/_ppl} with a JSON-escaped body. The inherited {@code + * executeQuery} raw-interpolates the query into {@code {"query":"%s"}}, so a contract query that + * contains a double quote (e.g. {@code grok field=body "%{WORD:w}"}) would break the request + * payload and surface a spurious core-REST parse error instead of the real engine behavior. Build + * the body with a JSON serializer so any query is sent faithfully. Asserts HTTP 200 (a non-200 + * surfaces as a ResponseException, which the rejection path expects). + */ + private JSONObject runPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private int schemaColumnIndex(JSONObject response, String column) { + if (!response.has("schema")) { + return -1; + } + JSONArray schema = response.getJSONArray("schema"); + for (int i = 0; i < schema.length(); i++) { + JSONObject col = schema.getJSONObject(i); + String name = col.optString("alias", col.optString("name", "")); + if (column.equals(name) || column.equals(col.optString("name", ""))) { + return i; + } + } + return -1; + } + + /** Observed backend behavior for one query, captured before asserting the expectation. */ + private static final class BackendObservation { + final boolean rejected; + final int status; + final JSONObject body; // rejection body, or null when accepted + final JSONObject response; // accepted 200 response, or null when rejected + + private BackendObservation(boolean rejected, int status, JSONObject body, JSONObject response) { + this.rejected = rejected; + this.status = status; + this.body = body; + this.response = response; + } + + static BackendObservation accepted(JSONObject response) { + return new BackendObservation(false, 200, null, response); + } + + static BackendObservation rejected(int status, JSONObject body) { + return new BackendObservation(true, status, body, null); + } + + JSONObject toJson() { + JSONObject o = new JSONObject().put("httpStatus", status).put("rejected", rejected); + if (body != null) { + JSONObject err = body.optJSONObject("error"); + if (err != null) { + o.put("type", err.opt("type")).put("reason", err.opt("reason")); + } + } + return o; + } + } + + // --- grammar bundle export ------------------------------------------------- + + /** + * Fetch the candidate runtime grammar bundle and write it plus a target manifest, so the + * detector-validation job can lint against the SAME grammar this backend built. Best-effort by + * design: a run without {@code -Dppl.lint.grammar.bundle} (local dev) exports nothing; in CI a + * fetch/write failure is a real failure — a missing bundle means the detector half cannot run. + */ + private void exportGrammarArtifacts(List failures) { + String bundlePath = System.getProperty("ppl.lint.grammar.bundle"); + if (bundlePath == null || bundlePath.isEmpty()) { + return; + } + try { + Response response = client().performRequest(new Request("GET", GRAMMAR_API_ENDPOINT)); + String bundleBody = getResponseBody(response, true); + Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); + + JSONObject bundle = new JSONObject(bundleBody); + String grammarHash = bundle.optString("grammarHash", ""); + + String targetPath = System.getProperty("ppl.lint.target"); + if (targetPath != null && !targetPath.isEmpty()) { + JSONObject target = + new JSONObject() + .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) + .put("grammarHash", grammarHash) + .put("grammarBundle", Paths.get(bundlePath).getFileName().toString()); + Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); + } + log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); + } catch (Exception e) { + failures.add( + "[grammar-export] failed to fetch/write " + GRAMMAR_API_ENDPOINT + ": " + e.getMessage()); + } + } + + // --- cluster settings ------------------------------------------------------ + + /** True when the contract's fixture leaves Calcite enabled (the default). */ + private boolean fixtureCalciteEnabled(JSONObject fixture) { + if (fixture == null) { + return true; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null || !settings.has("calcite")) { + return true; + } + return settings.getBoolean("calcite"); + } + + /** + * Apply the contract's cluster settings and return the list of settings changed so the caller can + * reset them afterwards. Grouped per-contract (not global) because contracts disagree: eventstats + * needs {@code calciteFallback=false} to force rejection, while dedup-consecutive needs it {@code + * true} to succeed via V2 fallback. + */ + private List applyClusterSettings(JSONObject fixture) throws IOException { + List applied = new ArrayList<>(); + if (fixture == null) { + return applied; + } + JSONObject settings = fixture.optJSONObject("clusterSettings"); + if (settings == null) { + return applied; + } + if (settings.has("calcite")) { + if (settings.getBoolean("calcite")) { + enableCalcite(); + } else { + disableCalcite(); + } + } + if (settings.has("calciteFallback")) { + if (settings.getBoolean("calciteFallback")) { + allowCalciteFallback(); + } else { + disallowCalciteFallback(); + } + } + if (settings.has("allJoinTypesAllowed")) { + String key = Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES.getKeyValue(); + String value = Boolean.toString(settings.getBoolean("allJoinTypesAllowed")); + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, value)); + applied.add(key); + } + return applied; + } + + /** + * Reset each explicitly-applied dynamic setting to its cluster default by writing a null value. + * (calcite/calciteFallback are toggled via the inherited helpers and re-set explicitly by each + * contract, so only the persistent settings applied here are reset.) + */ + private void resetClusterSettings(List appliedKeys) { + for (String key : appliedKeys) { + try { + updateClusterSettings(new PPLIntegTestCase.ClusterSetting("persistent", key, null)); + } catch (IOException e) { + // Best-effort reset; the next contract sets what it needs explicitly, so + // keep the failure visible without failing the suite. + System.err.println("[ppl-lint] failed to reset a cluster setting: " + e.getMessage()); + } + } + } + + // --- version gating -------------------------------------------------------- + + private int[] fetchClusterVersion() { + try { + Response response = client().performRequest(new Request("GET", "/")); + JSONObject body = new JSONObject(getResponseBody(response, false)); + String number = body.getJSONObject("version").getString("number"); + engineVersionRaw = number; + return parseVersion(number); + } catch (Exception e) { + // Unknown version → do not skip anything. + return null; + } + } + + /** + * Test a space-separated semver range (e.g. {@code ">=3.6.0 <3.8.0"}) against the candidate + * backend version. An empty/absent range or an unknown cluster version matches (do not + * over-filter). Supports the {@code >= > <= < =} comparators the design uses. + */ + private boolean versionMatchesRange(String range) { + if (range == null || range.trim().isEmpty()) { + return true; + } + if (clusterVersion == null) { + return true; + } + for (String token : range.trim().split("\\s+")) { + if (!satisfiesComparator(token)) { + return false; + } + } + return true; + } + + private boolean satisfiesComparator(String token) { + String op; + String ver; + if (token.startsWith(">=")) { + op = ">="; + ver = token.substring(2); + } else if (token.startsWith("<=")) { + op = "<="; + ver = token.substring(2); + } else if (token.startsWith(">")) { + op = ">"; + ver = token.substring(1); + } else if (token.startsWith("<")) { + op = "<"; + ver = token.substring(1); + } else if (token.startsWith("=")) { + op = "="; + ver = token.substring(1); + } else { + op = "="; + ver = token; + } + int cmp = compareVersion(clusterVersion, parseVersion(ver)); + switch (op) { + case ">=": + return cmp >= 0; + case "<=": + return cmp <= 0; + case ">": + return cmp > 0; + case "<": + return cmp < 0; + default: + return cmp == 0; + } + } + + private int compareVersion(int[] a, int[] b) { + for (int i = 0; i < 3; i++) { + if (a[i] != b[i]) { + return Integer.compare(a[i], b[i]); + } + } + return 0; + } + + private int[] parseVersion(String raw) { + String cleaned = raw.split("-")[0]; + String[] parts = cleaned.split("\\."); + int[] v = new int[] {0, 0, 0}; + for (int i = 0; i < 3 && i < parts.length; i++) { + try { + v[i] = Integer.parseInt(parts[i]); + } catch (NumberFormatException ignored) { + v[i] = 0; + } + } + return v; + } + + // --- contract loading ------------------------------------------------------ + + private List loadScheduledContracts() throws IOException { + List result = new ArrayList<>(); + for (String fileName : manifestContractNames()) { + JSONObject contract = loadContractFile(CONTRACT_DIR + "/" + fileName); + String contractSchedule = contract.optString("schedule", "pr"); + if ("pr".equals(schedule) && !"pr".equals(contractSchedule)) { + continue; // PR runs only PR-scheduled contracts; nightly runs all. + } + result.add(contract); + } + return result; + } + + private List manifestContractNames() throws IOException { + JSONObject manifest = loadContractFile(MANIFEST); + JSONArray contracts = manifest.getJSONArray("contracts"); + List names = new ArrayList<>(); + for (int i = 0; i < contracts.length(); i++) { + names.add(contracts.getString(i)); + } + return names; + } + + /** Union of index enums required by the contracts scheduled to run this session. */ + private Set requiredIndexEnums() throws IOException { + Set indices = new LinkedHashSet<>(); + for (JSONObject contract : loadScheduledContracts()) { + JSONObject fixture = contract.optJSONObject("backendFixture"); + if (fixture == null) { + continue; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + continue; + } + for (int i = 0; i < declared.length(); i++) { + indices.add(declared.getString(i)); + } + } + if (indices.isEmpty()) { + indices.add("ACCOUNT"); + } + return indices; + } + + private JSONObject loadContractFile(String resourcePath) throws IOException { + String path = TestUtils.getResourceFilePath(resourcePath); + return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); + } + + // --- reporting ------------------------------------------------------------- + + private JSONObject reportEntry( + String ruleId, String queryName, String role, String query, String kind) { + return new JSONObject() + .put("ruleId", ruleId) + .put("queryName", queryName) + .put("role", role) + .put("query", query) + .put("kind", kind); + } + + private void writeReport(JSONArray report) { + String target = System.getProperty("ppl.lint.report"); + if (target == null || target.isEmpty()) { + return; + } + try { + Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + System.err.println("[ppl-lint] could not write backend report to " + target + ": " + e); + } + } + + private void log(String ruleId, String caseId, String message) { + System.out.println( + String.format( + Locale.ROOT, "[ppl-lint-backend-contract] %s/%s: %s", ruleId, caseId, message)); + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json new file mode 100644 index 00000000000..ca414f6f103 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 3, + "ruleId": "dedup-consecutive-unsupported", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "dedup-consecutive-unsupported", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.3.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": true } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "dedup-consecutive-true": { + "role": "trigger", + "query": "source={{index}} | dedup firstname consecutive=true" + }, + "dedup-plain-control": { + "role": "control", + "query": "source={{index}} | dedup firstname" + } + }, + "expectations": [ + { + "version": ">=3.3.0", + "engine": "calcite", + "queries": { + "dedup-consecutive-true": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + "dedup-plain-control": { + "detectorCount": 0, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json new file mode 100644 index 00000000000..bd0769934ea --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 3, + "ruleId": "disabled-join-type", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "disabled-join-type", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false, "allJoinTypesAllowed": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "right-join-disabled": { + "role": "trigger", + "query": "source={{index}} | right join left=l right=r on l.account_number=r.account_number {{index}}" + }, + "cross-join-disabled": { + "role": "trigger", + "query": "source={{index}} | cross join left=l right=r on l.account_number=r.account_number {{index}}" + }, + "inner-join-control": { + "role": "control", + "query": "source={{index}} | join left=l right=r on l.account_number=r.account_number {{index}} | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "right-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + "cross-join-disabled": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + } + }, + "inner-join-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json new file mode 100644 index 00000000000..d9071cc4978 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 3, + "ruleId": "division-by-zero", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "division-by-zero", + "enabled": true, + "severity": "warning", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "divide-by-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1" + }, + "divide-by-nonzero-control": { + "role": "control", + "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" + }, + "modulo-by-zero-flagged": { + "role": "trigger", + "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "divide-by-zero-literal": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + }, + "divide-by-nonzero-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + }, + "modulo-by-zero-flagged": { + "detectorCount": 1, + "severity": "warning", + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json new file mode 100644 index 00000000000..c67343ef23a --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -0,0 +1,81 @@ +{ + "schemaVersion": 3, + "ruleId": "field-validation", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "field-validation", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "visibleIndices": ["{{index}}"], + "deriveFromMapping": { + "account_number": "long", + "balance": "long", + "age": "long", + "firstname": "text", + "lastname": "text", + "gender": "text", + "address": "text", + "employer": "text", + "email": "text", + "city": "text", + "state": "text" + } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "unknown-field-existence": { + "role": "trigger", + "query": "source={{index}} | where nonexistent_field > 3" + }, + "grok-field-slot-shape-typo": { + "role": "trigger", + "query": "source={{index}} | grok field=firstname \"%{WORD:w}\"" + }, + "known-field-control": { + "role": "control", + "query": "source={{index}} | where age > 30 | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "matchMessage": "nonexistent_field", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json new file mode 100644 index 00000000000..e86a19c1002 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 3, + "ruleId": "head-without-sort", + "grammarSurface": "compiled-simplified", + "schedule": "nightly", + "wiring": { + "detector": "head-without-sort", + "enabled": true, + "severity": "info", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "head-without-sort": { + "role": "trigger", + "query": "source={{index}} | head 5" + }, + "head-with-sort-control": { + "role": "control", + "query": "source={{index}} | sort age | head 5" + } + }, + "expectations": [ + { + "version": ">=0.0.0", + "queries": { + "head-without-sort": { + "detectorCount": 1, + "severity": "info", + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + }, + "head-with-sort-control": { + "detectorCount": 0, + "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json new file mode 100644 index 00000000000..805e1be9fd8 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": 3, + "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", + "contracts": [ + "unsupported-window-function-in-eventstats.spec.json", + "division-by-zero.spec.json", + "head-without-sort.spec.json", + "disabled-join-type.spec.json", + "field-validation.spec.json", + "dedup-consecutive-unsupported.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json" + ], + "enforced": [ + "unsupported-window-function-in-eventstats.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json" + ], + "pendingReview": [ + "field-validation.spec.json" + ], + "nonEnforcing": [ + "division-by-zero.spec.json", + "head-without-sort.spec.json", + "disabled-join-type.spec.json", + "dedup-consecutive-unsupported.spec.json" + ], + "notes": { + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required validation-result check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. field-validation self-suppresses without field context and is a semantic rule rather than a clean HTTP-400 grammar rejection.", + "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json new file mode 100644 index 00000000000..018d086fec3 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -0,0 +1,55 @@ +{ + "schemaVersion": 3, + "ruleId": "multisearch-min-subsearch", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["multisearchCommand", "subSearch"], + "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", + "wiring": { + "detector": "multisearch-min-subsearch", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "multisearch-single-subsearch": { + "role": "trigger", + "query": "multisearch [ search source={{index}} ]" + }, + "multisearch-two-subsearches-control": { + "role": "control", + "query": "multisearch [ search source={{index}} ] [ search source={{index}} ]" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "queries": { + "multisearch-single-subsearch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } + } + }, + "multisearch-two-subsearches-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json new file mode 100644 index 00000000000..8946dad60a9 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 3, + "ruleId": "replace-wildcard-asymmetry", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["replacePair", "stringLiteral"], + "wiring": { + "detector": "replace-wildcard-asymmetry", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "replace-wildcard-count-mismatch": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname" + }, + "replace-symmetric-control": { + "role": "control", + "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 1 wildcard(s), replacement has 2. Replacement must have same number of wildcards or none." + } + } + } + }, + "replace-symmetric-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json new file mode 100644 index 00000000000..7f110cc9423 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 3, + "ruleId": "union-min-datasets", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["unionCommand", "unionDataset", "pplCommands"], + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", + "wiring": { + "detector": "union-min-datasets", + "enabled": true, + "severity": "error", + "runtimeOnly": true, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.7.0", "engine": "calcite" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json new file mode 100644 index 00000000000..e1254b14583 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 3, + "ruleId": "unsupported-window-function-in-eventstats", + "grammarSurface": "compiled-simplified", + "schedule": "pr", + "wiring": { + "detector": "unsupported-window-function-in-eventstats", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": { "minVersion": "3.4.0" } + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "eventstats-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats rank() as rank_value" + }, + "eventstats-avg-control": { + "role": "control", + "query": "source={{index}} | eventstats avg(age) as avg_age" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh new file mode 100755 index 00000000000..295168f8aad --- /dev/null +++ b/scripts/ppl-lint-rule-validation.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Local developer entry point for the PPL lint rule validation contract. +# +# Runs both halves of the cross-repository check from a SQL checkout, in the same +# order as CI (design §3.1): +# 1. Backend: runs the Gradle integration test against a live /_plugins/_ppl +# endpoint on the SQL plugin built from this checkout, and — while the +# cluster is alive — exports the candidate runtime grammar bundle +# (ppl-grammar-bundle.json), a target manifest (target.json), and the +# observed backend report (backend-report.json). +# 2. Detector: bootstraps an OpenSearch-Dashboards (OSD) checkout, deserializes +# the candidate bundle through OSD's headless lint API, runs the real +# detectors against the same queries, and asserts the detector-vs-backend +# differential. +# +# The backend half must run first: the detector half lints against the bundle it +# exports. Use SKIP_BACKEND=1 only if you already have the three artifacts. +# +# Usage: +# # OSD main detector check plus SQL backend IT (fetches OSD into .ci/) +# ./scripts/ppl-lint-rule-validation.sh +# +# # Reuse an existing OSD checkout (skips clone + bootstrap if node_modules present) +# OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh +# +# # Reproduce a CI run against a specific OSD revision +# OSD_REF= ./scripts/ppl-lint-rule-validation.sh +# +# # Skip one half (detector needs the backend artifacts to exist already) +# SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +# SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Run the full nightly corpus (all rules + coverage assertion) +# PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh + +set -euo pipefail + +SQL_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$SQL_ROOT" + +OSD_REPO_URL="${OSD_REPO_URL:-https://github.com/opensearch-project/OpenSearch-Dashboards.git}" +OSD_REF="${OSD_REF:-main}" +DEFAULT_OSD_CHECKOUT="$SQL_ROOT/.ci/OpenSearch-Dashboards" +CONTRACT_DIR="$SQL_ROOT/integ-test/src/test/resources/ppl-lint/contracts" +DETECTOR_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" +IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" +# pr (fast, blocking subset) or nightly (full corpus + coverage assertion). +PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" + +# Candidate artifacts the backend half exports and the detector half consumes. +GRAMMAR_BUNDLE="$SQL_ROOT/ppl-grammar-bundle.json" +TARGET_MANIFEST="$SQL_ROOT/target.json" +BACKEND_REPORT="$SQL_ROOT/backend-report.json" +DETECTOR_REPORT="$SQL_ROOT/detector-report.json" + +log() { echo "[ppl-lint-rule-validation] $*"; } + +run_backend() { + log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ + -Dppl.lint.report="$BACKEND_REPORT" \ + -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" \ + -Dppl.lint.target="$TARGET_MANIFEST" + log "Backend integration test passed. Exported: $(basename "$GRAMMAR_BUNDLE"), $(basename "$TARGET_MANIFEST")." +} + +run_detector() { + local osd_checkout="$1" + + if [[ ! -f "$GRAMMAR_BUNDLE" ]]; then + log "ERROR: $GRAMMAR_BUNDLE not found. Run the backend half first (do not set SKIP_BACKEND=1)." + exit 2 + fi + + if [[ ! -d "$osd_checkout/node_modules" ]]; then + log "Bootstrapping OSD at $osd_checkout (this can take a while)..." + (cd "$osd_checkout" && yarn osd bootstrap) + else + log "Reusing bootstrapped OSD at $osd_checkout (node_modules present)." + fi + + log "Running detector validation against the candidate bundle (schedule=$PPL_LINT_SCHEDULE)..." + ( + cd "$osd_checkout" + PPL_LINT_CONTRACT_DIR="$CONTRACT_DIR" \ + PPL_LINT_SCHEDULE="$PPL_LINT_SCHEDULE" \ + PPL_LINT_GRAMMAR_BUNDLE="$GRAMMAR_BUNDLE" \ + PPL_LINT_TARGET_MANIFEST="$TARGET_MANIFEST" \ + PPL_LINT_BACKEND_REPORT="$BACKEND_REPORT" \ + PPL_LINT_REPORT="$DETECTOR_REPORT" \ + node -r ./src/setup_node_env "$DETECTOR_SCRIPT" + ) + log "Detector validation passed." +} + +if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then + run_backend +else + log "SKIP_BACKEND=1 — skipping the SQL backend integration test (using existing artifacts)." +fi + +if [[ "${SKIP_DETECTOR:-0}" != "1" ]]; then + if [[ -n "${OSD_SOURCE_PATH:-}" ]]; then + OSD_CHECKOUT="$(cd "$OSD_SOURCE_PATH" && pwd)" + log "Using existing OSD checkout: $OSD_CHECKOUT" + else + OSD_CHECKOUT="$DEFAULT_OSD_CHECKOUT" + if [[ ! -d "$OSD_CHECKOUT/.git" ]]; then + log "Cloning OSD ($OSD_REF) into $OSD_CHECKOUT ..." + mkdir -p "$(dirname "$OSD_CHECKOUT")" + git clone --depth 1 --branch "$OSD_REF" "$OSD_REPO_URL" "$OSD_CHECKOUT" 2>/dev/null || + git clone "$OSD_REPO_URL" "$OSD_CHECKOUT" + fi + log "Checking out OSD ref: $OSD_REF" + git -C "$OSD_CHECKOUT" fetch --depth 1 origin "$OSD_REF" 2>/dev/null || true + git -C "$OSD_CHECKOUT" checkout "$OSD_REF" 2>/dev/null || + git -C "$OSD_CHECKOUT" checkout FETCH_HEAD + fi + + OSD_SHA="$(git -C "$OSD_CHECKOUT" rev-parse HEAD)" + log "OSD revision under test: $OSD_SHA" + + run_detector "$OSD_CHECKOUT" +else + log "SKIP_DETECTOR=1 — skipping the OSD detector contract." +fi + +log "Done." diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md new file mode 100644 index 00000000000..e9578517608 --- /dev/null +++ b/scripts/ppl-lint/README.md @@ -0,0 +1,202 @@ +# PPL lint rule validation + +A required, cross-repository GitHub Actions check that proves the OpenSearch +Dashboards (OSD) PPL lint detectors and the SQL backend still agree — on the +**same candidate runtime grammar** built by a SQL pull request. + +PPL language behavior lives in SQL; PPL lint detectors live in OSD. A SQL change +can silently invalidate an OSD rule (a parser refactor stops a detector matching, +or a semantic change makes a flagged query valid) without touching OSD. Neither +repository's own unit tests catch that. This check does. + +- **Design:** `ppl-lint-ci-validation-design.md` +- **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) +- **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) + +## The pipeline + +Three jobs run in a line; artifacts are the only bridge between them. + +``` +backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.json)──▶ + detector-validation ──▶ validation-result (the single required check) +``` + +1. **backend-validation** (OpenSearch CI container). Builds the SQL PR, starts + the Gradle test cluster, runs each contract's trigger/control queries against + `POST /_plugins/_ppl`, and — while the cluster is alive — exports: + - `ppl-grammar-bundle.json` — the candidate runtime grammar (`GET /_plugins/_ppl/_grammar`); + - `target.json` — `{ engineVersion, grammarHash, grammarBundle }`; + - `backend-report.json` — the observed HTTP behavior per query. +2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a + Node code dependency (no OSD server, no Monaco, no browser), then runs + [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner + deserializes the candidate bundle through OSD's production headless lint API + (`src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint`) and lints + each query with the **real** detectors on the **candidate** grammar. It then + asserts the detector-vs-backend differential. +3. **validation-result**. `if: always()`, `needs: [backend-validation, + detector-validation]`. Fails unless both succeeded — so a skipped detector + (because the backend failed first) still reds the check instead of looking + green. It writes the per-rule PR summary and uploads `run-manifest.json`. This + is the **only** job repo admins pin to branch protection. + +## Workflow inputs and modes + +| Trigger | Mode | OSD ref | Enforcing? | +| --- | --- | --- | --- | +| `pull_request` | SQL PR validation | `main` | **Yes** — the required check | +| `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | +| `schedule` (nightly) | full corpus + coverage | `main` | No | + +`workflow_dispatch` inputs: + +- `osd_repo` — the OSD repository to check out, for validating an unmerged change + that lives on a fork. Defaults to `opensearch-project/OpenSearch-Dashboards`. + The `osd_ref` must exist in this repo (a purely local commit cannot be fetched). +- `osd_ref` — an OSD commit or branch to validate instead of `main`. Resolved to + an immutable commit SHA and recorded in the run manifest. A manual run **cannot** + satisfy branch protection; merge the OSD change first, then rerun the required + `pull_request` check against OSD `main`. +- `schedule` — `pr` (fast blocking subset) or `nightly` (full corpus). + +To validate an OSD change that is not yet merged, push it to a branch on your OSD +fork and dispatch with `osd_repo=/OpenSearch-Dashboards` and +`osd_ref=`. + +## Local reproduction + +From the SQL checkout: + +```bash +# Backend IT (exports the bundle) then detector check against OSD main. +./scripts/ppl-lint-rule-validation.sh + +# Reuse an already-bootstrapped OSD checkout. +OSD_SOURCE_PATH=../OpenSearch-Dashboards ./scripts/ppl-lint-rule-validation.sh + +# Reproduce a specific CI run's OSD revision (from run-manifest.json). +OSD_REF= ./scripts/ppl-lint-rule-validation.sh + +# Full nightly corpus + coverage assertion. +PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh + +# Re-run only one half (detector needs the backend artifacts to exist). +SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh +SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh +``` + +The backend half writes `ppl-grammar-bundle.json`, `target.json`, and +`backend-report.json` to the SQL repo root; the detector half consumes them and +writes `detector-report.json`. + +### Runner environment contract + +`run-frontend-contract.mjs` is run from inside the OSD checkout with +`node -r ./src/setup_node_env` and reads: + +| Env var | Meaning | +| --- | --- | +| `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | +| `PPL_LINT_SCHEDULE` | `pr` or `nightly` | +| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | +| `PPL_LINT_TARGET_MANIFEST` | `target.json` (engine version + grammar hash) | +| `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | +| `PPL_LINT_REPORT` | where to write `detector-report.json` | +| `PPL_LINT_CONTRACT_FILE` | (optional) run a single spec instead of the dir | + +## Contract format (schema v3) + +One JSON file per rule under `contracts/`, listed in `manifest.json`. Each file +has a top-level `queries` map (each `{ role: "trigger"|"control", query }`) and a +version-scoped `expectations[]`. Exactly one expectation must match the candidate +backend version (zero or more than one fails before any query runs). + +```jsonc +{ + "schemaVersion": 3, + "ruleId": "union-min-datasets", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "wiring": { "detector": "union-min-datasets", "enabled": true, "severity": "error", ... }, + "backendFixture": { "indices": ["ACCOUNT"], "clusterSettings": { "calcite": true, "calciteFallback": false } }, + "frontendContext": { "isCalcite": true }, + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { "role": "trigger", "query": "| union [ source={{index}} ]" }, + "union-two-datasets-control": { "role": "control", "query": "| union [ source={{index}} ] [ source={{index}} ]" } + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, "severity": "error", + "backend": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} +``` + +`backend.kind` is one of `rejection` (contracted 4xx + error type/reason), +`result-shape` (200 with datarow expectations), or `advisory` (soft 200-only +oracle). When a behavior changes in a new version, keep **both** version-scoped +expectations so the nightly matrix proves the rule still fires on the old version +while the candidate check proves the fix on the new one. + +### Pitfall: do not write pipe-first (`| command …`) trigger queries + +The detector half and the backend half must run the **byte-identical** query +(design's "Same queries" requirement). OSD's runtime lint path prepends a +synthetic `source=t ` prefix to any query that starts with a pipe, so linting +`| union [ source=idx ]` actually parses `source=t | union [ source=idx ]` — a +valid *mid-pipeline* union whose implicit upstream dataset makes the detector +stay silent. The backend, receiving the raw pipe-first query, still rejects it. +The two halves then disagree even though nothing is wrong. Write triggers in a +**query-initial** form (`union [ source=idx ]`, `multisearch [ search source=idx ]`) +that both sides accept verbatim. Until SQL emits `pipeStartRuleIndex` in the +grammar bundle (design §6, D-pipe), a pipe-first trigger with a distinct start +rule cannot be validated end to end. + +### The enforced set + +`manifest.json` partitions the corpus: + +- `enforced` — reviewed error rules with a deterministic backend rejection and a + valid negative control. These block `validation-result`. Phase one: + `unsupported-window-function-in-eventstats`, `multisearch-min-subsearch`, + `union-min-datasets`, `replace-wildcard-asymmetry`. +- `pendingReview` — error rules awaiting Peng/Chen usefulness review before + joining `enforced` (currently `field-validation`). +- `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the + nightly schedule for coverage and never block a PR. + +## Interpreting a failure + +| Failure | Meaning | +| --- | --- | +| Grammar bundle export fails | The candidate SQL build does not provide a usable runtime grammar. | +| Trigger no longer parses | The grammar changed ownership of the error or regressed. | +| Detector emits no diagnostic | The detector is incompatible with the candidate parse tree. | +| Detector flags the control | The detector became too broad. | +| Backend accepts the trigger | The lint rule's premise may be fixed or stale. | +| Backend rejects the control | Query, fixture, settings, or SQL behavior regressed. | +| No version expectation matches | The rule test does not cover the candidate version. | + +CI never rewrites expected results. A behavior change is an intentional, reviewed +edit to a versioned expectation **and** the corresponding OSD rule. If a SQL +change depends on an OSD rule update, merge the OSD change first, then rerun the +required SQL check against OSD `main`. + +## Artifacts + +Every run uploads: `run-manifest.json` (exact SQL SHA, OSD SHA, mode, backend +version, grammar hash, selected validation set), the candidate grammar bundle, +the backend and detector reports, the committed contracts used, and the job logs. diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs new file mode 100644 index 00000000000..840e631ef88 --- /dev/null +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -0,0 +1,160 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assemble the PPL lint validation run manifest and the compact per-rule PR + * summary in the result job (design §3.3, §4.4, T10). + * + * Inputs (env, all optional so a partial run still produces a manifest): + * SQL_SHA, OSD_REF, OSD_SHA, EVENT_NAME, SCHEDULE, + * BACKEND_RESULT, DETECTOR_RESULT, GITHUB_STEP_SUMMARY. + * Artifact files under ./artifacts (downloaded from both jobs): + * target.json (engineVersion + grammarHash), backend-report.json, + * detector-report.json. + * + * Outputs: + * run-manifest.json in the workspace root; a markdown table appended to + * $GITHUB_STEP_SUMMARY. + */ + +import fs from 'fs'; +import path from 'path'; + +const ARTIFACTS = 'artifacts'; + +function readJson(file) { + try { + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not parse ${file}: ${error.message}`); + } + return undefined; +} + +function main() { + const target = readJson(path.join(ARTIFACTS, 'target.json')) || {}; + const detector = readJson(path.join(ARTIFACTS, 'detector-report.json')) || {}; + const backend = readJson(path.join(ARTIFACTS, 'backend-report.json')) || []; + + const eventName = process.env.EVENT_NAME || ''; + const osdRef = process.env.OSD_REF || 'main'; + const osdRepo = process.env.OSD_REPO || 'opensearch-project/OpenSearch-Dashboards'; + const isUpstreamMain = osdRepo === 'opensearch-project/OpenSearch-Dashboards' && osdRef === 'main'; + const mode = + eventName === 'pull_request' + ? 'sql-pr-validation' + : eventName === 'schedule' + ? 'nightly' + : !isUpstreamMain + ? 'osd-branch-evidence' + : 'manual'; + + const backendResult = process.env.BACKEND_RESULT || 'unknown'; + const detectorResult = process.env.DETECTOR_RESULT || 'unknown'; + const passed = backendResult === 'success' && detectorResult === 'success'; + + // The selected validation set is the set of rules the detector run actually + // evaluated (post schedule filtering). + const validationSet = Array.from( + new Set((detector.results || []).map((r) => r.ruleId)) + ).sort(); + + const manifest = { + mode, + // A workflow_dispatch osd_ref run is pre-merge evidence, never a + // branch-protection result (design §4.1.1, T11). + requiredCheck: eventName === 'pull_request', + event: eventName, + schedule: process.env.SCHEDULE || detector.schedule || 'pr', + sqlSha: process.env.SQL_SHA || '', + osdRepo, + osdRef, + osdSha: process.env.OSD_SHA || '', + engineVersion: target.engineVersion || detector.engineVersion || '', + grammarHash: target.grammarHash || detector.grammarHash || '', + differential: !!detector.differential, + validationSet, + result: { + backend: backendResult, + detector: detectorResult, + passed, + }, + }; + + fs.writeFileSync('run-manifest.json', JSON.stringify(manifest, null, 2)); + + writeSummary(manifest, detector, backend); +} + +/** Compact per-rule PR summary: Rule | Version | Grammar | Detector | Backend | Result. */ +function writeSummary(manifest, detector, backend) { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) { + return; + } + + const backendByKey = new Map(); + for (const e of Array.isArray(backend) ? backend : []) { + backendByKey.set(`${e.ruleId}::${e.queryName}`, e); + } + + const shortHash = (h) => (h ? String(h).replace(/^sha256:/, '').slice(0, 12) : '—'); + + const lines = []; + lines.push('## PPL lint rule validation'); + lines.push(''); + lines.push(`- Mode: \`${manifest.mode}\`${manifest.requiredCheck ? ' (required)' : ' (non-enforcing)'}`); + lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); + lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (${manifest.osdRepo} @ \`${manifest.osdRef}\`)`); + lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); + lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); + lines.push( + `- Result: backend **${manifest.result.backend}**, detector **${manifest.result.detector}** → ` + + `**${manifest.result.passed ? 'PASS' : 'FAIL'}**` + ); + lines.push(''); + lines.push('| Rule | Query | Version | Grammar | Detector | Backend | Result |'); + lines.push('| ---- | ----- | ------- | ------- | -------- | ------- | ------ |'); + + for (const r of detector.results || []) { + const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); + const detectorCell = `${r.actual}/${r.expected}${r.severities && r.severities.length ? ` (${r.severities.join(',')})` : ''}`; + const backendCell = be + ? be.rejected + ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` + : 'accepted' + : '—'; + const ok = + r.actual === r.expected && (!be || (r.role === 'trigger' ? be.rejected : !be.rejected)); + lines.push( + `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ok ? 'Pass' : 'Fail'} |` + ); + } + + if ((detector.failures || []).length > 0) { + lines.push(''); + lines.push('

Failures'); + lines.push(''); + for (const f of detector.failures) { + lines.push(`- ${f}`); + } + lines.push(''); + lines.push('
'); + } + + lines.push(''); + try { + fs.appendFileSync(summaryPath, lines.join('\n') + '\n'); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-manifest] could not write step summary: ${error.message}`); + } +} + +main(); diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs new file mode 100644 index 00000000000..8bd1ff5bad1 --- /dev/null +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -0,0 +1,576 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * SQL-owned detector-validation runner for the PPL lint rule validation CI. + * + * This script is executed from inside an OpenSearch-Dashboards (OSD) checkout, + * for example: + * + * cd .ci/OpenSearch-Dashboards + * PPL_LINT_CONTRACT_DIR= \ + * PPL_LINT_SCHEDULE=pr \ + * PPL_LINT_GRAMMAR_BUNDLE= \ + * PPL_LINT_TARGET_MANIFEST= \ + * PPL_LINT_BACKEND_REPORT= \ + * PPL_LINT_REPORT= \ + * node -r ./src/setup_node_env \ + * "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + * + * `node -r ./src/setup_node_env` installs OSD's process-wide auto-transpilation + * hook (`@osd/optimizer`'s `registerNodeAutoTranspilation`), which transpiles + * `src/plugins/**` and `packages/osd-monaco/src/**` TypeScript on `require()` + * regardless of where the entry script lives. That is what lets this SQL-owned + * `.mjs` load OSD's Node-safe headless lint API without OSD's own Jest. + * + * This is the detector half of a schema-v3 cross-repository differential + * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). + * Unlike the earlier PoC — which linted with the compiled analyzer or a + * hand-rolled reparse against OSD `main`'s checked-in grammar — it lints against + * the *candidate* runtime grammar bundle the SQL backend job exported, via OSD's + * production headless API (`headless_ppl_lint`). Both halves therefore validate + * the exact same candidate grammar (design §4.3). + * + * It asserts, per contract: + * 1. Wiring: the OSD catalog entry deep-equals the contract's `wiring` block, + * so a silently removed/retyped/re-gated/re-severitied detector reds the + * build. + * 2. Detector: for the single version expectation that matches the candidate + * backend version, each query emits exactly the contracted number of + * `ruleId` diagnostics at the contracted severity. + * 3. Differential (when PPL_LINT_BACKEND_REPORT is supplied): the observed + * backend behavior for each query agrees with the observed detector output + * — a trigger the detector flags is one the backend rejected; a control the + * detector passes is one the backend accepted (design §3.2, §4.3). + * 4. Coverage (nightly only): every enabled catalog rule has a contract file. + */ + +import fs from 'fs'; +import path from 'path'; +import { createRequire } from 'module'; + +// OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved +// against the OSD checkout root, not this script's SQL-repo location. +const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +// The Monaco-free engine barrel (@osd/monaco/ppl-lint) exposes the catalog; the +// detector registry is a deep import used only for the wiring registration check. +const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; +const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-detector-contract] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-detector-contract] FATAL: ${message}`); + process.exit(2); +} + +/** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ +function loadContracts() { + const dir = process.env.PPL_LINT_CONTRACT_DIR; + const single = process.env.PPL_LINT_CONTRACT_FILE; + + if (single) { + if (!fs.existsSync(single)) { + fatal(`Contract file not found: ${single}`); + } + return [{ file: single, spec: JSON.parse(fs.readFileSync(single, 'utf8')) }]; + } + + if (!dir) { + fatal('Set PPL_LINT_CONTRACT_DIR (a directory of *.spec.json) or PPL_LINT_CONTRACT_FILE.'); + } + if (!fs.existsSync(dir)) { + fatal(`Contract directory not found: ${dir}`); + } + + const manifestPath = path.join(dir, 'manifest.json'); + let files; + if (fs.existsSync(manifestPath)) { + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + if (!Array.isArray(manifest.contracts)) { + fatal(`manifest.json must have a "contracts" array of file names.`); + } + files = manifest.contracts.map((name) => path.join(dir, name)); + } else { + files = fs + .readdirSync(dir) + .filter((f) => f.endsWith('.spec.json')) + .sort() + .map((f) => path.join(dir, f)); + } + + return files.map((file) => { + if (!fs.existsSync(file)) { + fatal(`Contract referenced by manifest not found: ${file}`); + } + return { file, spec: JSON.parse(fs.readFileSync(file, 'utf8')) }; + }); +} + +function loadOsd() { + const osdRoot = process.cwd(); + const require = createRequire(path.join(osdRoot, 'noop.js')); + + const resolveOsd = (relativeModule, { optional = false } = {}) => { + const absolute = path.join(osdRoot, relativeModule); + const exists = + fs.existsSync(absolute) || + fs.existsSync(`${absolute}.ts`) || + fs.existsSync(`${absolute}.js`); + if (!exists) { + if (optional) { + return undefined; + } + fatal( + `Expected OSD module not found under the checkout root: ${relativeModule}\n` + + `Resolved OSD root: ${osdRoot}\n` + + `Run this script from the OSD checkout (e.g. cd .ci/OpenSearch-Dashboards) after bootstrap.` + ); + } + try { + return require(absolute); + } catch (error) { + if (optional) { + return undefined; + } + throw error; + } + }; + + const headless = resolveOsd(HEADLESS_MODULE); + const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); + + const { deserializeBundleOrThrow, lintQueryWithBundle } = headless; + if (typeof deserializeBundleOrThrow !== 'function' || typeof lintQueryWithBundle !== 'function') { + fatal( + `Headless lint API not found in ${HEADLESS_MODULE}. ` + + `Expected exports deserializeBundleOrThrow + lintQueryWithBundle. ` + + `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` + ); + } + if (typeof getBundledCatalog !== 'function') { + fatal(`getBundledCatalog not found in ${CATALOG_MODULE}.`); + } + + const getDetector = registry && registry.getDetector; + return { deserializeBundleOrThrow, lintQueryWithBundle, getBundledCatalog, getDetector, osdRoot }; +} + +/** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ +function loadCandidateGrammar(osd) { + const bundlePath = process.env.PPL_LINT_GRAMMAR_BUNDLE; + if (!bundlePath) { + fatal( + 'PPL_LINT_GRAMMAR_BUNDLE is not set. Detector validation lints against the candidate ' + + 'runtime grammar bundle exported by the backend job; there is no compiled fallback.' + ); + } + if (!fs.existsSync(bundlePath)) { + fatal(`Candidate grammar bundle not found: ${bundlePath}`); + } + let bundle; + try { + bundle = JSON.parse(fs.readFileSync(bundlePath, 'utf8')); + } catch (error) { + fatal(`Could not parse grammar bundle ${bundlePath}: ${error.message}`); + } + try { + return osd.deserializeBundleOrThrow(bundle); + } catch (error) { + fatal(`Could not deserialize candidate grammar bundle: ${error.message}`); + } + return undefined; // unreachable +} + +/** Read the target manifest (engineVersion + grammarHash) written beside the bundle. */ +function loadTarget() { + const targetPath = process.env.PPL_LINT_TARGET_MANIFEST; + if (targetPath && fs.existsSync(targetPath)) { + try { + return JSON.parse(fs.readFileSync(targetPath, 'utf8')); + } catch (error) { + log(`WARN: could not parse target manifest ${targetPath}: ${error.message}`); + } + } + // Back-compat / local runs without a target manifest. + return { engineVersion: process.env.PPL_SQL_VERSION || '', grammarHash: '' }; +} + +/** Index the backend report by `${ruleId}::${queryName}` for the differential. */ +function loadBackendReport() { + const reportPath = process.env.PPL_LINT_BACKEND_REPORT; + if (!reportPath || !fs.existsSync(reportPath)) { + return undefined; + } + let entries; + try { + entries = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + } catch (error) { + log(`WARN: could not parse backend report ${reportPath}: ${error.message}`); + return undefined; + } + const byKey = new Map(); + for (const entry of Array.isArray(entries) ? entries : []) { + byKey.set(`${entry.ruleId}::${entry.queryName}`, entry); + } + return byKey; +} + +/** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ +function parseVersion(v) { + if (!v) return undefined; + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v)); + if (!m) return undefined; + return [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)]; +} + +function compareVersion(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * Test a space-separated semver range (e.g. ">=3.6.0 <3.8.0") against the + * candidate backend version. An empty range or an unknown version matches (do + * not over-filter). Mirrors PplLintRuleValidationIT.versionMatchesRange. + */ +function versionMatchesRange(range, version) { + if (!range || !range.trim()) return true; + const have = parseVersion(version); + if (!have) return true; + for (const token of range.trim().split(/\s+/)) { + let op = '='; + let ver = token; + if (token.startsWith('>=')) { + op = '>='; + ver = token.slice(2); + } else if (token.startsWith('<=')) { + op = '<='; + ver = token.slice(2); + } else if (token.startsWith('>')) { + op = '>'; + ver = token.slice(1); + } else if (token.startsWith('<')) { + op = '<'; + ver = token.slice(1); + } else if (token.startsWith('=')) { + op = '='; + ver = token.slice(1); + } + const cmp = compareVersion(have, parseVersion(ver) || [0, 0, 0]); + const ok = + (op === '>=' && cmp >= 0) || + (op === '<=' && cmp <= 0) || + (op === '>' && cmp > 0) || + (op === '<' && cmp < 0) || + (op === '=' && cmp === 0); + if (!ok) return false; + } + return true; +} + +/** + * Select the single expectation that applies to the candidate version + engine. + * Exactly one must match (design §5.3): zero means the rule test does not cover + * this version; more than one means overlapping ranges. Both fail. + */ +function selectExpectation(spec, version, isCalcite, failures) { + const expectations = spec.expectations || []; + const matches = expectations.filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && isCalcite !== true) return false; + return true; + }); + if (matches.length === 1) { + return matches[0]; + } + const label = version || 'unknown'; + if (matches.length === 0) { + failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + } else { + failures.push( + `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} (exactly one required).` + ); + } + return undefined; +} + +/** + * Assert the OSD catalog entry deep-equals the contract's `wiring` block. This is + * the primary OSD-drift tripwire: if a detector is removed, retyped, re-gated or + * its severity changed, this fails before any query runs. + */ +function checkWiring(spec, catalog, getDetector, failures) { + const { ruleId, wiring } = spec; + const entry = catalog.find((c) => c.id === ruleId); + if (!entry) { + failures.push(`[${ruleId}] not present in the OSD bundled catalog.`); + return undefined; + } + if (!wiring) { + return entry; // no wiring block to assert + } + + const checks = [ + ['detector', wiring.detector, entry.detector], + ['enabled', wiring.enabled, entry.enabled], + ['severity', wiring.severity, entry.severity], + ['runtimeOnly', !!wiring.runtimeOnly, !!entry.runtimeOnly], + ['needsContext', !!wiring.needsContext, !!entry.needsContext], + ['needsExplain', !!wiring.needsExplain, !!entry.needsExplain], + ]; + for (const [name, expected, actual] of checks) { + if (expected !== undefined && expected !== actual) { + failures.push( + `[${ruleId}] wiring.${name} expected ${JSON.stringify(expected)} but catalog has ${JSON.stringify(actual)}.` + ); + } + } + + if (wiring.appliesTo) { + const a = entry.appliesTo || {}; + for (const key of ['minVersion', 'maxVersion', 'engine']) { + if (wiring.appliesTo[key] !== undefined && wiring.appliesTo[key] !== a[key]) { + failures.push( + `[${ruleId}] wiring.appliesTo.${key} expected ${JSON.stringify(wiring.appliesTo[key])} but catalog has ${JSON.stringify(a[key])}.` + ); + } + } + } + + if (wiring.detector && typeof getDetector === 'function' && typeof getDetector(wiring.detector) !== 'function') { + failures.push(`[${ruleId}] has no registered detector "${wiring.detector}".`); + } + + return entry; +} + +/** + * Build the per-contract lint context passed to `lintQueryWithBundle`. Derives + * `fields`/`typeMap` from the `deriveFromMapping` block (a single source shared + * with the backend seeding), pins `dataSourceVersion`/`knownVersion` to the + * candidate backend version so version filtering matches the backend, and sets + * an enable override for default-off rules that declare `forceEnable`. + */ +function buildContext(spec, engineVersion) { + const fc = spec.frontendContext || {}; + const context = { + isCalcite: fc.isCalcite !== false, + dataSourceVersion: engineVersion || undefined, + // Pin the "latest verified engine" to the candidate version rather than the + // hardcoded OSD_KNOWN_VERSION ('3.7.0'), which can mis-filter rules near a + // version boundary (design §4.3, D-version). + knownVersion: engineVersion || undefined, + }; + + const mapping = fc.deriveFromMapping; + if (mapping && typeof mapping === 'object') { + const fields = new Set(); + const typeMap = new Map(); + for (const [name, type] of Object.entries(mapping)) { + fields.add(name); + typeMap.set(name, type); + } + context.fields = fields; + context.typeMap = typeMap; + } + if (Array.isArray(fc.disabledObjectFields) && fc.disabledObjectFields.length > 0) { + context.disabledObjectFields = new Set(fc.disabledObjectFields); + } + if (Array.isArray(fc.visibleIndices) && fc.visibleIndices.length > 0) { + context.visibleIndices = fc.visibleIndices.map((i) => i.split('{{index}}').join(spec.index)); + } + if (fc.settings && typeof fc.settings === 'object') { + context.settings = fc.settings; + } + if (fc.forceEnable) { + context.overrides = { [spec.ruleId]: { enabled: true } }; + } + return context; +} + +function main() { + const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; + const reportPath = process.env.PPL_LINT_REPORT; + + const osd = loadOsd(); + const { getBundledCatalog, getDetector, lintQueryWithBundle, osdRoot } = osd; + const catalog = getBundledCatalog(); + + const grammar = loadCandidateGrammar(osd); + const target = loadTarget(); + const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; + const backendReport = loadBackendReport(); + + const contracts = loadContracts(); + const failures = []; + const report = { + osdRoot, + schedule, + engineVersion, + grammarHash: target.grammarHash || '', + differential: !!backendReport, + results: [], + }; + + log(`OSD root: ${osdRoot}`); + log( + `schedule=${schedule} engineVersion=${engineVersion || '(unset)'} ` + + `grammarHash=${target.grammarHash || '(unset)'} differential=${!!backendReport} ` + + `contracts=${contracts.length}` + ); + + for (const { file, spec } of contracts) { + const ruleId = spec.ruleId; + const index = spec.index; + + // A contract runs on PR only when scheduled for PR; nightly runs everything. + const contractSchedule = spec.schedule || 'pr'; + if (schedule === 'pr' && contractSchedule !== 'pr') { + log(`SKIP ${ruleId} (schedule=${contractSchedule}, running ${schedule}) — ${path.basename(file)}`); + continue; + } + + const entry = checkWiring(spec, catalog, getDetector, failures); + if (!entry) { + continue; + } + + const context = buildContext(spec, engineVersion); + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures); + if (!expectation) { + continue; + } + + const queries = spec.queries || {}; + const expectedQueries = expectation.queries || {}; + for (const queryName of Object.keys(expectedQueries)) { + const queryDef = queries[queryName]; + if (!queryDef) { + failures.push(`[${ruleId}] expectation references unknown query "${queryName}".`); + continue; + } + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + const expected = expectedQueries[queryName]; + const expectedCount = expected.detectorCount; + + const result = lintQueryWithBundle(query, grammar, context); + const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + const actual = matches.length; + const ok = actual === expectedCount; + + log( + ` ${ok ? 'PASS' : 'FAIL'} ${ruleId}/${queryName} (${role}): ` + + `expected ${expectedCount}, got ${actual} — ${query}` + ); + + const severityOk = + !expected.severity || actual === 0 || matches.every((m) => m.severity === expected.severity); + const messageOk = + !expected.matchMessage || matches.some((m) => (m.message || '').includes(expected.matchMessage)); + + const resultEntry = { + ruleId, + queryName, + role, + query, + expected: expectedCount, + actual, + severities: matches.map((m) => m.severity), + }; + + if (!ok) { + failures.push( + `[${ruleId}/${queryName}] expected ${expectedCount} "${ruleId}" diagnostic(s), got ${actual} for: ${query}` + ); + } + if (!severityOk) { + failures.push(`[${ruleId}/${queryName}] expected severity "${expected.severity}" for: ${query}`); + } + if (!messageOk) { + failures.push(`[${ruleId}/${queryName}] expected message to contain "${expected.matchMessage}" for: ${query}`); + } + + // Differential: the observed backend behavior must agree with the observed + // detector output through the shared contract (design §3.2, §4.3). A + // rejection-kind query the backend rejected must be one the detector flags; + // a success/advisory query the backend accepted must be one the detector + // passes. This catches drift the two halves would otherwise hide by both + // pinning to the same JSON. + if (backendReport) { + const backendKind = expected.backend && expected.backend.kind; + const expectRejected = backendKind === 'rejection'; + const be = backendReport.get(`${ruleId}::${queryName}`); + if (!be) { + failures.push(`[${ruleId}/${queryName}] no backend report entry (backend did not run this query).`); + } else { + resultEntry.backendRejected = !!be.rejected; + if (!!be.rejected !== expectRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: backend ${be.rejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Trigger/control cross-check against the detector's own verdict. + const detectorFlagged = actual > 0; + if (role === 'trigger' && detectorFlagged !== !!be.rejected) { + failures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + if (role === 'control' && (detectorFlagged || be.rejected)) { + failures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + } + } + + report.results.push(resultEntry); + } + } + + // Nightly-only coverage: every enabled catalog rule must have a contract file. + if (schedule === 'nightly') { + const covered = new Set(contracts.map(({ spec }) => spec.ruleId)); + for (const rule of catalog) { + if (rule.enabled && !covered.has(rule.id)) { + failures.push(`[coverage] enabled catalog rule "${rule.id}" has no contract file.`); + } + } + } + + if (reportPath) { + report.failures = failures; + try { + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); + log(`wrote report to ${reportPath}`); + } catch (error) { + log(`WARN: could not write report to ${reportPath}: ${error.message}`); + } + } + + if (failures.length > 0) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-detector-contract] FAIL: ${failures.length} problem(s):\n- ${failures.join('\n- ')}` + ); + process.exit(1); + } + + log(`PASS: all contracts agreed with the OSD detectors on the candidate bundle (schedule=${schedule}).`); +} + +main();