Add an enterprise-candidate MySQL profile

.github/workflows/release.yml

@@ -6,25 +6,92 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 jobs:
+  source:
+    name: Verify release source
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: python3 tools/check_release.py
+
   verify:
+    needs: source
     uses: ./.github/workflows/ci.yml
 
   build:
-    needs: verify
+    needs: [source, verify]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Find exact-SHA slow result
+        id: slow-run
+        uses: actions/github-script@v7
         with:
-          fetch-depth: 0
+          script: |
+            const artifactName = `ray-doris-slow-result-${context.sha}`;
+            const runs = await github.paginate(
+              github.rest.actions.listWorkflowRunsForRepo,
+              {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                head_sha: context.sha,
+                status: "success",
+                per_page: 100,
+              }
+            );
+            const reusablePath =
+              `${context.repo.owner}/${context.repo.repo}/.github/workflows/slow-integration.yml@`;
+            for (const summary of runs) {
+              const runResponse = await github.rest.actions.getWorkflowRun({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: summary.id,
+              });
+              const run = runResponse.data;
+              const directPath = run.path.split("@", 1)[0];
+              const direct = directPath === ".github/workflows/slow-integration.yml";
+              const reusable = (run.referenced_workflows || []).some((workflow) =>
+                workflow.path.startsWith(reusablePath) &&
+                workflow.sha === context.sha
+              );
+              if (!direct && !reusable) {
+                continue;
+              }
+              const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: run.id,
+                name: artifactName,
+                per_page: 100,
+              });
+              if (artifacts.data.artifacts.some((artifact) => !artifact.expired)) {
+                core.setOutput("run-id", String(run.id));
+                return;
+              }
+            }
+            core.setFailed(`No successful full slow result exists for ${context.sha}`);
+      - uses: actions/download-artifact@v4
+        with:
+          name: ray-doris-slow-result-${{ github.sha }}
+          path: slow-result
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ steps.slow-run.outputs.run-id }}
       - uses: astral-sh/setup-uv@v6
         with:
           python-version: "3.12"
       - run: uv venv --python 3.12
       - run: uv pip install build twine
-      - name: Verify release source and package version
-        run: .venv/bin/python tools/check_release.py
+      - name: Verify exact-SHA slow result
+        run: >-
+          .venv/bin/python tools/check_slow_result.py
+          slow-result/slow-result.json
+          --expected-commit "$GITHUB_SHA"
+          --expected-run-id "${{ steps.slow-run.outputs.run-id }}"
       - run: .venv/bin/python -m build
       - run: .venv/bin/twine check dist/*
       - uses: actions/upload-artifact@v4

.github/workflows/slow-integration.yml

@@ -1,8 +1,36 @@
 name: Slow distributed integration
 
 on:
+  workflow_call:
+    inputs:
+      profile:
+        description: Slow integration profile
+        required: false
+        default: full
+        type: string
+      row_count:
+        description: Rows loaded into each slow integration table
+        required: false
+        default: "10000"
+        type: string
+      stress_seconds:
+        description: Minimum duration of repeated Flight SQL reads
+        required: false
+        default: "5"
+        type: string
+      be_memory_limit:
+        description: Memory limit for each Doris BE container
+        required: false
+        default: 2g
+        type: string
   workflow_dispatch:
     inputs:
+      profile:
+        description: Slow integration profile
+        required: true
+        default: full
+        type: choice
+        options: [full, core]
       row_count:
         description: Rows loaded into each slow integration table
         required: true
@@ -18,6 +46,8 @@ on:
         required: true
         default: "2g"
         type: string
+  schedule:
+    - cron: "17 3 * * 0"
 
 permissions:
   contents: read
@@ -32,9 +62,10 @@ jobs:
     runs-on: [self-hosted, linux, x64, ray-doris-slow-it]
     timeout-minutes: 30
     env:
-      RAY_DORIS_ROW_COUNT: ${{ inputs.row_count }}
-      RAY_DORIS_STRESS_SECONDS: ${{ inputs.stress_seconds }}
-      RAY_DORIS_BE_MEMORY_LIMIT: ${{ inputs.be_memory_limit }}
+      RAY_DORIS_SLOW_PROFILE: ${{ inputs.profile || 'full' }}
+      RAY_DORIS_ROW_COUNT: ${{ inputs.row_count || '10000' }}
+      RAY_DORIS_STRESS_SECONDS: ${{ inputs.stress_seconds || '5' }}
+      RAY_DORIS_BE_MEMORY_LIMIT: ${{ inputs.be_memory_limit || '2g' }}
     steps:
       - uses: actions/checkout@v4
       - name: Run distributed cluster scenario
@@ -47,3 +78,13 @@ jobs:
           name: ray-doris-slow-it-logs
           path: ${{ runner.temp }}/ray-doris-slow-it/*.log
           if-no-files-found: warn
+          overwrite: true
+          retention-days: 30
+      - uses: actions/upload-artifact@v4
+        if: ${{ success() && env.RAY_DORIS_SLOW_PROFILE == 'full' }}
+        with:
+          name: ray-doris-slow-result-${{ github.sha }}
+          path: ${{ runner.temp }}/ray-doris-slow-it/slow-result.json
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 90

CHANGELOG.md

@@ -12,6 +12,13 @@ All notable changes to this project are documented in this file.
 - Mark Python 3.9 as Alpha legacy compatibility and Flight SQL and automatic transport selection as
   experimental.
 - Validate release ancestry against the `master` default branch.
+- Add environment-referenced credentials that resolve independently on the driver and workers
+  without serializing the resolved value.
+- Add a private-CA option and independent timeout for query-plan HTTPS, plus fixed redacted MySQL
+  TLS setup errors.
+- Move distributed worker-retry and backend-failure evidence to the minimum-privilege MySQL path.
+- Bind releases to a successful full slow-suite manifest from the exact release commit and workflow
+  run while keeping FE cluster HA as a deployment responsibility.
 
 ## 0.1.0a1
 

CONTRIBUTING.md

@@ -64,7 +64,10 @@ tests/slow_integration/run.sh
 Size the dedicated host for the requested container limits; the functional profile keeps each BE
 at 2 GiB by default.
 
-The slow suite is manual and is not a required check in the default CI workflow.
+The slow workflow supports manual, reusable, and scheduled full runs. It is not a required check in
+the default CI workflow. A successful full run emits `slow-result.json`; release verification
+requires that artifact from a successful workflow run on the exact release SHA. The `core` profile
+is diagnostic only and never produces release evidence.
 
 ## Documentation checks
 

README.md

@@ -63,7 +63,7 @@ dataset = read_doris(
     table="analytics.events",
     host="doris-fe.example.com",
     user="ray_reader",
-    password="...",
+    password_env="DORIS_PASSWORD",
     columns=["event_id", "created_at", "score"],
     filter="score >= 80",
     tablet_size=32,
@@ -99,13 +99,16 @@ read_doris(
     flight_scheme="grpc",
     user="root",
     password="",
+    password_env=None,
     columns=None,
     filter=None,
     transport="mysql",
     on_query_plan_error="single_task",
     tablet_size=1,
     batch_size=10_000,
     connect_timeout=10.0,
+    query_plan_timeout=None,
+    http_ca_file=None,
     client_kwargs=None,
     flight_options=None,
     concurrency=None,
@@ -114,21 +117,29 @@ read_doris(
 )
 ```
 
-`http_scheme` accepts `http` or `https`. HTTPS requires an HTTPS endpoint, commonly a TLS reverse
-proxy in front of the Doris FE HTTP API. `flight_scheme` accepts `grpc` or `grpc+tls`; configure
-certificates and other ADBC settings with `flight_options`.
+`password_env` stores an environment-variable name in the datasource and resolves its value before
+each driver request and worker connection attempt. It is mutually exclusive with a non-empty
+`password`. The variable name and resolved value are redacted from representations; the resolved
+value is never stored in the serialized datasource or ReadTask.
 
-`connect_timeout` is passed to the `_query_plan` HTTP request, each PyMySQL connection attempt,
-and the ADBC Flight SQL connect RPC. It does not set a deadline for an established MySQL socket
-read or a Flight SQL query/fetch RPC.
+`http_scheme` accepts `http` or `https`. HTTPS requires an HTTPS endpoint, commonly a TLS reverse
+proxy in front of the Doris FE HTTP API. Set `http_ca_file` for a private CA; hostname verification
+remains enabled. `flight_scheme` accepts `grpc` or `grpc+tls`; configure certificates and other ADBC
+settings with `flight_options`.
+
+`connect_timeout` is passed to each PyMySQL connection attempt and the ADBC Flight SQL connect RPC.
+`query_plan_timeout` controls the `_query_plan` HTTP request and defaults to `connect_timeout` when
+unset. Neither value sets a deadline for an established MySQL socket read or a Flight SQL
+query/fetch RPC.
 Configure those limits explicitly when required:
 
 ```python
 dataset = read_doris(
     table="analytics.events",
     host="doris-fe.example.com",
-    transport="auto",
+    transport="mysql",
     connect_timeout=10.0,
+    query_plan_timeout=30.0,
     client_kwargs={"read_timeout": 300, "write_timeout": 30},
     flight_options={
         "adbc.flight.sql.rpc.timeout_seconds.query": "300",
@@ -203,12 +214,18 @@ Ray may call `get_read_tasks()` more than once while constructing one read, so e
 call. Treat an instance as one logical read and create a new instance to discover table or tablet
 changes made later. This planning cache does not provide snapshot isolation.
 
-Ray serializes datasource configuration to workers. Passwords and transport option values are
-redacted from representations and logs, but they still exist in serialized task state. Use this
-package only on a trusted Ray cluster and private network, and inject secrets at runtime. Configure
-MySQL TLS through `client_kwargs`, set `http_scheme="https"` for a protected query-plan endpoint,
-and set `flight_scheme="grpc+tls"` with the required certificate `flight_options` for Flight TLS.
-The defaults are unencrypted and must only be used on a trusted private network.
+Ray serializes datasource configuration to workers. A literal `password` therefore remains in task
+state for compatibility and is suitable only for a trusted cluster. The enterprise-candidate MySQL
+profile uses `password_env`, injects the same variable into the driver and every Ray worker, and
+resolves it separately for each request or connection attempt without serializing the value.
+Transport option values are redacted from representations and logs but remain serialized, so TLS
+paths and other sensitive option values still require a trusted Ray control plane and object store.
+
+Configure MySQL TLS through `client_kwargs`; set `http_scheme="https"` and `http_ca_file` for a
+protected query-plan endpoint. The enterprise-candidate profile also uses
+`on_query_plan_error="error"`, explicit query-plan/MySQL timeouts, and a minimum-privilege reader.
+The defaults are unencrypted and must only be used on a trusted private network. Flight TLS remains
+deployment-specific and experimental.
 
 The Doris reader account needs access to the FE MySQL and HTTP ports and `SELECT` on the target
 internal-catalog table. Flight reads additionally need the FE Flight SQL port. The `_query_plan`
@@ -218,6 +235,11 @@ Tablet planning and task execution do not provide snapshot isolation. Concurrent
 therefore produce a result that reflects different moments across splits. If a Ray task fails after
 reading part of a split, Ray can retry the whole task; the connector does not resume a partial split.
 
+Configure one logical FE hostname that is valid for both HTTPS and MySQL TLS. `ray-doris` validates
+and uses that endpoint but doesn't discover FE members or implement leader election, quorum, health
+checks, or cross-endpoint failover. Production deployments must provide and validate those HA
+properties in Doris and their external load balancer.
+
 The required Doris 4.0.6 integration suite uses the default HTTP endpoint. The distributed suite
 uses the same fixed Doris version, validates native MySQL TLS, and validates certificate-checked
 HTTPS through an HAProxy ingress that forwards to the FE HTTP endpoint. It does not enable Doris
@@ -284,8 +306,9 @@ The opt-in slow suite runs the following isolated topology:
 - certificate-verified HTTPS query planning at the ingress and native Doris MySQL TLS;
 - explicit Arrow Flight SQL reads, with no automatic MySQL fallback;
 - per-BE Flight session and byte counters proving that all three BE services receive traffic;
-- a Ray worker failure after the first Flight block and a retry on another worker;
-- a Doris BE failure, proxy health removal, and a complete read from surviving replicas;
+- a minimum-privilege MySQL read distributed across all three Ray workers;
+- a Ray worker failure after the first MySQL block and a complete-split retry on another worker;
+- a Doris BE failure and a complete MySQL read from surviving replicas;
 - 10,000 rows by default and repeated checksum-validated Flight reads for at least five seconds.
 
 It is excluded from the default pytest discovery paths and from the regular CI workflow. Run it
@@ -326,6 +349,10 @@ Size the dedicated host for the requested container limits. The script refuses t
 existing `ray-doris-it` Compose project, preserves pytest, Ray, Doris, and HAProxy logs, and removes
 only the resources created by that exact project.
 
+Successful `full` runs write `slow-result.json`. The scheduled/reusable workflow uploads it under an
+artifact name bound to the tested commit; release verification accepts only a successful full
+manifest whose commit and workflow run ID exactly match the downloaded artifact source.
+
 See [CONTRIBUTING.md](CONTRIBUTING.md) for the complete checks.
 
 Build and validate the documentation with the commands in the

SECURITY.md

@@ -8,11 +8,17 @@ impact, and any suggested mitigation.
 
 ## Credential handling
 
-`ray-doris` sends credentials to Ray workers as part of datasource configuration. Run it only on a
-trusted Ray cluster and network. Inject secrets at runtime, grant the Doris account only `SELECT`
-on required tables, and avoid placing credentials in source code, logs, or issue reports.
+A literal `password` remains in Ray's serialized datasource and task state for compatibility. The
+enterprise-candidate MySQL profile instead uses `password_env`: only the environment-variable name
+is serialized, and the driver and each worker resolve its value immediately before connecting.
+Inject the same variable into every eligible Ray process, run only on a trusted Ray control plane
+and object store, grant the Doris account only `SELECT` on required tables, and avoid placing
+credentials in source code, logs, or issue reports.
 
 The default MySQL, query-plan HTTP, and Flight URI schemes are unencrypted. Configure MySQL TLS
-through `client_kwargs`, use `http_scheme="https"` with a trusted TLS endpoint for query planning,
-and use `flight_scheme="grpc+tls"` with certificate settings in `flight_options`. Do not send
-production credentials over the default schemes outside a trusted private network.
+through `client_kwargs`, use `http_scheme="https"` with `http_ca_file` when a private CA protects
+query planning, and use `flight_scheme="grpc+tls"` with certificate settings in `flight_options`.
+Do not send production credentials over the default schemes outside a trusted private network.
+
+The connector accepts one logical FE hostname. Doris and the deployment platform remain
+responsible for FE membership, leader election, quorum, health checks, and load-balancer failover.

doc/source/api/api.md

@@ -12,6 +12,12 @@ This page documents the curated public package surface. Signatures and docstring
 
 The connector uses Ray's documented Datasource extension interfaces without importing `ray.data._internal`. Ray classifies `ReadTask` as DeveloperAPI, so compatibility is limited to the tested Ray minor window. MySQL is the production-candidate transport; Flight SQL and `auto` are experimental.
 
+For the enterprise-candidate MySQL profile, use `password_env` rather than a literal password,
+strict MySQL TLS in `client_kwargs`, HTTPS with `http_ca_file`, an explicit
+`query_plan_timeout`, and `on_query_plan_error="error"`. The environment credential is resolved
+again in every driver request and worker attempt; its value isn't stored in the datasource or
+ReadTask payload.
+
 ## Read a Doris table
 
 (ray-doris-api-read-doris)=

doc/source/architecture.md

@@ -38,7 +38,7 @@ Only `read_doris`, `DorisDatasource`, and the exception hierarchy are public pac
 
 ## Plan on the driver
 
-The driver performs two metadata operations. MySQL `DESCRIBE` produces column names, types, and nullability. An authenticated HTTP POST to `/_query_plan` produces predicate-pruned tablet IDs.
+The driver performs two metadata operations. MySQL `DESCRIBE` produces column names, types, and nullability. An authenticated HTTP POST to `/_query_plan` produces predicate-pruned tablet IDs. When `password_env` is configured, each operation resolves the credential immediately before its own network request. A custom `http_ca_file` is loaded into a hostname-verifying TLS context for query planning.
 
 The query-plan client disables redirects and validates both HTTP failures and Doris's body envelope. Doris can report application errors inside an HTTP 200 response, so HTTP status alone isn't enough.
 
@@ -50,7 +50,7 @@ The MySQL reader distinguishes normal EOF from consumer abort. Normal completion
 
 MySQL is the production-candidate data path. Flight SQL and worker-local `auto` selection remain experimental and aren't part of a stable compatibility profile.
 
-Transport option mappings are deep-copied into immutable tuple storage so caller mutation can't change a constructed datasource. Construction also verifies the actual configuration with Ray's worker serialization protocol and rejects unsupported values before network access. The configuration representation exposes option keys for diagnosis but redacts the password, filter, and every option value.
+Transport option mappings are deep-copied into immutable tuple storage so caller mutation can't change a constructed datasource. Construction also verifies the actual configuration with Ray's worker serialization protocol and rejects unsupported values before network access. The configuration representation exposes option keys for diagnosis but redacts the password, password environment name, HTTP CA path, filter, and every option value. An environment credential is resolved per process and never written back into the serialized configuration.
 
 ## Keep SQL generation narrow
 
@@ -71,6 +71,10 @@ Configuration, schema, planning, authentication, permission, and worker read fai
 
 Automatic transport fallback uses an internal setup-only exception. The Flight reader translates eligible dependency, I/O, timeout, and unsupported-operation failures before a reader exists. It doesn't translate stream and conversion errors into fallback signals.
 
+The connector accepts one logical FE host and never discovers cluster members. Doris and the
+deployment platform own FE election, quorum, health checking, and load-balancer failover. The
+connector owns TLS verification and failure classification for the configured endpoint.
+
 ## Define the consistency boundary
 
 The connector doesn't coordinate a Doris transaction across tablet tasks. Metadata discovery, query planning, and split reads are separate requests. Task retry is at-least-once at the split level because a replacement task re-executes the complete split.