diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5adb43a..7832252 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ env: # Cap on-disk cache size to protect host storage. SCCACHE_CACHE_SIZE: 20G CHDB_RUST_REPO: https://github.com/hyperbyte-cloud/chdb-rust.git - CHDB_RUST_REF: feat_arrow_insert + CHDB_RUST_REF: main jobs: doc_links: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 604c6bc..1d647e8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ env: SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache SCCACHE_CACHE_SIZE: 10G CHDB_RUST_REPO: https://github.com/hyperbyte-cloud/chdb-rust.git - CHDB_RUST_REF: feat_arrow_insert + CHDB_RUST_REF: main REGISTRY: ghcr.io IMAGE_NAME: hyperbyte-cloud/hyperbytedb PROXY_IMAGE_NAME: hyperbyte-cloud/hyperbytedb-proxy diff --git a/.gitignore b/.gitignore index 65a92e6..0fad045 100644 --- a/.gitignore +++ b/.gitignore @@ -113,5 +113,5 @@ Desktop.ini .cursorrules .cursor/ hyperbytedb-operator -influx-multiplay -chdb-rust \ No newline at end of file +chdb-rust +docs/audits/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e7ab9d9..8f65b20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,9 +157,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ "arrow-arith", "arrow-array", @@ -178,23 +178,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -202,30 +202,34 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.15.5", - "num", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-cast" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "atoi", @@ -233,15 +237,15 @@ dependencies = [ "chrono", "half", "lexical-core", - "num", + "num-traits", "ryu", ] [[package]] name = "arrow-csv" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ "arrow-array", "arrow-cast", @@ -249,62 +253,66 @@ dependencies = [ "chrono", "csv", "csv-core", - "lazy_static", "regex", ] [[package]] name = "arrow-data" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ipc" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", ] [[package]] name = "arrow-json" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ "arrow-array", "arrow-buffer", @@ -315,9 +323,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,32 +336,32 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "bitflags 2.11.0", ] [[package]] name = "arrow-select" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash 0.8.12", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "54.3.1" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" dependencies = [ "arrow-array", "arrow-buffer", @@ -361,7 +369,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -1293,11 +1301,11 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flatbuffers" -version = "24.12.23" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.0", "rustc_version", ] @@ -1864,6 +1872,7 @@ dependencies = [ "parking_lot", "regex", "reqwest 0.12.28", + "rmp", "rmp-serde", "rocksdb", "serde", @@ -2550,20 +2559,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -2592,28 +2587,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" diff --git a/Dockerfile.proxy-runtime b/Dockerfile.proxy-runtime index 79c146f..a3ca501 100644 --- a/Dockerfile.proxy-runtime +++ b/Dockerfile.proxy-runtime @@ -18,9 +18,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY ${TARGETARCH}/hyperbytedb-proxy /usr/local/bin/hyperbytedb-proxy -EXPOSE 8086 +EXPOSE 8086 8087 ENV HYPERBYTEDB_PROXY_LISTEN=0.0.0.0:8086 \ + HYPERBYTEDB_PROXY_ADMIN_LISTEN=0.0.0.0:8087 \ HYPERBYTEDB_PROXY_BACKEND_PORT=8086 ENTRYPOINT ["/usr/local/bin/hyperbytedb-proxy"] diff --git a/config.toml.example b/config.toml.example index 2023ec5..6942b1c 100644 --- a/config.toml.example +++ b/config.toml.example @@ -38,6 +38,8 @@ schema_cache_max_entries = 10000 [auth] enabled = false +# InfluxDB v1-style ?u=&p= on the query string (unsafe: may appear in logs/Referer) +# allow_query_param_credentials = false [cardinality] max_tag_values_per_measurement = 100000 diff --git a/deploy/compose/grafana/dashboards/hyperbytedb-cluster.json b/deploy/compose/grafana/dashboards/hyperbytedb-cluster.json index 09e946f..9bebb73 100644 --- a/deploy/compose/grafana/dashboards/hyperbytedb-cluster.json +++ b/deploy/compose/grafana/dashboards/hyperbytedb-cluster.json @@ -2181,8 +2181,8 @@ }, { "editorMode": "code", - "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", quantile=\"0.5\"}", - "legendFormat": "{{instance}} - wal read p50", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.5\"}", + "legendFormat": "{{instance}} - wal read prepared p50", "range": true, "refId": "B" }, @@ -2257,6 +2257,13 @@ "range": true, "refId": "N" }, + { + "editorMode": "code", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"native\", quantile=\"0.5\"}", + "legendFormat": "{{instance}} - wal read native p50", + "range": true, + "refId": "P" + }, { "editorMode": "code", "expr": "hyperbytedb_flush_prepare_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.5\"}", @@ -2366,11 +2373,18 @@ }, { "editorMode": "code", - "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", quantile=\"0.99\"}", - "legendFormat": "{{instance}} - wal read p50", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.99\"}", + "legendFormat": "{{instance}} - wal read prepared p99", "range": true, "refId": "B" }, + { + "editorMode": "code", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"native\", quantile=\"0.99\"}", + "legendFormat": "{{instance}} - wal read native p99", + "range": true, + "refId": "P" + }, { "editorMode": "code", "expr": "hyperbytedb_flush_prepare_seconds{instance=~\"$instance\", quantile=\"0.99\"}", diff --git a/deploy/kind/grafana/dashboards/hyperbytedb-cluster.json b/deploy/kind/grafana/dashboards/hyperbytedb-cluster.json index 266ee3c..583a44d 100644 --- a/deploy/kind/grafana/dashboards/hyperbytedb-cluster.json +++ b/deploy/kind/grafana/dashboards/hyperbytedb-cluster.json @@ -2431,8 +2431,8 @@ }, { "editorMode": "code", - "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", quantile=\"0.5\"}", - "legendFormat": "{{instance}} - wal read p50", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.5\"}", + "legendFormat": "{{instance}} - wal read prepared p50", "range": true, "refId": "B" }, @@ -2507,6 +2507,13 @@ "range": true, "refId": "N" }, + { + "editorMode": "code", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"native\", quantile=\"0.5\"}", + "legendFormat": "{{instance}} - wal read native p50", + "range": true, + "refId": "P" + }, { "editorMode": "code", "expr": "hyperbytedb_flush_prepare_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.5\"}", @@ -2616,11 +2623,18 @@ }, { "editorMode": "code", - "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", quantile=\"0.99\"}", - "legendFormat": "{{instance}} - wal read p50", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"prepared\", quantile=\"0.99\"}", + "legendFormat": "{{instance}} - wal read prepared p99", "range": true, "refId": "B" }, + { + "editorMode": "code", + "expr": "hyperbytedb_flush_wal_read_seconds{instance=~\"$instance\", path=\"native\", quantile=\"0.99\"}", + "legendFormat": "{{instance}} - wal read native p99", + "range": true, + "refId": "P" + }, { "editorMode": "code", "expr": "hyperbytedb_flush_prepare_seconds{instance=~\"$instance\", quantile=\"0.99\"}", diff --git a/deploy/kind/manifests/monitoring/prometheus.yaml b/deploy/kind/manifests/monitoring/prometheus.yaml index 80cd233..faa7b6f 100644 --- a/deploy/kind/manifests/monitoring/prometheus.yaml +++ b/deploy/kind/manifests/monitoring/prometheus.yaml @@ -158,7 +158,7 @@ data: action: replace target_label: __address__ regex: (.+) - replacement: $1:8086 + replacement: $1:8087 - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: instance diff --git a/deploy/kind/setup.sh b/deploy/kind/setup.sh index 8c58540..d639705 100755 --- a/deploy/kind/setup.sh +++ b/deploy/kind/setup.sh @@ -481,14 +481,13 @@ show_access_info() { echo -e "${BOLD}════════════════════════════════════════════════${NC}" echo echo -e " ${BOLD}Quick test:${NC}" - echo -e " curl -s http://localhost:8086/ping ${CYAN}# proxied to a backend${NC}" - echo -e " curl -s http://localhost:8086/health ${CYAN}# proxied to a backend${NC}" - echo -e " curl -s http://localhost:8086/readyz ${CYAN}# proxy itself${NC}" - echo -e " curl -s http://localhost:8086/admin/backends ${CYAN}# proxy view of backends${NC}" + echo -e " curl -s 'http://localhost:8086/query?q=SHOW+DATABASES' ${CYAN}# public listener${NC}" + echo -e " curl -s http://localhost:8087/readyz ${CYAN}# proxy admin listener${NC}" + echo -e " curl -s http://localhost:8087/admin/backends ${CYAN}# proxy view of backends${NC}" echo - echo -e " ${BOLD}Inspect cluster:${NC}" + echo -e " ${BOLD}Inspect cluster (direct to DB pods, not via proxy):${NC}" + echo -e " kubectl port-forward -n hyperbytedb pod/hyperbytedb-0 8086:8086" echo -e " curl -s http://localhost:8086/cluster/metrics | jq ." - echo -e " curl -s http://localhost:8086/internal/sync/manifest | jq ." echo echo -e " ${BOLD}Run load test:${NC}" echo -e " bash scripts/load.sh cluster 127.0.0.1 8086" diff --git a/docs/user-guide/cli.md b/docs/user-guide/cli.md index 6424332..e6a42e7 100644 --- a/docs/user-guide/cli.md +++ b/docs/user-guide/cli.md @@ -139,8 +139,8 @@ Global flags work before or after subcommands (e.g. `hyperbytedb-cli query -host |----------|-------------| | `HYPERBYTEDB_HOST` | Server URL | | `HYPERBYTEDB_DATABASE` | Default database | -| `HYPERBYTEDB_USERNAME` | Username | -| `HYPERBYTEDB_PASSWORD` | Password | +| `HYPERBYTEDB_USERNAME` | Username (must be paired with password) | +| `HYPERBYTEDB_PASSWORD` | Password (must be paired with username) | | `HYPERBYTEDB_CLI_CONFIG` | Path to config file | | `HYPERBYTEDB_CLI_HISTORY` | REPL history file (default `~/.hyperbytedb_history`) | @@ -163,6 +163,10 @@ username = "reader" Use `--profile prod` to select a profile. Passwords should be supplied via environment variables or prompts, not stored in the config file. +### Authentication + +The CLI sends credentials via the HTTP `Authorization: Basic …` header (never as `u`/`p` query parameters). **Both username and password are required when either is set** — InfluxDB v1’s token-style `Authorization: Token username:` (username without password) is not supported. Supply `-password` (or `-p ""` to prompt), set `HYPERBYTEDB_PASSWORD`, or use the REPL `auth` command. Anonymous access works when neither credential is configured. + --- ## REPL meta-commands @@ -190,7 +194,7 @@ These are handled locally and are **not** sent to `/query`: | `history` | History hint (use up-arrow) | | `exit`, `quit` | Exit shell | -Any other input is TimeseriesQL. Semicolon-separated statements run in sequence. +Any other input is TimeseriesQL. Semicolon-separated statements run in sequence; semicolons inside single- or double-quoted strings and regex literals do not split the batch. --- diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 5f7c974..d64e117 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -88,8 +88,11 @@ Authentication configuration. | Key | Type | Default | Description | |-----|------|---------|-------------| | `enabled` | boolean | `false` | Enable authentication on `/write` and `/query` | +| `allow_query_param_credentials` | boolean | `false` | Accept InfluxDB v1-style `?u=` / `?p=` credentials on the query string | -When enabled, `/write` and `/query` require valid credentials. Health/metrics and other public routes, plus **admin-only** internal/cluster APIs, are documented in **[Authentication](authentication.md)**. +When enabled, `/write` and `/query` require valid credentials. By default, credentials must be sent via HTTP headers (`Authorization: Basic` or `Authorization: Token`); query-string passwords are disabled because they can leak into access logs, reverse-proxy logs, browser history, and `Referer` headers. Set `allow_query_param_credentials = true` only when you need InfluxDB v1 URL compatibility and accept that risk. + +Health/metrics and other public routes, plus **admin-only** internal/cluster APIs, are documented in **[Authentication](authentication.md)**. --- diff --git a/docs/user-guide/operator/hyperbytedb-proxy.md b/docs/user-guide/operator/hyperbytedb-proxy.md index acb029b..dff3085 100644 --- a/docs/user-guide/operator/hyperbytedb-proxy.md +++ b/docs/user-guide/operator/hyperbytedb-proxy.md @@ -48,9 +48,9 @@ The forward path treats a **503** whose body matches drain/lifecycle substrings --- -## Admin endpoints (proxy process) +## Admin endpoints (admin listener) -These are served on the **same listen address** as client traffic, registered **before** the catch-all proxy so they never get forwarded upstream: +These run on **`HYPERBYTEDB_PROXY_ADMIN_LISTEN`** (default `0.0.0.0:8087`), separate from client traffic. The client-facing Service should expose only the public port so ingress never routes to admin paths. | Path | Method | Purpose | |------|--------|---------| @@ -58,8 +58,20 @@ These are served on the **same listen address** as client traffic, registered ** | `/readyz` | GET | **Readiness** — 200 only when **≥1** backend is `Active`; 503 otherwise | | `/metrics` | GET | Prometheus exposition (if the recorder is installed) | | `/admin/backends` | GET | JSON snapshot of pool: address, health, inflight, probe stats | +| `/admin/backends/{ip}/exclude` | POST | Operator: stop routing to a backend before pod delete | +| `/admin/backends/{ip}/include` | POST | Operator: resume routing after replacement is healthy | +| `/admin/pool` | GET | Full pool status including exclusion flags | -Configure Kubernetes probes to use **`/healthz`** for liveness and **`/readyz`** for readiness so the proxy is not marked ready until at least one HyperbyteDB pod is healthy. +Configure Kubernetes probes against the **admin** port (`8087` by default), not the public Service port. + +## Public listener (client traffic) + +Only **`/write`** and **`/query`** are accepted on **`HYPERBYTEDB_PROXY_LISTEN`** (default `0.0.0.0:8086`). Any other path returns **404** — cluster/internal hyperbytedb routes (`/cluster/*`, `/internal/*`, `/ping`, `/metrics`, …) are not reachable through ingress aimed at the proxy Service. + +| Path | Methods | Purpose | +|------|---------|---------| +| `/write` | any | Proxied to a healthy backend (InfluxDB v1 write) | +| `/query` | any | Proxied to a healthy backend (InfluxDB v1 query) | Hop-by-hop headers (e.g. `Connection`, `Transfer-Encoding`) are stripped on forward; see `HOP_BY_HOP` in [`proxy.rs`](../../../hyperbytedb-proxy/src/proxy.rs). @@ -71,7 +83,8 @@ All settings use the `HYPERBYTEDB_PROXY_` prefix. **Required:** backend service | Variable | Default | Description | |----------|---------|-------------| -| `HYPERBYTEDB_PROXY_LISTEN` | `0.0.0.0:8086` | Bind address for client + admin traffic | +| `HYPERBYTEDB_PROXY_LISTEN` | `0.0.0.0:8086` | Public bind: **`/write`** and **`/query`** only | +| `HYPERBYTEDB_PROXY_ADMIN_LISTEN` | `0.0.0.0:8087` | Admin bind: probes, metrics, `/admin/*` (not on client Service) | | `HYPERBYTEDB_PROXY_BACKEND_SERVICE` | *(required)* | Hostname resolving to backend pod IPs (headless Service) | | `HYPERBYTEDB_PROXY_BACKEND_PORT` | `8086` | Port on each backend | | `HYPERBYTEDB_PROXY_DISCOVERY_INTERVAL_SECS` | `5` | DNS refresh / pool reconcile period | @@ -83,11 +96,16 @@ All settings use the `HYPERBYTEDB_PROXY_` prefix. **Required:** backend service | `HYPERBYTEDB_PROXY_MAX_RETRIES` | `2` | See [Retry semantics](#retry-semantics) | | `HYPERBYTEDB_PROXY_SHUTDOWN_GRACE_SECS` | `30` | After SIGTERM, max time before forced exit watchdog | | `HYPERBYTEDB_PROXY_SELF_IP` | *(unset)* | Optional pod IP (Downward API); that IP is **never** added as a backend (prevents accidental self-proxy loops) | +| `HYPERBYTEDB_PROXY_HTTP2_PRIOR_KNOWLEDGE` | `false` | When `true`, upstream `reqwest` uses cleartext HTTP/2 prior knowledge. HyperbyteDB pods use HTTP/1.1 via `axum::serve` by default — leave this `false` unless every backend is h2-capable | **Logging:** `RUST_LOG` / standard tracing; `LOG_FORMAT=json` enables JSON logs. **Source of truth:** [`config.rs`](../../../hyperbytedb-proxy/src/config.rs) (`ProxyConfig::from_env`). +### Upstream HTTP version + +The proxy forwards to hyperbytedb pods over plain HTTP. HyperbyteDB serves **HTTP/1.1** (`axum::serve` in `runtime/mod.rs`). The upstream client therefore defaults to HTTP/1.1 with ALPN negotiation. Setting `HYPERBYTEDB_PROXY_HTTP2_PRIOR_KNOWLEDGE=true` skips the upgrade and sends an HTTP/2 connection preface immediately — use only when all backends are known to accept cleartext h2; otherwise connections fail at the transport layer and surface as retryable errors. + ### Retry semantics After each **retryable** failure, `attempt` is incremented; the loop continues while `attempt < max_retries`. With the default `max_retries = 2`, a single request can therefore be forwarded up to **three** times (initial try plus two more backends). See the `handle` loop in [`proxy.rs`](../../../hyperbytedb-proxy/src/proxy.rs). @@ -106,8 +124,9 @@ Examples (labels may vary by build): ## Relationship to HyperbyteDB -- Clients keep using **InfluxDB v1** URLs (`/write`, `/query`, `/ping`, …); the proxy forwards them **unchanged** in path and query string. +- Clients use **InfluxDB v1** URLs on the public listener: **`/write`** and **`/query`** only (path and query string forwarded unchanged). - HyperbyteDB configuration (`HYPERBYTEDB__…`) applies to **database pods**, not to the proxy. +- Cluster/admin routes on database pods (`/cluster/*`, `/internal/*`, …) are **not** exposed through the proxy Service; reach them via headless Service or port-forward when needed. - For **TLS** termination at the proxy, terminate TLS on the proxy’s Service and use `http` to backends, or extend the proxy to support outgoing TLS if needed (not in the default crate). --- diff --git a/hyperbytedb-cli/src/client/auth.rs b/hyperbytedb-cli/src/client/auth.rs index 9192b3f..5ac8665 100644 --- a/hyperbytedb-cli/src/client/auth.rs +++ b/hyperbytedb-cli/src/client/auth.rs @@ -1,4 +1,5 @@ use crate::config::ConnectionConfig; +use crate::error::{CliError, Result}; #[derive(Debug, Clone)] pub struct Credentials { @@ -14,17 +15,32 @@ impl Credentials { } } - pub fn authorization_header(&self) -> Option<(String, String)> { - if let (Some(u), Some(p)) = (&self.username, &self.password) { - use base64::Engine as _; - let token = base64::engine::general_purpose::STANDARD.encode(format!("{u}:{p}")); - Some(("Authorization".to_string(), format!("Basic {token}"))) - } else { - self.username - .as_ref() - .map(|u| ("Authorization".to_string(), format!("Token {u}:"))) + /// Require both username and password when either is set. + /// + /// HyperbyteDB CLI uses HTTP Basic authentication only. InfluxDB v1's + /// `Token username:` header (username without password) is not supported. + pub fn validate(&self) -> Result<()> { + match (&self.username, &self.password) { + (Some(_), None) => Err(CliError::Auth( + "password is required when username is set \ + (use -password, HYPERBYTEDB_PASSWORD, or the REPL `auth` command)" + .to_string(), + )), + (None, Some(_)) => Err(CliError::Auth( + "username is required when password is set \ + (use -username, HYPERBYTEDB_USERNAME, or the REPL `auth` command)" + .to_string(), + )), + _ => Ok(()), } } + + pub fn authorization_header(&self) -> Option<(String, String)> { + let (u, p) = (self.username.as_ref()?, self.password.as_ref()?); + use base64::Engine as _; + let token = base64::engine::general_purpose::STANDARD.encode(format!("{u}:{p}")); + Some(("Authorization".to_string(), format!("Basic {token}"))) + } } #[cfg(test)] @@ -47,4 +63,47 @@ mod tests { assert_eq!(creds.username.as_deref(), Some("admin")); assert_eq!(creds.password.as_deref(), Some("secret")); } + + #[test] + fn basic_auth_header_when_both_set() { + let creds = Credentials { + username: Some("admin".to_string()), + password: Some("secret".to_string()), + }; + creds.validate().expect("valid"); + let (k, v) = creds.authorization_header().expect("header"); + assert_eq!(k, "Authorization"); + assert!(v.starts_with("Basic ")); + } + + #[test] + fn rejects_username_without_password() { + let creds = Credentials { + username: Some("admin".to_string()), + password: None, + }; + let err = creds.validate().expect_err("must reject"); + assert!(err.to_string().contains("password is required")); + assert!(creds.authorization_header().is_none()); + } + + #[test] + fn rejects_password_without_username() { + let creds = Credentials { + username: None, + password: Some("secret".to_string()), + }; + let err = creds.validate().expect_err("must reject"); + assert!(err.to_string().contains("username is required")); + } + + #[test] + fn no_credentials_is_valid() { + let creds = Credentials { + username: None, + password: None, + }; + creds.validate().expect("anonymous ok"); + assert!(creds.authorization_header().is_none()); + } } diff --git a/hyperbytedb-cli/src/client/http_transport.rs b/hyperbytedb-cli/src/client/http_transport.rs index fbcec60..b206d1f 100644 --- a/hyperbytedb-cli/src/client/http_transport.rs +++ b/hyperbytedb-cli/src/client/http_transport.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use bytes::Bytes; use http_body_util::{BodyExt, Full}; use hyper::Request; @@ -9,6 +11,11 @@ use crate::error::{CliError, Result}; #[cfg(unix)] use hyperlocal::Uri; +/// Maximum time to establish a TCP/TLS connection. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Maximum time for a full HTTP request (connect + send + read body). +const READ_TIMEOUT: Duration = Duration::from_secs(300); + pub struct RawResponse { pub status: u16, pub body: Vec, @@ -60,18 +67,17 @@ impl HttpBackend { } } - let mut builder = reqwest::Client::builder(); + let mut builder = reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .timeout(READ_TIMEOUT) + .pool_idle_timeout(Duration::from_secs(90)); if config.unsafe_ssl { builder = builder.danger_accept_invalid_certs(true); } if verbose { eprintln!("[verbose] using HTTP {}", config.base_url()); } - Ok(Self::Reqwest( - builder - .build() - .map_err(|e| CliError::Connection(e.to_string()))?, - )) + Ok(Self::Reqwest(builder.build()?)) } pub async fn request(&self, req: HttpRequest<'_>) -> Result { @@ -104,10 +110,7 @@ impl HttpBackend { if let Some(body) = req.body { http_req = http_req.body(body); } - let resp = http_req - .send() - .await - .map_err(|e| CliError::Connection(e.to_string()))?; + let resp = http_req.send().await?; let status = resp.status().as_u16(); let version = resp .headers() @@ -119,11 +122,7 @@ impl HttpBackend { .get("X-Influxdb-Build") .and_then(|v| v.to_str().ok()) .map(str::to_string); - let body = resp - .text() - .await - .map_err(|e| CliError::Connection(e.to_string()))? - .into_bytes(); + let body = resp.text().await?.into_bytes(); Ok(RawResponse { status, body, @@ -139,13 +138,8 @@ impl HttpBackend { for (k, v) in req.headers { builder = builder.header(*k, *v); } - let hyper_req = builder - .body(payload) - .map_err(|e| CliError::Connection(e.to_string()))?; - let resp = client - .request(hyper_req) - .await - .map_err(|e| CliError::Connection(e.to_string()))?; + let hyper_req = builder.body(payload)?; + let resp = client.request(hyper_req).await?; let status = resp.status().as_u16(); let version = resp .headers() @@ -157,13 +151,7 @@ impl HttpBackend { .get("X-Influxdb-Build") .and_then(|v| v.to_str().ok()) .map(str::to_string); - let body = resp - .into_body() - .collect() - .await - .map_err(|e| CliError::Connection(e.to_string()))? - .to_bytes() - .to_vec(); + let body = resp.into_body().collect().await?.to_bytes().to_vec(); Ok(RawResponse { status, body, @@ -174,3 +162,15 @@ impl HttpBackend { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reqwest_client_has_timeouts() { + let cfg = crate::config::ConnectionConfig::default(); + let backend = HttpBackend::from_config(&cfg, false).expect("client"); + assert!(matches!(backend, HttpBackend::Reqwest(_))); + } +} diff --git a/hyperbytedb-cli/src/client/mod.rs b/hyperbytedb-cli/src/client/mod.rs index 7f9848b..31e9d14 100644 --- a/hyperbytedb-cli/src/client/mod.rs +++ b/hyperbytedb-cli/src/client/mod.rs @@ -25,11 +25,13 @@ pub struct HyperbytedbClient { impl HyperbytedbClient { pub fn new(config: &ConnectionConfig, verbose: bool) -> Result { + let credentials = Credentials::from_config(config); + credentials.validate()?; Ok(Self { backend: Arc::new(HttpBackend::from_config(config, verbose)?), base: config.base_url(), config: config.clone(), - credentials: Credentials::from_config(config), + credentials, verbose, }) } diff --git a/hyperbytedb-cli/src/client/query.rs b/hyperbytedb-cli/src/client/query.rs index 1fa41a9..7678302 100644 --- a/hyperbytedb-cli/src/client/query.rs +++ b/hyperbytedb-cli/src/client/query.rs @@ -167,6 +167,15 @@ impl HyperbytedbClient { } pub async fn query_raw(&self, q: &str, opts: &QueryOptions) -> Result { + let resp = if q.len() > 2048 { + self.query_raw_post(q, opts).await? + } else { + self.query_raw_get(q, opts).await? + }; + self.finish_query_raw(resp) + } + + async fn query_raw_get(&self, q: &str, opts: &QueryOptions) -> Result { let query = self.build_query_string(q, opts)?; let mut headers = self.accept_header(opts.format); for (k, v) in self.auth_headers() { @@ -176,10 +185,37 @@ impl HyperbytedbClient { .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect(); + self.request("GET", "/query", &query, &header_refs, None) + .await + } - let resp = self - .request("GET", "/query", &query, &header_refs, None) - .await?; + async fn query_raw_post(&self, q: &str, opts: &QueryOptions) -> Result { + let body_pairs = self.build_body_pairs(q, opts)?; + let encoded = + serde_urlencoded::to_string(&body_pairs).map_err(|e| CliError::Query(e.to_string()))?; + let mut headers = vec![( + "Content-Type".to_string(), + "application/x-www-form-urlencoded".to_string(), + )]; + headers.extend(self.accept_header(opts.format)); + for (k, v) in self.auth_headers() { + headers.push((k, v)); + } + let header_refs: Vec<(&str, &str)> = headers + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + self.request( + "POST", + "/query", + "", + &header_refs, + Some(encoded.into_bytes()), + ) + .await + } + + fn finish_query_raw(&self, resp: super::RawResponse) -> Result { if !(200..300).contains(&resp.status) { let body = String::from_utf8_lossy(&resp.body); return Err(CliError::from_status( diff --git a/hyperbytedb-cli/src/config/mod.rs b/hyperbytedb-cli/src/config/mod.rs index 244b85b..d5798d5 100644 --- a/hyperbytedb-cli/src/config/mod.rs +++ b/hyperbytedb-cli/src/config/mod.rs @@ -101,6 +101,11 @@ impl ConnectionConfig { env_override_opt(&mut self.password, "INFLUX_PASSWORD"); } + /// Validate credential pairs after profile/env/CLI flags are merged. + pub fn validate_credentials(&self) -> Result<()> { + crate::client::Credentials::from_config(self).validate() + } + pub fn base_url(&self) -> String { if self.socket.is_some() { return String::new(); diff --git a/hyperbytedb-cli/src/error.rs b/hyperbytedb-cli/src/error.rs index 631ed0d..1f4fe40 100644 --- a/hyperbytedb-cli/src/error.rs +++ b/hyperbytedb-cli/src/error.rs @@ -1,10 +1,56 @@ use serde_json::Value; use thiserror::Error; +/// Transport-layer failure (HTTP client, socket, body read). +#[derive(Debug, Error)] +pub enum ConnectionError { + #[error(transparent)] + Reqwest(#[from] reqwest::Error), + + #[error(transparent)] + Hyper(#[from] hyper::Error), + + #[error("{0}")] + Other(String), +} + +impl ConnectionError { + pub fn is_timeout(&self) -> bool { + match self { + Self::Reqwest(e) => e.is_timeout(), + _ => false, + } + } +} + +impl From for CliError { + fn from(e: reqwest::Error) -> Self { + Self::Connection(e.into()) + } +} + +impl From for CliError { + fn from(e: hyper::Error) -> Self { + Self::Connection(e.into()) + } +} + +impl From for CliError { + fn from(e: hyper::http::Error) -> Self { + Self::Connection(ConnectionError::Other(e.to_string())) + } +} + +impl From for CliError { + fn from(e: hyper_util::client::legacy::Error) -> Self { + Self::Connection(ConnectionError::Other(e.to_string())) + } +} + #[derive(Debug, Error)] pub enum CliError { - #[error("connection failed: {0}")] - Connection(String), + #[error("connection failed")] + Connection(#[from] ConnectionError), #[error("authentication failed: {0}")] Auth(String), @@ -141,4 +187,11 @@ mod tests { assert!(matches!(err, CliError::Http { status: 400, .. })); assert_eq!(err.to_string(), "HTTP 400: query parse: missing GROUP BY"); } + + #[test] + fn connection_error_has_source() { + let cli_err = CliError::Connection(ConnectionError::Other("refused".into())); + let debug = format!("{cli_err:?}"); + assert!(debug.contains("Connection")); + } } diff --git a/hyperbytedb-cli/src/export/mod.rs b/hyperbytedb-cli/src/export/mod.rs index e6f9faa..5f3b0d0 100644 --- a/hyperbytedb-cli/src/export/mod.rs +++ b/hyperbytedb-cli/src/export/mod.rs @@ -96,6 +96,8 @@ pub async fn run_export(client: &HyperbytedbClient, opts: &ExportOptions) -> Res let tag_keys = list_tag_keys(client, &opts.database).await?; let time_filter = build_time_filter(&opts.start, &opts.end); let mut point_count = 0u64; + let total_measurements = measurements.len(); + let mut skipped: Vec<(String, String)> = Vec::new(); for m in measurements { let q = if time_filter.is_empty() { @@ -106,6 +108,7 @@ pub async fn run_export(client: &HyperbytedbClient, opts: &ExportOptions) -> Res let resp = client.query(&q, &qopts).await?; if resp.has_errors() { + skipped.push((m, resp.format_errors())); continue; } let empty = HashSet::new(); @@ -139,6 +142,22 @@ pub async fn run_export(client: &HyperbytedbClient, opts: &ExportOptions) -> Res } } + if !skipped.is_empty() { + eprintln!( + "export warning: skipped {} of {} measurement(s):", + skipped.len(), + total_measurements + ); + for (measurement, err) in &skipped { + eprintln!(" {measurement}: {err}"); + } + return Err(CliError::Export(format!( + "partial export: {} of {} measurements failed", + skipped.len(), + total_measurements + ))); + } + eprintln!("export complete: {point_count} points"); Ok(point_count) } diff --git a/hyperbytedb-cli/src/lib.rs b/hyperbytedb-cli/src/lib.rs index a46807f..d9f806c 100644 --- a/hyperbytedb-cli/src/lib.rs +++ b/hyperbytedb-cli/src/lib.rs @@ -12,5 +12,5 @@ pub mod session; pub use client::HyperbytedbClient; pub use config::ConnectionConfig; -pub use error::CliError; +pub use error::{CliError, ConnectionError}; pub use session::Session; diff --git a/hyperbytedb-cli/src/main.rs b/hyperbytedb-cli/src/main.rs index 10bfe0a..09e02d4 100644 --- a/hyperbytedb-cli/src/main.rs +++ b/hyperbytedb-cli/src/main.rs @@ -266,7 +266,7 @@ async fn main() -> ExitCode { match run().await { Ok(()) => ExitCode::SUCCESS, Err(e) => { - eprintln!("{e}"); + eprintln!("{e:#}"); ExitCode::from(1) } } @@ -361,6 +361,7 @@ fn build_connection(cli: &Cli) -> hyperbytedb_cli::error::Result String { } pub fn format_column(response: &QueryResponse) -> String { - let style = DisplayStyle::detect(); + format_column_with_style(response, DisplayStyle::detect()) +} + +fn format_column_with_style(response: &QueryResponse, style: DisplayStyle) -> String { let mut out = String::new(); for (result_idx, result) in response.results.iter().enumerate() { @@ -84,7 +92,8 @@ pub fn format_column(response: &QueryResponse) -> String { let mut table = Table::new(); table .load_preset(UTF8_NO_BORDERS) - .set_content_arrangement(ContentArrangement::Dynamic); + .set_content_arrangement(ContentArrangement::Dynamic) + .set_width(MIN_TABLE_WIDTH); table.set_header( series .columns @@ -147,7 +156,8 @@ fn format_series_header( .into_iter() .map(|(key, value)| { if style.color { - format!("\x1b[1;36m{key}\x1b[0m\x1b[2;36m={value}\x1b[0m") + // Keep `key=value` contiguous so copy/paste and tests stay stable. + format!("\x1b[36m{key}={value}\x1b[0m") } else { format!("{key}={value}") } @@ -275,7 +285,7 @@ mod tests { error: None, }], }; - let out = format_column(&resp); + let out = format_column_with_style(&resp, DisplayStyle { color: false }); assert!(out.contains("cpu")); assert!(out.contains("host=srv1")); assert!(out.contains("42")); diff --git a/hyperbytedb-cli/src/repl/mod.rs b/hyperbytedb-cli/src/repl/mod.rs index f0acae3..9bd9171 100644 --- a/hyperbytedb-cli/src/repl/mod.rs +++ b/hyperbytedb-cli/src/repl/mod.rs @@ -1,5 +1,6 @@ mod complete; mod meta; +mod split; use std::sync::Arc; use std::time::Instant; @@ -93,14 +94,10 @@ pub async fn run_repl(mut session: Session) -> Result<()> { } pub async fn execute_query(session: &Session, client: &HyperbytedbClient, q: &str) -> Result<()> { - let statements: Vec<&str> = q - .split(';') - .map(str::trim) - .filter(|s| !s.is_empty()) - .collect(); + let statements = split::split_statements(q); let start = Instant::now(); - for stmt in statements { + for stmt in &statements { let opts = QueryOptions { db: session.effective_database().map(|s| s.to_string()), retention_policy: session.retention_policy.clone(), @@ -117,12 +114,12 @@ pub async fn execute_query(session: &Session, client: &HyperbytedbClient, q: &st }; if session.format == crate::session::OutputFormat::Csv { - let raw = client.query_raw(stmt, &opts).await?; + let raw = client.query_raw(stmt.as_str(), &opts).await?; print!("{raw}"); continue; } - let resp = client.query(stmt, &opts).await?; + let resp = client.query(stmt.as_str(), &opts).await?; if resp.has_errors() { return Err(CliError::Query(resp.format_errors())); } diff --git a/hyperbytedb-cli/src/repl/split.rs b/hyperbytedb-cli/src/repl/split.rs new file mode 100644 index 0000000..4e74b6d --- /dev/null +++ b/hyperbytedb-cli/src/repl/split.rs @@ -0,0 +1,156 @@ +/// Split a TimeseriesQL batch on semicolons, respecting string and regex literals. +/// +/// Mirrors server-side `timeseriesql::lexer::split_statements` closely enough +/// for REPL / `-execute` multi-statement batches. +pub fn split_statements(input: &str) -> Vec { + let mut statements = Vec::new(); + let mut start = 0usize; + let mut i = 0usize; + let bytes = input.as_bytes(); + let mut in_single = false; + let mut in_double = false; + let mut in_regex = false; + let mut begin_depth = 0i32; + let mut prev_sig: Option = None; + + while i < bytes.len() { + let c = bytes[i] as char; + + if in_regex { + if c == '\\' { + i += 2; + continue; + } + if c == '/' { + in_regex = false; + prev_sig = Some('/'); + } + i += 1; + continue; + } + if in_single { + if c == '\'' { + if i + 1 < bytes.len() && bytes[i + 1] == b'\'' { + i += 2; + continue; + } + in_single = false; + prev_sig = Some('\''); + } + i += 1; + continue; + } + if in_double { + if c == '"' { + if i + 1 < bytes.len() && bytes[i + 1] == b'"' { + i += 2; + continue; + } + in_double = false; + prev_sig = Some('"'); + } + i += 1; + continue; + } + + let at_word_boundary = i == 0 || !is_ident_continue(bytes[i - 1] as char); + + match c { + '\'' => in_single = true, + '"' => in_double = true, + '/' if matches!(prev_sig, Some('~') | Some('(') | Some(',') | Some('=')) => { + in_regex = true; + } + ';' if begin_depth == 0 => { + let slice = input[start..i].trim(); + if !slice.is_empty() { + statements.push(slice.to_string()); + } + start = i + 1; + } + _ if is_ident_start(c) && at_word_boundary && matches_keyword_at(input, i, "BEGIN") => { + begin_depth += 1 + } + _ if is_ident_start(c) + && at_word_boundary + && begin_depth > 0 + && matches_keyword_at(input, i, "END") => + { + begin_depth -= 1; + } + _ => {} + } + if !c.is_whitespace() { + prev_sig = Some(c); + } + i += 1; + } + + let tail = input[start..].trim(); + if !tail.is_empty() { + statements.push(tail.to_string()); + } + statements +} + +fn is_ident_start(c: char) -> bool { + c.is_ascii_alphabetic() || c == '_' +} + +fn is_ident_continue(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' +} + +fn matches_keyword_at(input: &str, start: usize, kw: &str) -> bool { + let rest = input.as_bytes().get(start..); + let Some(rest) = rest else { + return false; + }; + if rest.len() < kw.len() || !rest[..kw.len()].eq_ignore_ascii_case(kw.as_bytes()) { + return false; + } + !matches!(rest.get(kw.len()), Some(b) if is_ident_continue(*b as char)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_simple_statements() { + let stmts = split_statements("SHOW DATABASES; SHOW MEASUREMENTS"); + assert_eq!(stmts, vec!["SHOW DATABASES", "SHOW MEASUREMENTS"]); + } + + #[test] + fn ignores_semicolon_in_single_quoted_string() { + let stmts = split_statements("SELECT * FROM cpu WHERE msg = 'a;b'; SHOW DATABASES"); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains("'a;b'")); + assert_eq!(stmts[1], "SHOW DATABASES"); + } + + #[test] + fn ignores_semicolon_in_double_quoted_identifier() { + let stmts = split_statements(r#"SELECT * FROM "meas;ure"; SHOW DATABASES"#); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains(r#""meas;ure""#)); + } + + #[test] + fn ignores_semicolon_in_regex() { + let stmts = split_statements(r#"SELECT * FROM cpu WHERE host =~ /a;b/; SHOW DATABASES"#); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains("/a;b/")); + assert_eq!(stmts[1], "SHOW DATABASES"); + } + + #[test] + fn begin_end_block_keeps_internal_semicolons() { + let input = "BEGIN; SELECT 1; SELECT 2; END; SHOW DATABASES"; + let stmts = split_statements(input); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].starts_with("BEGIN")); + assert_eq!(stmts[1], "SHOW DATABASES"); + } +} diff --git a/hyperbytedb-proxy/Dockerfile b/hyperbytedb-proxy/Dockerfile index 8366224..f4e2458 100644 --- a/hyperbytedb-proxy/Dockerfile +++ b/hyperbytedb-proxy/Dockerfile @@ -70,9 +70,11 @@ COPY --from=builder /artifacts/hyperbytedb-proxy /usr/local/bin/hyperbytedb-prox # Same default port as hyperbytedb itself: clients (Grafana, Telegraf) flip # their endpoint URL to point at the proxy Service with no port change. -EXPOSE 8086 +# Admin/probes listen on 8087 and are not exposed on the client Service. +EXPOSE 8086 8087 ENV HYPERBYTEDB_PROXY_LISTEN=0.0.0.0:8086 \ + HYPERBYTEDB_PROXY_ADMIN_LISTEN=0.0.0.0:8087 \ HYPERBYTEDB_PROXY_BACKEND_PORT=8086 ENTRYPOINT ["/usr/local/bin/hyperbytedb-proxy"] diff --git a/hyperbytedb-proxy/benches/support/mod.rs b/hyperbytedb-proxy/benches/support/mod.rs index f1a4929..1a43035 100644 --- a/hyperbytedb-proxy/benches/support/mod.rs +++ b/hyperbytedb-proxy/benches/support/mod.rs @@ -12,6 +12,7 @@ use tokio::runtime::Runtime; pub fn test_config() -> ProxyConfig { ProxyConfig { listen_addr: "127.0.0.1:0".into(), + admin_listen_addr: "127.0.0.1:0".into(), backend_service: "bench.local".into(), backend_port: 8086, discovery_interval: Duration::from_secs(5), @@ -23,6 +24,7 @@ pub fn test_config() -> ProxyConfig { max_retries: 2, shutdown_grace: Duration::from_secs(30), self_ip: None, + http2_prior_knowledge: false, } } diff --git a/hyperbytedb-proxy/src/admin.rs b/hyperbytedb-proxy/src/admin.rs index c63a966..a71da89 100644 --- a/hyperbytedb-proxy/src/admin.rs +++ b/hyperbytedb-proxy/src/admin.rs @@ -1,13 +1,14 @@ -//! Proxy-local endpoints. These aren't proxied; they answer about the proxy -//! itself. +//! Proxy-local endpoints on the **admin listener** (`HYPERBYTEDB_PROXY_ADMIN_LISTEN`). +//! These are not proxied; they answer about the proxy itself. //! //! - `GET /healthz` — liveness, always 200 once the process is up. //! - `GET /readyz` — readiness, 200 only when ≥1 backend is routable (Active and not excluded). //! - `GET /metrics` — Prometheus exposition. //! - `GET /admin/backends` — JSON dump of the current pool, for debugging. +//! - `POST /admin/backends/{ip}/exclude|include` — operator-driven routing control. +//! - `GET /admin/pool` — pool status including exclusion flags. //! -//! Routes are chosen so they can be allowlisted before the catch-all proxy -//! handler, with no risk of colliding with a hyperbytedb path. +//! The public listener exposes only `/write` and `/query`. use std::net::IpAddr; use std::sync::Arc; diff --git a/hyperbytedb-proxy/src/config.rs b/hyperbytedb-proxy/src/config.rs index 5b833fb..05728fc 100644 --- a/hyperbytedb-proxy/src/config.rs +++ b/hyperbytedb-proxy/src/config.rs @@ -7,12 +7,19 @@ use std::time::Duration; use anyhow::{Context, Result}; +/// Default admin bind when `HYPERBYTEDB_PROXY_ADMIN_LISTEN` is unset. +pub const DEFAULT_ADMIN_LISTEN: &str = "0.0.0.0:8087"; + /// All knobs the proxy understands. #[derive(Debug, Clone)] pub struct ProxyConfig { - /// `host:port` we bind for client traffic _and_ admin endpoints. + /// `host:port` for client-facing InfluxDB v1 traffic (`/write`, `/query` only). pub listen_addr: String, + /// `host:port` for proxy-local admin: probes, metrics, `/admin/*`. + /// Not exposed on the client Service; reachable via pod IP or loopback. + pub admin_listen_addr: String, + /// DNS name that resolves to one A record per backend pod (typically a /// Kubernetes headless Service: `-headless..svc.cluster.local`). pub backend_service: String, @@ -57,6 +64,12 @@ pub struct ProxyConfig { /// that would otherwise let the proxy proxy to itself and infinitely /// recurse until the pod OOMs. pub self_ip: Option, + + /// When true, the upstream `reqwest` client uses HTTP/2 prior knowledge + /// (no ALPN upgrade). HyperbyteDB pods speak HTTP/1.1 via `axum::serve` + /// today, so this defaults to `false`. Enable only when every backend is + /// known to accept cleartext HTTP/2. + pub http2_prior_knowledge: bool, } impl ProxyConfig { @@ -65,6 +78,7 @@ impl ProxyConfig { pub fn from_env() -> Result { Ok(Self { listen_addr: env_or("HYPERBYTEDB_PROXY_LISTEN", "0.0.0.0:8086"), + admin_listen_addr: env_or("HYPERBYTEDB_PROXY_ADMIN_LISTEN", DEFAULT_ADMIN_LISTEN), backend_service: env_required("HYPERBYTEDB_PROXY_BACKEND_SERVICE")?, backend_port: env_u32("HYPERBYTEDB_PROXY_BACKEND_PORT", 8086)? as u16, discovery_interval: Duration::from_secs(env_u32( @@ -93,6 +107,7 @@ impl ProxyConfig { 30, )? as u64), self_ip: env_optional_ip("HYPERBYTEDB_PROXY_SELF_IP")?, + http2_prior_knowledge: env_bool("HYPERBYTEDB_PROXY_HTTP2_PRIOR_KNOWLEDGE", false)?, }) } } @@ -124,3 +139,32 @@ fn env_u32(key: &str, default: u32) -> Result { Err(_) => Ok(default), } } + +fn env_bool(key: &str, default: bool) -> Result { + match env::var(key) { + Ok(v) if v.is_empty() => Ok(default), + Ok(v) => match v.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Ok(true), + "0" | "false" | "no" | "off" => Ok(false), + _ => Err(anyhow::anyhow!("env var {key}={v} is not a valid bool")), + }, + Err(_) => Ok(default), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_admin_listen_is_separate_from_public() { + assert_eq!(DEFAULT_ADMIN_LISTEN, "0.0.0.0:8087"); + } + + #[test] + fn http2_prior_knowledge_defaults_false() { + let v = env_bool("HYPERBYTEDB_PROXY_HTTP2_PRIOR_KNOWLEDGE_TEST_UNSET", false) + .expect("parse bool"); + assert!(!v); + } +} diff --git a/hyperbytedb-proxy/src/lib.rs b/hyperbytedb-proxy/src/lib.rs index 530e7a9..0486659 100644 --- a/hyperbytedb-proxy/src/lib.rs +++ b/hyperbytedb-proxy/src/lib.rs @@ -18,6 +18,7 @@ use axum::routing::{any, get, post}; use metrics_exporter_prometheus::PrometheusBuilder; use tokio::net::TcpListener; use tokio::signal::unix::{SignalKind, signal}; +use tokio::sync::watch; use tower_http::trace::TraceLayer; use crate::admin::AdminState; @@ -29,6 +30,15 @@ pub async fn run() -> Result<()> { init_tracing(); let cfg = ProxyConfig::from_env()?; tracing::info!(?cfg, "hyperbytedb-proxy starting"); + if cfg.http2_prior_knowledge { + tracing::warn!( + "HYPERBYTEDB_PROXY_HTTP2_PRIOR_KNOWLEDGE=true: upstream client requires \ + cleartext HTTP/2; hyperbytedb pods use HTTP/1.1 unless you have enabled h2 \ + elsewhere" + ); + } else { + tracing::info!("upstream client uses HTTP/1.1 with ALPN negotiation (hyperbytedb default)"); + } let prometheus_handle = PrometheusBuilder::new() .install_recorder() @@ -52,9 +62,16 @@ pub async fn run() -> Result<()> { prometheus: Some(prometheus_handle), }; - // Admin routes (kubelet probes, metrics, debug). Kept in a separate - // sub-router with no TraceLayer so a not-yet-warm /readyz returning 503 - // doesn't show up as ERROR in the logs every 2s during startup. + // Public listener: InfluxDB v1 write/query only. Cluster/internal routes on + // hyperbytedb pods are never reachable through ingress aimed at this port. + let public_router = Router::new() + .route("/write", any(proxy::handle)) + .route("/query", any(proxy::handle)) + .fallback(proxy::not_found) + .with_state(proxy_state) + .layer(TraceLayer::new_for_http()); + + // Admin listener: kubelet probes, Prometheus, operator backend exclusion. let admin_router = Router::new() .route("/healthz", get(admin::healthz)) .route("/readyz", get(admin::readyz)) @@ -65,41 +82,44 @@ pub async fn run() -> Result<()> { .route("/admin/pool", get(admin::pool_status)) .with_state(admin_state); - // Order matters: admin routes first, then the catch-all proxy fallback. - // The TraceLayer only wraps the proxy fallback so admin probes stay quiet. - let app = admin_router.fallback_service( - Router::new() - .fallback(any(proxy::handle)) - .with_state(proxy_state) - .layer(TraceLayer::new_for_http()), - ); + let public_listener = TcpListener::bind(&cfg.listen_addr) + .await + .with_context(|| format!("bind public listener {}", cfg.listen_addr))?; + tracing::info!(addr = %cfg.listen_addr, "public listener (write/query only)"); - let listener = TcpListener::bind(&cfg.listen_addr) + let admin_listener = TcpListener::bind(&cfg.admin_listen_addr) .await - .with_context(|| format!("bind {}", cfg.listen_addr))?; - tracing::info!(addr = %cfg.listen_addr, "proxy listening"); + .with_context(|| format!("bind admin listener {}", cfg.admin_listen_addr))?; + tracing::info!(addr = %cfg.admin_listen_addr, "admin listener (probes/metrics/admin)"); let shutdown_grace = cfg.shutdown_grace; - let serve = axum::serve(listener, app).with_graceful_shutdown(async move { + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + tokio::spawn(async move { wait_for_shutdown_signal().await; tracing::info!( grace_secs = shutdown_grace.as_secs(), - "shutdown signal received; draining" + "shutdown signal received; draining in-flight requests" ); - tokio::spawn(async move { - tokio::time::sleep(shutdown_grace).await; - tracing::warn!( - grace_secs = shutdown_grace.as_secs(), - "drain grace expired; forcing exit" - ); - std::process::exit(0); - }); + let _ = shutdown_tx.send(true); }); - if let Err(e) = serve.await { - tracing::error!(error = %e, "proxy server error"); - return Err(e.into()); - } + let mut public_shutdown = shutdown_rx.clone(); + let pool_for_drain = Arc::clone(&pool); + let public_serve = + axum::serve(public_listener, public_router).with_graceful_shutdown(async move { + let _ = public_shutdown.changed().await; + drain_inflight(&pool_for_drain, shutdown_grace).await; + }); + + let mut admin_shutdown = shutdown_rx; + let admin_serve = + axum::serve(admin_listener, admin_router).with_graceful_shutdown(async move { + let _ = admin_shutdown.changed().await; + }); + + tokio::try_join!(public_serve, admin_serve)?; + tracing::info!("proxy shut down cleanly"); Ok(()) } @@ -113,6 +133,28 @@ async fn wait_for_shutdown_signal() { } } +/// Poll aggregate backend inflight until zero or `grace` elapses. +async fn drain_inflight(pool: &BackendPool, grace: std::time::Duration) { + let deadline = tokio::time::Instant::now() + grace; + loop { + let inflight = pool.total_inflight().await; + if inflight == 0 { + tracing::info!("all in-flight proxy requests drained"); + break; + } + if tokio::time::Instant::now() >= deadline { + tracing::warn!( + grace_secs = grace.as_secs(), + inflight, + "drain grace expired with requests still in flight" + ); + break; + } + tracing::debug!(inflight, "waiting for in-flight proxy requests to drain"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } +} + fn init_tracing() { let filter = tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { tracing_subscriber::EnvFilter::new("info,hyperbytedb_proxy=debug,tower_http=info") diff --git a/hyperbytedb-proxy/src/pool.rs b/hyperbytedb-proxy/src/pool.rs index d93b13e..d51ffd0 100644 --- a/hyperbytedb-proxy/src/pool.rs +++ b/hyperbytedb-proxy/src/pool.rs @@ -124,6 +124,13 @@ impl BackendPool { self.excluded.read().await.contains(ip) } + /// Sum of in-flight proxied requests across all backends. Used during + /// graceful shutdown to wait for active forwards to finish. + pub async fn total_inflight(&self) -> usize { + let snap = self.snapshot().await; + snap.iter().map(|b| b.inflight()).sum() + } + /// JSON-serializable snapshot of the pool for `GET /admin/pool`. pub async fn pool_status(&self) -> Vec { let snap = self.snapshot().await; diff --git a/hyperbytedb-proxy/src/proxy.rs b/hyperbytedb-proxy/src/proxy.rs index 1d5f2a8..a60ea50 100644 --- a/hyperbytedb-proxy/src/proxy.rs +++ b/hyperbytedb-proxy/src/proxy.rs @@ -9,7 +9,9 @@ //! `max_retries` times. //! 4. On final failure surface 503 to the client. +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use std::time::Instant; use axum::body::Body; @@ -17,8 +19,12 @@ use axum::extract::{Request, State}; use axum::http::{HeaderName, StatusCode}; use axum::response::{IntoResponse, Response}; use bytes::Bytes; +use futures::Stream; +use futures::StreamExt; +use futures::ready; use http_body_util::BodyExt; +use crate::backend::InflightGuard; use crate::pool::BackendPool; /// Headers that hop-by-hop semantics (RFC 7230 §6.1) say we must not forward. @@ -44,16 +50,18 @@ pub struct ProxyState { impl ProxyState { pub fn new(pool: Arc) -> anyhow::Result { let cfg = pool.config(); - let client = reqwest::Client::builder() + let mut client_builder = reqwest::Client::builder() .timeout(cfg.request_timeout) // Per-host pool sized to absorb a moderate burst without // re-handshaking; keep_alive_while_idle keeps connections warm // across the typical inter-request gap of a Grafana refresh. .pool_max_idle_per_host(64) .pool_idle_timeout(std::time::Duration::from_secs(90)) - .tcp_nodelay(true) - .http2_prior_knowledge() // hyperbytedb supports h2; saves the upgrade - .build()?; + .tcp_nodelay(true); + if cfg.http2_prior_knowledge { + client_builder = client_builder.http2_prior_knowledge(); + } + let client = client_builder.build()?; Ok(Self { pool, client }) } } @@ -64,14 +72,11 @@ pub async fn handle(State(state): State, req: Request) -> Response { let cfg = state.pool.config(); // Buffer the request body once. We may need to send it more than once if - // a backend returns a transient failure mid-restart. - // - // For very large writes this would be a regression — fortunately - // hyperbytedb's `/write` body cap is `server.max_body_size_bytes` (25 MiB - // by default), so buffering is bounded and predictable. + // a backend returns a transient failure mid-restart. Wrapped in `Arc` so + // retries clone a pointer, not the payload. let (parts, body) = req.into_parts(); let body_bytes = match body.collect().await { - Ok(c) => c.to_bytes(), + Ok(c) => Arc::new(c.to_bytes()), Err(e) => { tracing::warn!(error = %e, "failed to read incoming request body"); return error_response(StatusCode::BAD_REQUEST, "could not read request body"); @@ -111,7 +116,7 @@ pub async fn handle(State(state): State, req: Request) -> Response { let mut current = first; loop { - let _guard = current.enter(); + let guard = current.enter(); let url = format!("{}{}", current.origin, path_query); tracing::debug!( attempt, @@ -126,7 +131,7 @@ pub async fn handle(State(state): State, req: Request) -> Response { &url, &parts.method, &parts.headers, - body_bytes.clone(), + Arc::clone(&body_bytes), ) .await; @@ -141,7 +146,7 @@ pub async fn handle(State(state): State, req: Request) -> Response { .increment(1); metrics::histogram!("hyperbytedb_proxy_request_duration_seconds") .record(started.elapsed().as_secs_f64()); - return resp; + return attach_inflight_guard(resp, guard); } ForwardOutcome::Retryable { status, msg } => { tracing::info!( @@ -153,6 +158,8 @@ pub async fn handle(State(state): State, req: Request) -> Response { ); last_status = status; last_err = msg; + // Attempt failed before a response was handed to the client. + drop(guard); } ForwardOutcome::Fatal(resp) => { metrics::counter!( @@ -160,7 +167,7 @@ pub async fn handle(State(state): State, req: Request) -> Response { "outcome" => "fatal", ) .increment(1); - return resp; + return attach_inflight_guard(resp, guard); } } @@ -217,7 +224,7 @@ async fn forward_once( url: &str, method: &http::Method, headers: &http::HeaderMap, - body: Bytes, + body: Arc, ) -> ForwardOutcome { let mut rb = client.request(method.clone(), url); for (name, value) in headers { @@ -230,7 +237,7 @@ async fn forward_once( rb = rb.header(name, value); } if !body.is_empty() { - rb = rb.body(body); + rb = rb.body(body.as_ref().clone()); } let resp = match rb.send().await { @@ -248,60 +255,143 @@ async fn forward_once( let upstream_status = StatusCode::from_u16(status.as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - // Buffer body so we can return it as a fixed-size axum body. Streaming - // would be nicer for large query results, but the simpler form keeps - // the retry-with-buffered-request path consistent. - let resp_headers = resp.headers().clone(); - let body_bytes = match resp.bytes().await { - Ok(b) => b, - Err(e) => { + // 502/504 from the backend itself = transient infra problem upstream; + // try another node. Body is not needed for the retry decision. + if matches!(upstream_status.as_u16(), 502 | 504,) { + return ForwardOutcome::Retryable { + status: Some(upstream_status), + msg: Some("backend returned bad-gateway/timeout".into()), + }; + } + + // 503 with the well-known draining marker → another backend may serve us. + // These envelopes are small JSON; buffer to inspect before deciding. + if upstream_status == StatusCode::SERVICE_UNAVAILABLE { + let resp_headers = resp.headers().clone(); + let body_bytes = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + return ForwardOutcome::Retryable { + status: Some(upstream_status), + msg: Some(format!("response body read: {e}")), + }; + } + }; + if looks_like_drain(&body_bytes) { return ForwardOutcome::Retryable { status: Some(upstream_status), - msg: Some(format!("response body read: {e}")), + msg: Some("backend reports draining/syncing".into()), }; } - }; - - // 503 with the well-known draining marker → another backend may serve us. - if upstream_status == StatusCode::SERVICE_UNAVAILABLE && looks_like_drain(&body_bytes) { - return ForwardOutcome::Retryable { - status: Some(upstream_status), - msg: Some("backend reports draining/syncing".into()), + let resp = build_buffered_response(upstream_status, &resp_headers, body_bytes); + return if upstream_status.is_client_error() { + ForwardOutcome::Fatal(resp) + } else { + ForwardOutcome::Ok(resp) }; } - // 502/504 from the backend itself = transient infra problem upstream; - // try another node. - if matches!(upstream_status.as_u16(), 502 | 504,) { - return ForwardOutcome::Retryable { - status: Some(upstream_status), - msg: Some("backend returned bad-gateway/timeout".into()), + if upstream_status.is_client_error() { + let resp_headers = resp.headers().clone(); + let body_bytes = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + return ForwardOutcome::Retryable { + status: Some(upstream_status), + msg: Some(format!("response body read: {e}")), + }; + } }; + return ForwardOutcome::Fatal(build_buffered_response( + upstream_status, + &resp_headers, + body_bytes, + )); } - let mut out = Response::builder().status(upstream_status); - let out_headers = out.headers_mut().expect("response builder has headers map"); - for (name, value) in resp_headers.iter() { + // Success and other non-retryable responses: stream upstream body through + // without buffering the full payload in proxy memory. + ForwardOutcome::Ok(build_streaming_response(upstream_status, resp)) +} + +fn build_buffered_response( + status: StatusCode, + resp_headers: &reqwest::header::HeaderMap, + body_bytes: Bytes, +) -> Response { + let mut out = Response::builder().status(status); + copy_upstream_headers( + resp_headers, + out.headers_mut().expect("response builder has headers map"), + ); + out.body(Body::from(body_bytes)) + .expect("axum response body construction is infallible") +} + +fn build_streaming_response(status: StatusCode, resp: reqwest::Response) -> Response { + let resp_headers = resp.headers().clone(); + let mut out = Response::builder().status(status); + copy_upstream_headers( + &resp_headers, + out.headers_mut().expect("response builder has headers map"), + ); + let stream = resp + .bytes_stream() + .map(|result| result.map_err(std::io::Error::other)); + out.body(Body::from_stream(stream)) + .expect("axum response body construction is infallible") +} + +fn copy_upstream_headers(upstream: &reqwest::header::HeaderMap, out_headers: &mut http::HeaderMap) { + for (name, value) in upstream.iter() { if HOP_BY_HOP .iter() .any(|h| name.as_str().eq_ignore_ascii_case(h)) { continue; } - out_headers.insert( - HeaderName::from_bytes(name.as_ref()).expect("valid hyper header name"), - value.clone(), - ); + match HeaderName::from_bytes(name.as_ref()) { + Ok(header_name) => { + out_headers.insert(header_name, value.clone()); + } + Err(_) => { + tracing::warn!(header = ?name, "skipping upstream header with invalid name"); + } + } } - let resp = out - .body(Body::from(body_bytes)) - .expect("axum response body construction is infallible"); +} - if upstream_status.is_client_error() { - // 4xx is the client's problem; don't burn retries. - ForwardOutcome::Fatal(resp) - } else { - ForwardOutcome::Ok(resp) +/// Keep the backend inflight counter elevated until the response body is fully +/// consumed (including streamed query results). +fn attach_inflight_guard(resp: Response, guard: InflightGuard) -> Response { + let (parts, body) = resp.into_parts(); + let stream = body + .into_data_stream() + .map(|result| result.map_err(std::io::Error::other)); + let guarded = GuardedBodyStream { + inner: Box::pin(stream), + guard: Some(guard), + }; + Response::from_parts(parts, Body::from_stream(guarded)) +} + +struct GuardedBodyStream { + inner: Pin> + Send>>, + guard: Option, +} + +impl Stream for GuardedBodyStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + match ready!(this.inner.as_mut().poll_next(cx)) { + Some(item) => Poll::Ready(Some(item)), + None => { + this.guard.take(); + Poll::Ready(None) + } + } } } @@ -328,3 +418,11 @@ fn error_response(status: StatusCode, msg: &str) -> Response { ); (status, [("content-type", "application/json")], body).into_response() } + +/// Reject paths outside the public InfluxDB v1 surface. +pub async fn not_found() -> Response { + error_response( + StatusCode::NOT_FOUND, + "only /write and /query are exposed on the public listener", + ) +} diff --git a/hyperbytedb/Cargo.toml b/hyperbytedb/Cargo.toml index ab2dafd..9bfe0cc 100644 --- a/hyperbytedb/Cargo.toml +++ b/hyperbytedb/Cargo.toml @@ -24,10 +24,11 @@ tokio = { version = "1", features = [ # Line protocol parsing (from InfluxDB IOx) influxdb-line-protocol = "2" +rmp = "0.8" rmp-serde = "1.3" -# Arrow (columnar msgpack ingest uses RecordBatch for validation benchmarks) -arrow = { version = "54", features = ["ipc"] } +# Arrow (columnar msgpack ingest + chDB RecordBatch inserts; must match chdb-rust) +arrow = { version = "59", features = ["ipc"] } # chdb — local checkout: provides the Arrow C Data Interface insert path # (`arrow_insert`, `Session::connection`) that the published 1.4.0 provides. diff --git a/hyperbytedb/src/adapters/chdb/catalog.rs b/hyperbytedb/src/adapters/chdb/catalog.rs index 6f7b378..4b6df32 100644 --- a/hyperbytedb/src/adapters/chdb/catalog.rs +++ b/hyperbytedb/src/adapters/chdb/catalog.rs @@ -41,21 +41,23 @@ pub async fn persist_default_database_metadata( match result { Ok(qr) => qr .data_utf8() - .map_err(|e| HyperbytedbError::Chdb(e.to_string())), - Err(e) => Err(HyperbytedbError::Chdb(e.to_string())), + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))), + Err(e) => Err(HyperbytedbError::Chdb( + crate::error::ChainedError::from_error(e), + )), } }) }) .await .map_err(|e| { - HyperbytedbError::Internal(format!("chDB default database query join error: {e}")) + HyperbytedbError::Internal(format!("chDB default database query join error: {e}").into()) })??; let uuid = raw.lines().next().unwrap_or_default().trim(); if uuid.is_empty() { - return Err(HyperbytedbError::Chdb( - "default database uuid missing from system.databases".into(), - )); + return Err(HyperbytedbError::Chdb(crate::error::ChainedError::new( + "default database uuid missing from system.databases", + ))); } write_default_database_sql(session_path, uuid)?; @@ -74,16 +76,22 @@ pub async fn reload_persisted_tables(session: &SharedSession) -> Result Result Result<(), HyperbytedbE } let default_sql = fs::read_to_string(&default_sql_path).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to read chDB database metadata {}: {e}", - default_sql_path.display() - )) + HyperbytedbError::Chdb( + format!( + "failed to read chDB database metadata {}: {e}", + default_sql_path.display() + ) + .into(), + ) })?; let Some(uuid) = parse_default_database_uuid(&default_sql) else { return Ok(()); @@ -130,34 +140,43 @@ fn repair_atomic_default_symlink(session_path: &Path) -> Result<(), HyperbytedbE let store_dir = store_path_for_uuid(session_path, uuid); if !store_dir.is_dir() { - return Err(HyperbytedbError::Chdb(format!( - "expected chDB store directory for default database uuid {uuid}: {}", - store_dir.display() - ))); + return Err(HyperbytedbError::Chdb( + format!( + "expected chDB store directory for default database uuid {uuid}: {}", + store_dir.display() + ) + .into(), + )); } fs::remove_dir_all(&default_meta).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to remove chDB metadata dir {} before symlink repair: {e}", - default_meta.display() - )) + HyperbytedbError::Chdb( + format!( + "failed to remove chDB metadata dir {} before symlink repair: {e}", + default_meta.display() + ) + .into(), + ) })?; #[cfg(unix)] { std::os::unix::fs::symlink(&store_dir, &default_meta).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to create chDB metadata symlink {} -> {}: {e}", - default_meta.display(), - store_dir.display() - )) + HyperbytedbError::Chdb( + format!( + "failed to create chDB metadata symlink {} -> {}: {e}", + default_meta.display(), + store_dir.display() + ) + .into(), + ) })?; } #[cfg(not(unix))] { - return Err(HyperbytedbError::Chdb( - "chDB cold start metadata repair requires Unix symlinks".into(), - )); + return Err(HyperbytedbError::Chdb(crate::error::ChainedError::new( + "chDB cold start metadata repair requires Unix symlinks", + ))); } tracing::info!( @@ -171,22 +190,28 @@ fn repair_atomic_default_symlink(session_path: &Path) -> Result<(), HyperbytedbE fn write_default_database_sql(session_path: &Path, uuid: &str) -> Result<(), HyperbytedbError> { let default_sql = session_path.join("metadata/default.sql"); let Some(parent) = default_sql.parent() else { - return Err(HyperbytedbError::Chdb( - "default.sql path has no parent directory".to_string(), - )); + return Err(HyperbytedbError::Chdb(crate::error::ChainedError::new( + "default.sql path has no parent directory", + ))); }; fs::create_dir_all(parent).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to create chDB metadata dir {}: {e}", - parent.display() - )) + HyperbytedbError::Chdb( + format!( + "failed to create chDB metadata dir {}: {e}", + parent.display() + ) + .into(), + ) })?; let statement = format!("ATTACH DATABASE default ENGINE=Atomic UUID '{uuid}'\n"); fs::write(&default_sql, statement).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to write chDB database metadata {}: {e}", - default_sql.display() - )) + HyperbytedbError::Chdb( + format!( + "failed to write chDB database metadata {}: {e}", + default_sql.display() + ) + .into(), + ) })?; tracing::info!( path = %default_sql.display(), @@ -243,13 +268,15 @@ async fn query_tab_separated( match result { Ok(qr) => qr .data_utf8() - .map_err(|e| HyperbytedbError::Chdb(e.to_string())), - Err(e) => Err(HyperbytedbError::Chdb(e.to_string())), + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))), + Err(e) => Err(HyperbytedbError::Chdb( + crate::error::ChainedError::from_error(e), + )), } }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB catalog query join error: {e}")))? + .map_err(|e| HyperbytedbError::Internal(format!("chDB catalog query join error: {e}").into()))? } async fn execute_statement(session: &SharedSession, sql: &str) -> Result<(), HyperbytedbError> { @@ -259,11 +286,13 @@ async fn execute_statement(session: &SharedSession, sql: &str) -> Result<(), Hyp pool.with_connection(|conn| { execute_connection(conn, &sql, OutputFormat::TabSeparated) .map(|_| ()) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB catalog attach join error: {e}")))? + .map_err(|e| { + HyperbytedbError::Internal(format!("chDB catalog attach join error: {e}").into()) + })? } #[cfg(test)] diff --git a/hyperbytedb/src/adapters/chdb/connection_pool.rs b/hyperbytedb/src/adapters/chdb/connection_pool.rs index 176d16f..140279f 100644 --- a/hyperbytedb/src/adapters/chdb/connection_pool.rs +++ b/hyperbytedb/src/adapters/chdb/connection_pool.rs @@ -35,18 +35,22 @@ pub struct ChdbConnectionPool { impl ChdbConnectionPool { /// Open `pool_size` connections to `data_path` (same `--path` for all). pub fn open(data_path: &str, pool_size: usize) -> Result { - std::fs::create_dir_all(data_path) - .map_err(|e| HyperbytedbError::Chdb(format!("failed to create chDB data dir: {e}")))?; + std::fs::create_dir_all(data_path).map_err(|e| { + HyperbytedbError::Chdb(format!("failed to create chDB data dir: {e}").into()) + })?; let pool_size = clamp_pool_size(pool_size); let path_arg = format!("--path={data_path}"); let mut slots = Vec::with_capacity(pool_size); for i in 0..pool_size { let conn = Connection::open(&[&path_arg]).map_err(|e| { - HyperbytedbError::Chdb(format!( - "failed to open chDB connection {} / {pool_size} at {data_path}: {e}", - i + 1 - )) + HyperbytedbError::Chdb( + format!( + "failed to open chDB connection {} / {pool_size} at {data_path}: {e}", + i + 1 + ) + .into(), + ) })?; slots.push(Mutex::new(conn)); } diff --git a/hyperbytedb/src/adapters/chdb/native_adapter.rs b/hyperbytedb/src/adapters/chdb/native_adapter.rs index bc95de9..aef42d0 100644 --- a/hyperbytedb/src/adapters/chdb/native_adapter.rs +++ b/hyperbytedb/src/adapters/chdb/native_adapter.rs @@ -37,11 +37,12 @@ use std::sync::Arc; use lru::LruCache; +use arrow::array::Array; use arrow::array::{ - ArrayRef, DictionaryArray, Float64Builder, Int32Array, Int64Builder, RecordBatch, StringArray, - StringBuilder, TimestampNanosecondArray, UInt8Builder, UInt64Array, UInt64Builder, + ArrayRef, DictionaryArray, Float64Array, Float64Builder, Int32Array, Int64Builder, RecordBatch, + StringArray, StringBuilder, TimestampNanosecondArray, UInt8Builder, UInt64Array, UInt64Builder, }; -use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use arrow::datatypes::{DataType, Field, Int32Type, Schema, TimeUnit}; use async_trait::async_trait; use chdb_rust::InsertOptions; use chdb_rust::arrow_insert::insert_record_batch_direct; @@ -351,16 +352,18 @@ impl ChdbNativeAdapter { let result = execute_connection(conn, sql, chdb_rust::format::OutputFormat::TabSeparated); match result { - Ok(qr) => qr - .data_utf8() - .map_err(|e| HyperbytedbError::Chdb(e.to_string())), - Err(e) => Err(HyperbytedbError::Chdb(e.to_string())), + Ok(qr) => qr.data_utf8().map_err(|e| { + HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e)) + }), + Err(e) => Err(HyperbytedbError::Chdb( + crate::error::ChainedError::from_error(e), + )), } }) }) .await .map_err(|e| { - HyperbytedbError::Internal(format!("chDB materialization sync join error: {e}")) + HyperbytedbError::Internal(format!("chDB materialization sync join error: {e}").into()) })??; let attached: HashSet = raw @@ -379,7 +382,7 @@ impl ChdbNativeAdapter { for key in keys { let fact = unquoted_table_name(&key.db, &key.rp, &key.measurement); let series = unquoted_series_table_name(&key.db, &key.rp, &key.measurement); - if attached.contains(&fact) + if attached.contains(fact.as_str()) && fact_writers.get_mut(&key).is_some_and(|schema| { if !schema.materialized { schema.materialized = true; @@ -391,7 +394,7 @@ impl ChdbNativeAdapter { { synced += 1; } - if attached.contains(&series) + if attached.contains(series.as_str()) && let Some(schema) = series_writers.get_mut(&key) && !schema.materialized { @@ -445,8 +448,8 @@ impl ChdbNativeAdapter { /// True if `series_id` is already registered (dimension row inserted + /// persisted) for this table. fn series_known(&self, key: &TableKey, sid: u64) -> bool { - let mut map = self.known_series.write(); - map.get(key).is_some_and(|s| s.contains(&sid)) + let map = self.known_series.read(); + map.peek(key).is_some_and(|s| s.contains(&sid)) } /// Record `series_id`s as registered for this table. @@ -460,6 +463,30 @@ impl ChdbNativeAdapter { } } + /// Persist newly-flushed series + tag metadata durably before updating the + /// in-memory dedup cache. + async fn persist_new_series_metadata( + &self, + key: &TableKey, + entries: &[(u64, BTreeMap)], + ) -> Result<(), HyperbytedbError> { + let Some(meta) = &self.metadata else { + return Ok(()); + }; + if entries.is_empty() { + return Ok(()); + } + meta.register_series_batch(&key.db, &key.rp, &key.measurement, entries) + .await?; + let mut tag_pairs: Vec<(String, String)> = Vec::new(); + for (_, tags) in entries { + for (k, v) in tags { + tag_pairs.push((k.clone(), v.clone())); + } + } + backfill_tag_metadata(meta, &key.db, &key.rp, &key.measurement, tag_pairs).await + } + async fn ddl_mutex(&self, key: &TableKey) -> Arc> { let mut map = self.ddl_locks.lock().await; if !map.contains(key) { @@ -483,9 +510,9 @@ impl ChdbNativeAdapter { let result = execute_connection(conn, &sql, chdb_rust::format::OutputFormat::TabSeparated); match result { - Ok(qr) => qr - .data_utf8() - .map_err(|e| HyperbytedbError::Chdb(e.to_string())), + Ok(qr) => qr.data_utf8().map_err(|e| { + HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e)) + }), Err(e) => { let msg = e.to_string(); if msg.contains("doesn't exist") @@ -494,14 +521,16 @@ impl ChdbNativeAdapter { { Ok(String::new()) } else { - Err(HyperbytedbError::Chdb(msg)) + Err(HyperbytedbError::Chdb(crate::error::ChainedError::new(msg))) } } } }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB describe join error: {e}")))??; + .map_err(|e| { + HyperbytedbError::Internal(format!("chDB describe join error: {e}").into()) + })??; let skip = ["time", "origin_node_id", "ingest_seq", "series_id"]; let mut out = HashMap::new(); @@ -539,7 +568,7 @@ impl ChdbNativeAdapter { let pool = self.session.pool()?; tokio::task::spawn_blocking(move || run_sync(&pool, &sql)) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB DDL join error: {e}")))? + .map_err(|e| HyperbytedbError::Internal(format!("chDB DDL join error: {e}").into()))? } /// Compute the union of tag keys across `points`, then ensure the @@ -550,8 +579,12 @@ impl ChdbNativeAdapter { key: &TableKey, points: &[P], ) -> Result { - let table = quoted_table_name(&key.db, &key.rp, &key.measurement); - let series_table = quoted_series_table_name(&key.db, &key.rp, &key.measurement); + let table = quoted_table_name(&key.db, &key.rp, &key.measurement) + .as_str() + .to_string(); + let series_table = quoted_series_table_name(&key.db, &key.rp, &key.measurement) + .as_str() + .to_string(); let table_unquoted = unquoted_table_name(&key.db, &key.rp, &key.measurement); // Discover required columns from this batch. A fixed-schema telemetry @@ -694,59 +727,67 @@ impl ChdbNativeAdapter { }); } - // Slow path: serialise DDL on this table (both fact + series). + // Slow path: serialise DDL planning on this table (both fact + series). + // Hold the mutex only while reading caches and building statements; release + // before chDB I/O so slow DDL does not block other flush tasks. let ddl_lock = self.ddl_mutex(key).await; - let _guard = ddl_lock.lock().await; + let (create_fact, create_series, ddl_stmts) = { + let _guard = ddl_lock.lock().await; + + // Re-read caches under the DDL lock; another writer may have already + // added the columns we needed while we were waiting. + let cached = { + let mut guard = self.schemas.write(); + schema_cache_get(&mut guard, key) + } + .unwrap_or_default(); + let series_cached = { + let mut guard = self.series_schemas.write(); + schema_cache_get(&mut guard, key) + } + .unwrap_or_default(); - // Re-read caches under the DDL lock; another writer may have already - // added the columns we needed while we were waiting. - let cached = { - let mut guard = self.schemas.write(); - schema_cache_get(&mut guard, key) - } - .unwrap_or_default(); - let series_cached = { - let mut guard = self.series_schemas.write(); - schema_cache_get(&mut guard, key) - } - .unwrap_or_default(); + // Fact table: fields only. ALTERs run only against an already-existing + // table (materialized) or a warmed-from-metadata entry; a cold CREATE + // already includes every field, so no redundant ADD COLUMN. + let create_fact = if !cached.materialized { + Some(build_create_table_sql(&table, &field_phys, None)) + } else { + None + }; + let mut fact_alters = if cached.materialized || !cached.columns.is_empty() { + let mut alters = build_alter_add_field_columns(&table, &cached, &field_phys); + alters.extend(build_alter_reconcile_field_widening( + &table, + &cached, + &field_phys, + )); + alters + } else { + Vec::new() + }; - // Fact table: fields only. ALTERs run only against an already-existing - // table (materialized) or a warmed-from-metadata entry; a cold CREATE - // already includes every field, so no redundant ADD COLUMN. - let create_fact = if !cached.materialized { - Some(build_create_table_sql(&table, &field_phys, None)) - } else { - None - }; - let mut fact_alters = if cached.materialized || !cached.columns.is_empty() { - let mut alters = build_alter_add_field_columns(&table, &cached, &field_phys); - alters.extend(build_alter_reconcile_field_widening( - &table, - &cached, - &field_phys, - )); - alters - } else { - Vec::new() - }; + // Series (dimension) table: tag columns only. + let create_series = if !series_cached.materialized { + Some(build_create_series_table_sql(&series_table, &tag_phys)) + } else { + None + }; + let mut series_alters = + if series_cached.materialized || !series_cached.columns.is_empty() { + build_alter_add_series_columns(&series_table, &series_cached, &tag_phys) + } else { + Vec::new() + }; + if !series_cached.materialized && !series_cached.columns.is_empty() { + // Warmed-from-metadata series table may still carry LowCardinality + // tags that have since crossed TAG_LOW_CARDINALITY_MAX. + series_alters.extend(build_alter_reconcile_tag_strings(&series_table, &tag_phys)); + } - // Series (dimension) table: tag columns only. - let create_series = if !series_cached.materialized { - Some(build_create_series_table_sql(&series_table, &tag_phys)) - } else { - None - }; - let mut series_alters = if series_cached.materialized || !series_cached.columns.is_empty() { - build_alter_add_series_columns(&series_table, &series_cached, &tag_phys) - } else { - Vec::new() + fact_alters.append(&mut series_alters); + (create_fact, create_series, fact_alters) }; - if !series_cached.materialized && !series_cached.columns.is_empty() { - // Warmed-from-metadata series table may still carry LowCardinality - // tags that have since crossed TAG_LOW_CARDINALITY_MAX. - series_alters.extend(build_alter_reconcile_tag_strings(&series_table, &tag_phys)); - } if let Some(sql) = create_fact { tracing::debug!(table = %table_unquoted, "creating chDB native fact table"); @@ -756,28 +797,30 @@ impl ChdbNativeAdapter { tracing::debug!(table = %table_unquoted, "creating chDB native series table"); self.execute(sql).await?; } - fact_alters.append(&mut series_alters); - for stmt in fact_alters { + for stmt in ddl_stmts { tracing::debug!(table = %table_unquoted, alter = %stmt, "altering chDB native table"); self.execute(stmt).await?; } // Update the fact cache (fields) and series cache (tags). { - let mut writers = self.schemas.write(); - let entry = schema_cache_entry(&mut writers, key); - for (_, phys, d) in &field_phys { - entry.columns.insert(phys.clone(), ColumnKind::Field(*d)); + let _guard = ddl_lock.lock().await; + { + let mut writers = self.schemas.write(); + let entry = schema_cache_entry(&mut writers, key); + for (_, phys, d) in &field_phys { + entry.columns.insert(phys.clone(), ColumnKind::Field(*d)); + } + entry.materialized = true; } - entry.materialized = true; - } - { - let mut writers = self.series_schemas.write(); - let entry = schema_cache_entry(&mut writers, key); - for (_, phys, kind) in &tag_phys { - entry.columns.insert(phys.clone(), *kind); + { + let mut writers = self.series_schemas.write(); + let entry = schema_cache_entry(&mut writers, key); + for (_, phys, kind) in &tag_phys { + entry.columns.insert(phys.clone(), *kind); + } + entry.materialized = true; } - entry.materialized = true; } if let Err(e) = catalog::persist_default_database_metadata(&self.session).await { @@ -831,52 +874,27 @@ impl ChdbNativeAdapter { batch, InsertOptions::default_bulk(), ) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) }) .await .map_err(|e| { - HyperbytedbError::Internal(format!("chDB series insert join error: {e}")) + HyperbytedbError::Internal(format!("chDB series insert join error: {e}").into()) })??; } else { let sql = build_series_insert_sql(ensured, &new_series); self.execute(sql).await?; } - // Mark known first so concurrent/subsequent flushes skip the insert. + let entries: Vec<(u64, BTreeMap)> = new_series + .iter() + .map(|(id, p)| (*id, p.tags.clone())) + .collect(); + self.persist_new_series_metadata(key, &entries).await?; + // Mark known only after durable metadata write succeeds so concurrent + // flushes cannot skip re-registration when RocksDB persist fails. self.mark_series(key, new_series.iter().map(|(id, _)| *id)); - // Persist to the metadata layer (local-deterministic, never via Raft). - // Non-fatal: the dimension rows are already in chDB; on persist failure - // a post-restart warm simply re-registers (idempotent). - if let Some(meta) = &self.metadata { - let entries: Vec<(u64, BTreeMap)> = new_series - .iter() - .map(|(id, p)| (*id, p.tags.clone())) - .collect(); - if let Err(e) = meta - .register_series_batch(&key.db, &key.rp, &key.measurement, &entries) - .await - { - tracing::warn!(error = %e, "failed to persist series metadata; re-registers after restart"); - } else { - let mut tag_pairs: Vec<(String, String)> = Vec::new(); - for (_, p) in &new_series { - for (k, v) in &p.tags { - tag_pairs.push((k.clone(), v.clone())); - } - } - if let Err(e) = - backfill_tag_metadata(meta, &key.db, &key.rp, &key.measurement, tag_pairs).await - { - tracing::warn!( - error = %e, - "failed to backfill tag metadata from flushed series" - ); - } - } - } - Ok(()) } @@ -959,20 +977,24 @@ impl ChdbNativeAdapter { return Ok(values.len()); }; + let batch_values: HashSet = points + .iter() + .filter_map(|p| p.borrow().tags.get(tag_key).cloned()) + .collect(); + let base = meta .count_tag_values(db, rp, tag_key, Some(measurement)) .await?; - let mut novel: HashSet = HashSet::new(); - for p in points { - if let Some(v) = p.borrow().tags.get(tag_key) - && !meta - .tag_value_is_known(db, rp, measurement, tag_key, v) - .await? + let mut novel = 0usize; + for v in &batch_values { + if !meta + .tag_value_is_known(db, rp, measurement, tag_key, v) + .await? { - novel.insert(v.clone()); + novel += 1; } } - Ok(base + novel.len()) + Ok(base + novel) } /// Create fact + `_series` tables for `meta` when they do not yet exist. @@ -987,8 +1009,10 @@ impl ChdbNativeAdapter { rp: rp.to_string(), measurement: meta.name.clone(), }; - let table = quoted_table_name(db, rp, &meta.name); - let series_table = quoted_series_table_name(db, rp, &meta.name); + let table = quoted_table_name(db, rp, &meta.name).as_str().to_string(); + let series_table = quoted_series_table_name(db, rp, &meta.name) + .as_str() + .to_string(); let field_name_set: HashSet<&str> = meta.field_types.keys().map(String::as_str).collect(); let tag_phys: Vec<(String, String, ColumnKind)> = meta @@ -1025,45 +1049,64 @@ impl ChdbNativeAdapter { } let ddl_lock = self.ddl_mutex(&key).await; - let _guard = ddl_lock.lock().await; + let (create_fact, create_series) = { + let _guard = ddl_lock.lock().await; - let cached = { - let mut guard = self.schemas.write(); - schema_cache_get(&mut guard, &key) - } - .unwrap_or_default(); - let series_cached = { - let mut guard = self.series_schemas.write(); - schema_cache_get(&mut guard, &key) - } - .unwrap_or_default(); + let cached = { + let mut guard = self.schemas.write(); + schema_cache_get(&mut guard, &key) + } + .unwrap_or_default(); + let series_cached = { + let mut guard = self.series_schemas.write(); + schema_cache_get(&mut guard, &key) + } + .unwrap_or_default(); + + let create_fact = if !cached.materialized { + Some(build_create_table_sql( + &table, + &field_phys, + summing_columns_from_meta(meta), + )) + } else { + None + }; + let create_series = if !series_cached.materialized { + Some(build_create_series_table_sql(&series_table, &tag_phys)) + } else { + None + }; + (create_fact, create_series) + }; - if !cached.materialized { - let sql = build_create_table_sql(&table, &field_phys, summing_columns_from_meta(meta)); + if let Some(sql) = create_fact { tracing::debug!(table = %meta.name, "creating chDB native fact table for MV destination"); self.execute(sql).await?; } - if !series_cached.materialized { - let sql = build_create_series_table_sql(&series_table, &tag_phys); + if let Some(sql) = create_series { tracing::debug!(table = %meta.name, "creating chDB native series table for MV destination"); self.execute(sql).await?; } { - let mut writers = self.schemas.write(); - let entry = schema_cache_entry(&mut writers, &key); - for (_, phys, d) in &field_phys { - entry.columns.insert(phys.clone(), ColumnKind::Field(*d)); + let _guard = ddl_lock.lock().await; + { + let mut writers = self.schemas.write(); + let entry = schema_cache_entry(&mut writers, &key); + for (_, phys, d) in &field_phys { + entry.columns.insert(phys.clone(), ColumnKind::Field(*d)); + } + entry.materialized = true; } - entry.materialized = true; - } - { - let mut writers = self.series_schemas.write(); - let entry = schema_cache_entry(&mut writers, &key); - for (_, phys, kind) in &tag_phys { - entry.columns.insert(phys.clone(), *kind); + { + let mut writers = self.series_schemas.write(); + let entry = schema_cache_entry(&mut writers, &key); + for (_, phys, kind) in &tag_phys { + entry.columns.insert(phys.clone(), *kind); + } + entry.materialized = true; } - entry.materialized = true; } if let Err(e) = catalog::persist_default_database_metadata(&self.session).await { @@ -1121,6 +1164,95 @@ impl ChdbNativeAdapter { }) } + /// Build a prepared WAL slot directly from a columnar wire batch, avoiding + /// intermediate `Vec` expansion on the ingest hot path. + #[cfg(feature = "columnar-ingest")] + pub async fn build_prepared_wal_slot_from_columnar( + &self, + db: &str, + rp: &str, + origin_node_id: u64, + wire: &crate::application::columnar_msgpack::ColumnarMsgpackBatch, + precision: Option<&str>, + ) -> Result { + use crate::application::columnar_msgpack::columnar_timestamps_ns; + use crate::domain::prepared_wal::{PreparedMeasurementBatch, PreparedWalSlot}; + use crate::domain::series::series_id; + + if wire.field.is_empty() { + return Err(HyperbytedbError::ColumnarMsgpackParse { + reason: "field name must be non-empty".into(), + }); + } + + let n = wire.values.len(); + if n == 0 { + return Ok(PreparedWalSlot { + database: db.to_string(), + retention_policy: rp.to_string(), + origin_node_id, + measurements: Vec::new(), + }); + } + + let rep = columnar_representative_point(wire, precision)?; + let key = TableKey { + db: db.to_string(), + rp: rp.to_string(), + measurement: wire.measurement.clone(), + }; + let ensured = self.ensure_table(&key, std::slice::from_ref(&rep)).await?; + let sid = series_id(&wire.measurement, &wire.tags); + let ts_ns = columnar_timestamps_ns(wire, precision)?; + + let mut min_time = i64::MAX; + let mut max_time = i64::MIN; + for &t in &ts_ns { + min_time = min_time.min(t); + max_time = max_time.max(t); + } + + let origins = vec![origin_node_id; n]; + let sids = vec![sid; n]; + let seqs: Vec = (0..n).map(|i| i as u64).collect(); + let batch = build_columnar_fact_record_batch( + &ensured, + &ts_ns, + &origins, + &seqs, + &sids, + &wire.field, + &wire.values, + )?; + + let new_series_batch = if self.series_known(&key, sid) { + None + } else if self.use_arrow { + Some(Arc::new(build_series_record_batch( + &ensured, + &[(sid, &rep)], + )?)) + } else { + None + }; + + Ok(PreparedWalSlot { + database: db.to_string(), + retention_policy: rp.to_string(), + origin_node_id, + measurements: vec![PreparedMeasurementBatch { + measurement: wire.measurement.clone(), + table_name: ensured.table.clone(), + series_table_name: ensured.series_table.clone(), + batch: Arc::new(batch), + row_count: n, + min_time, + max_time, + new_series_batch, + }], + }) + } + async fn prepare_measurement_batch + Sync>( &self, key: &TableKey, @@ -1199,22 +1331,27 @@ impl ChdbNativeAdapter { batch, InsertOptions::default_bulk(), ) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB series insert join error: {e}")))??; + .map_err(|e| { + HyperbytedbError::Internal(format!("chDB series insert join error: {e}").into()) + })??; if series_batch.num_rows() > 0 { + let entries = series_entries_from_batch(series_batch, &ensured.tag_phys)?; + self.persist_new_series_metadata(key, &entries).await?; let sid_col = series_batch .column(0) .as_any() .downcast_ref::() - .ok_or_else(|| HyperbytedbError::Internal("series batch sid col".into()))?; - let ids: Vec = (0..series_batch.num_rows()) - .map(|i| sid_col.value(i)) - .collect(); - self.mark_series(key, ids.iter().copied()); + .ok_or_else(|| { + HyperbytedbError::Internal(crate::error::ChainedError::new( + "series batch sid col", + )) + })?; + self.mark_series(key, (0..series_batch.num_rows()).map(|i| sid_col.value(i))); } Ok(()) } @@ -1292,12 +1429,14 @@ impl PointsSinkPort for ChdbNativeAdapter { tokio::task::spawn_blocking(move || { pool.with_connection(|conn| { insert_record_batch_direct(conn, &table, batch, InsertOptions::default_bulk()) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| { + HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e)) + }) }) }) .await .map_err(|e| { - HyperbytedbError::Internal(format!("chDB arrow insert join error: {e}")) + HyperbytedbError::Internal(format!("chDB arrow insert join error: {e}").into()) })??; histogram!("hyperbytedb_flush_sink_chdb_insert_seconds") .record(insert_start.elapsed().as_secs_f64()); @@ -1345,26 +1484,21 @@ impl PointsSinkPort for ChdbNativeAdapter { measurement: batch.measurement.clone(), }; + // Re-align legacy sparse prepared batches (pre-coalesce WAL entries) + // to the current metadata-driven table schema before insert. + let ensured = self.ensure_table::(&key, &[]).await?; + if let Some(ref series_batch) = batch.new_series_batch { - let ensured = EnsuredTable { - table: batch.table_name.clone(), - series_table: batch.series_table_name.clone(), - tag_phys: Vec::new(), - field_phys: Vec::new(), - }; self.insert_prepared_series(&key, &ensured, series_batch) .await?; } if !self.use_arrow { - return Err(HyperbytedbError::Internal( - "prepared batch path requires Arrow inserts".into(), - )); + return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "prepared batch path requires Arrow inserts", + ))); } - // Re-align legacy sparse prepared batches (pre-coalesce WAL entries) - // to the current metadata-driven table schema before insert. - let ensured = self.ensure_table::(&key, &[]).await?; let padded = pad_record_batch_to_ensured(&batch.batch, &ensured)?; let pool = self.session.pool()?; @@ -1375,11 +1509,13 @@ impl PointsSinkPort for ChdbNativeAdapter { let batch = (*fact).clone(); pool.with_connection(|conn| { insert_record_batch_direct(conn, &table, batch, InsertOptions::default_bulk()) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB prepared insert join: {e}")))??; + .map_err(|e| { + HyperbytedbError::Internal(format!("chDB prepared insert join: {e}").into()) + })??; histogram!("hyperbytedb_flush_sink_chdb_insert_seconds", "path" => "prepared") .record(insert_start.elapsed().as_secs_f64()); @@ -1401,6 +1537,19 @@ impl PointsSinkPort for ChdbNativeAdapter { .await } + #[cfg(feature = "columnar-ingest")] + async fn build_prepared_wal_slot_from_columnar( + &self, + db: &str, + rp: &str, + origin_node_id: u64, + wire: &crate::application::columnar_msgpack::ColumnarMsgpackBatch, + precision: Option<&str>, + ) -> Result { + self.build_prepared_wal_slot_from_columnar(db, rp, origin_node_id, wire, precision) + .await + } + async fn ensure_measurement_schema( &self, db: &str, @@ -1426,8 +1575,10 @@ impl PointsSinkPort for ChdbNativeAdapter { rp: rp.to_string(), measurement: measurement.to_string(), }; - let table = quoted_table_name(db, rp, measurement); - let series_table = quoted_series_table_name(db, rp, measurement); + let table = quoted_table_name(db, rp, measurement).as_str().to_string(); + let series_table = quoted_series_table_name(db, rp, measurement) + .as_str() + .to_string(); self.execute(format!("DROP TABLE IF EXISTS {table}")) .await?; self.execute(format!("DROP TABLE IF EXISTS {series_table}")) @@ -1444,7 +1595,7 @@ fn run_sync(pool: &ChdbConnectionPool, sql: &str) -> Result<(), HyperbytedbError pool.with_connection(|conn| { execute_connection(conn, sql, OutputFormat::JSONEachRow) .map(|_| ()) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) } @@ -1689,7 +1840,7 @@ fn pad_record_batch_to_ensured( for name in fixed { let idx = schema.index_of(name).map_err(|e| { - HyperbytedbError::Internal(format!("prepared batch missing {name}: {e}")) + HyperbytedbError::Internal(format!("prepared batch missing {name}: {e}").into()) })?; fields.push(schema.field(idx).clone()); columns.push(batch.column(idx).clone()); @@ -1705,7 +1856,7 @@ fn pad_record_batch_to_ensured( } RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) - .map_err(|e| HyperbytedbError::Internal(format!("pad prepared RecordBatch: {e}"))) + .map_err(|e| HyperbytedbError::Internal(format!("pad prepared RecordBatch: {e}").into())) } /// Arrow logical type for a tag column, aligned with [`ColumnKind::ch_column_type`]. @@ -1750,7 +1901,9 @@ fn build_series_tag_column( let dict = DictionaryArray::try_new(Int32Array::from(keys), Arc::new(dictionary_values)) .map_err(|e| { - HyperbytedbError::Internal(format!("build dictionary tag column: {e}")) + HyperbytedbError::Internal( + format!("build dictionary tag column: {e}").into(), + ) })?; Ok(Arc::new(dict)) } @@ -1764,6 +1917,47 @@ fn build_series_tag_column( } } +/// All-null column for a field of the given type discriminant. +fn null_field_column(disc: u8, n: usize) -> ArrayRef { + match disc { + 1 => { + let mut b = Int64Builder::with_capacity(n); + for _ in 0..n { + b.append_null(); + } + Arc::new(b.finish()) + } + 2 => { + let mut b = UInt64Builder::with_capacity(n); + for _ in 0..n { + b.append_null(); + } + Arc::new(b.finish()) + } + 3 => { + let mut b = StringBuilder::new(); + for _ in 0..n { + b.append_null(); + } + Arc::new(b.finish()) + } + 4 => { + let mut b = UInt8Builder::with_capacity(n); + for _ in 0..n { + b.append_null(); + } + Arc::new(b.finish()) + } + _ => { + let mut b = Float64Builder::with_capacity(n); + for _ in 0..n { + b.append_null(); + } + Arc::new(b.finish()) + } + } +} + /// Build one Arrow column for a field, nulling rows that lack the field or /// whose value doesn't match the column's resolved type. fn build_field_column>(points: &[P], logical: &str, disc: u8) -> ArrayRef { @@ -1833,6 +2027,77 @@ fn build_field_column>(points: &[P], logical: &str, disc: u8) - /// schema, with per-row `origins` and `sids` (parallel to `points`). Columns are /// `time`, `origin_node_id`, `ingest_seq`, `series_id`, then the field columns /// (nullable). Returns the batch plus the observed `(min_time, max_time)`. +#[cfg(feature = "columnar-ingest")] +fn columnar_representative_point( + wire: &crate::application::columnar_msgpack::ColumnarMsgpackBatch, + precision: Option<&str>, +) -> Result { + use crate::application::columnar_msgpack::columnar_timestamps_ns; + + let ts = columnar_timestamps_ns(wire, precision)?; + let mut fields = BTreeMap::new(); + fields.insert( + wire.field.clone(), + FieldValue::Float(wire.values.first().copied().unwrap_or(0.0)), + ); + Ok(Point { + measurement: wire.measurement.clone(), + tags: wire.tags.clone(), + fields, + timestamp: ts.first().copied().unwrap_or(0), + }) +} + +#[cfg(feature = "columnar-ingest")] +fn build_columnar_fact_record_batch( + ensured: &EnsuredTable, + ts_ns: &[i64], + origins: &[u64], + seqs: &[u64], + sids: &[u64], + field_logical: &str, + values: &[f64], +) -> Result { + let n = values.len(); + debug_assert_eq!(ts_ns.len(), n); + debug_assert_eq!(origins.len(), n); + debug_assert_eq!(seqs.len(), n); + debug_assert_eq!(sids.len(), n); + + let mut fields: Vec = Vec::with_capacity(4 + ensured.field_phys.len()); + fields.push(Field::new( + "time", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + false, + )); + fields.push(Field::new("origin_node_id", DataType::UInt64, false)); + fields.push(Field::new("ingest_seq", DataType::UInt64, false)); + fields.push(Field::new("series_id", DataType::UInt64, false)); + for (_, phys, disc) in &ensured.field_phys { + fields.push(Field::new(phys, field_arrow_type(*disc), true)); + } + + let mut columns: Vec = Vec::with_capacity(fields.len()); + columns.push(Arc::new( + TimestampNanosecondArray::from(ts_ns.to_vec()).with_timezone("UTC"), + )); + columns.push(Arc::new(UInt64Array::from(origins.to_vec()))); + columns.push(Arc::new(UInt64Array::from(seqs.to_vec()))); + columns.push(Arc::new(UInt64Array::from(sids.to_vec()))); + + for (logical, _, disc) in &ensured.field_phys { + if logical == field_logical && *disc == 0 { + columns.push(Arc::new(Float64Array::from(values.to_vec()))); + } else { + columns.push(null_field_column(*disc, n)); + } + } + + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).map_err(|e| { + HyperbytedbError::Internal(format!("build columnar Arrow RecordBatch: {e}").into()) + }) +} + fn build_record_batch>( ensured: &EnsuredTable, origins: &[u64], @@ -1881,7 +2146,7 @@ fn build_record_batch>( } let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) - .map_err(|e| HyperbytedbError::Internal(format!("build Arrow RecordBatch: {e}")))?; + .map_err(|e| HyperbytedbError::Internal(format!("build Arrow RecordBatch: {e}").into()))?; Ok((batch, min_time, max_time)) } @@ -1907,8 +2172,66 @@ fn build_series_record_batch( columns.push(build_series_tag_column(new_series, logical, *kind)?); } - RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) - .map_err(|e| HyperbytedbError::Internal(format!("build series Arrow RecordBatch: {e}"))) + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).map_err(|e| { + HyperbytedbError::Internal(format!("build series Arrow RecordBatch: {e}").into()) + }) +} + +/// Read a tag column value from a series dimension batch row. +fn arrow_tag_value_at_row(col: &ArrayRef, row: usize) -> String { + if col.is_null(row) { + return String::new(); + } + if let Some(arr) = col.as_any().downcast_ref::() { + return arr.value(row).to_string(); + } + if let Some(dict) = col.as_any().downcast_ref::>() { + let keys = dict.keys(); + if let Some(values) = dict.values().as_any().downcast_ref::() { + return values.value(keys.value(row) as usize).to_string(); + } + } + String::new() +} + +/// Decode `(series_id, logical tags)` from a prepared series `RecordBatch`. +type SeriesTagEntries = Vec<(u64, BTreeMap)>; + +fn series_entries_from_batch( + batch: &RecordBatch, + tag_phys: &[(String, String, ColumnKind)], +) -> Result { + let n = batch.num_rows(); + if n == 0 { + return Ok(Vec::new()); + } + let schema = batch.schema(); + let sid_col = batch + .column(0) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + HyperbytedbError::Internal(crate::error::ChainedError::new("series batch sid col")) + })?; + + let mut entries = Vec::with_capacity(n); + for row in 0..n { + let sid = sid_col.value(row); + let mut tags = BTreeMap::new(); + for (logical, phys, _) in tag_phys { + let idx = schema.index_of(phys).map_err(|e| { + HyperbytedbError::Internal( + format!("series batch missing tag column {phys}: {e}").into(), + ) + })?; + tags.insert( + logical.clone(), + arrow_tag_value_at_row(batch.column(idx), row), + ); + } + entries.push((sid, tags)); + } + Ok(entries) } fn build_insert_sql( @@ -2486,6 +2809,39 @@ mod tests { assert!(sql.contains(&format!("({}, 'b')", id2))); } + #[test] + fn series_entries_from_batch_decodes_logical_tags() { + let ensured = EnsuredTable { + table: "`db_rp_m`".to_string(), + series_table: "`db_rp_m_series`".to_string(), + tag_phys: vec![ + ( + "host".to_string(), + "host".to_string(), + ColumnKind::TagLowCardinality, + ), + ( + "region".to_string(), + "region".to_string(), + ColumnKind::TagString, + ), + ], + field_phys: vec![], + }; + let p1 = make_point(0, &[("host", "a"), ("region", "us")], &[]); + let p2 = make_point(0, &[("host", "b"), ("region", "eu")], &[]); + let batch = + build_series_record_batch(&ensured, &[(1u64, &p1), (2u64, &p2)]).expect("batch"); + + let entries = series_entries_from_batch(&batch, &ensured.tag_phys).expect("entries"); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].0, 1); + assert_eq!(entries[0].1.get("host"), Some(&"a".to_string())); + assert_eq!(entries[0].1.get("region"), Some(&"us".to_string())); + assert_eq!(entries[1].0, 2); + assert_eq!(entries[1].1.get("host"), Some(&"b".to_string())); + } + #[test] fn quoted_string_escapes_single_quote_and_backslash() { let mut out = String::new(); diff --git a/hyperbytedb/src/adapters/chdb/query_adapter.rs b/hyperbytedb/src/adapters/chdb/query_adapter.rs index d7472af..86ffa9e 100644 --- a/hyperbytedb/src/adapters/chdb/query_adapter.rs +++ b/hyperbytedb/src/adapters/chdb/query_adapter.rs @@ -24,8 +24,9 @@ impl ChdbQueryAdapter { /// Used by tests, which create many adapters in one process; only the first /// one to actually run a query binds the singleton engine. pub fn new(data_path: &str) -> Result { - std::fs::create_dir_all(data_path) - .map_err(|e| HyperbytedbError::Chdb(format!("failed to create chDB data dir: {e}")))?; + std::fs::create_dir_all(data_path).map_err(|e| { + HyperbytedbError::Chdb(format!("failed to create chDB data dir: {e}").into()) + })?; tracing::info!(path = %data_path, "chDB connection pool will be built lazily on first query"); Ok(Self { session: SharedSession::new(data_path), @@ -71,11 +72,11 @@ impl QueryPort for ChdbQueryAdapter { pool.with_connection(|conn| { execute_connection(conn, "SELECT 1", OutputFormat::JSONEachRow) .map(|_| ()) - .map_err(|e| HyperbytedbError::Chdb(e.to_string())) + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))) }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB ping join error: {e}")))? + .map_err(|e| HyperbytedbError::Internal(format!("chDB ping join error: {e}").into()))? } async fn execute_sql(&self, sql: &str) -> Result { @@ -98,7 +99,7 @@ impl QueryPort for ChdbQueryAdapter { match qr { Ok(result) => result .data_utf8() - .map_err(|e| HyperbytedbError::Chdb(e.to_string())), + .map_err(|e| HyperbytedbError::Chdb(crate::error::ChainedError::from_error(e))), Err(e) => { let msg = e.to_string(); if msg.contains("CANNOT_EXTRACT_TABLE_STRUCTURE") @@ -110,14 +111,14 @@ impl QueryPort for ChdbQueryAdapter { tracing::warn!(error = %msg, "chDB missing-table error, treating as empty result"); Ok(String::new()) } else { - Err(HyperbytedbError::Chdb(msg)) + Err(HyperbytedbError::Chdb(msg.into())) } } } }) }) .await - .map_err(|e| HyperbytedbError::Internal(format!("chDB task join error: {e}")))??; + .map_err(|e| HyperbytedbError::Internal(format!("chDB task join error: {e}").into()))??; tracing::debug!(result_len = result.len(), "chDB query completed"); Ok(result) diff --git a/hyperbytedb/src/adapters/chdb/session.rs b/hyperbytedb/src/adapters/chdb/session.rs index 1437744..47b74ff 100644 --- a/hyperbytedb/src/adapters/chdb/session.rs +++ b/hyperbytedb/src/adapters/chdb/session.rs @@ -92,7 +92,9 @@ impl SharedSession { }); match res { Ok(p) => Ok(Arc::clone(p)), - Err(msg) => Err(HyperbytedbError::Chdb(msg.clone())), + Err(msg) => Err(HyperbytedbError::Chdb(crate::error::ChainedError::new( + msg.clone(), + ))), } } diff --git a/hyperbytedb/src/adapters/cluster/hinted_handoff.rs b/hyperbytedb/src/adapters/cluster/hinted_handoff.rs index 3a43c6c..5995ec7 100644 --- a/hyperbytedb/src/adapters/cluster/hinted_handoff.rs +++ b/hyperbytedb/src/adapters/cluster/hinted_handoff.rs @@ -73,13 +73,18 @@ impl HintedHandoff { if self.max_hints_per_peer > 0 { let count = self.pending_count(peer_id)?; if count >= self.max_hints_per_peer { - tracing::warn!( - peer_id = peer_id, - count = count, - limit = self.max_hints_per_peer, - "hinted handoff queue full for peer, dropping oldest hint" - ); - self.drop_oldest(peer_id)?; + metrics::counter!( + "hyperbytedb_hinted_handoff_rejected_total", + "peer_id" => peer_id.to_string() + ) + .increment(1); + return Err(HyperbytedbError::Internal( + format!( + "hinted handoff queue full for peer {peer_id} ({count}/{})", + self.max_hints_per_peer + ) + .into(), + )); } } @@ -88,7 +93,7 @@ impl HintedHandoff { let value = payload.encode_hint_value()?; self.db .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Internal(format!("enqueue hint: {e}")))?; + .map_err(|e| HyperbytedbError::Internal(format!("enqueue hint: {e}").into()))?; metrics::counter!("hyperbytedb_hinted_handoff_enqueued_total", "peer_id" => peer_id.to_string()) .increment(1); @@ -205,29 +210,6 @@ impl HintedHandoff { } Ok(count) } - - fn drop_oldest(&self, peer_id: u64) -> Result<(), HyperbytedbError> { - let cf = self - .db - .cf_handle(HH_CF) - .ok_or_else(|| HyperbytedbError::Internal("hinted_handoff CF not found".into()))?; - - let prefix = hh_peer_prefix(peer_id); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(&prefix, rocksdb::Direction::Forward), - ); - - if let Some(Ok((key, _))) = iter.into_iter().next() - && key.starts_with(&prefix) - { - self.db - .delete_cf(&cf, &key) - .map_err(|e| HyperbytedbError::Internal(format!("drop oldest hint: {e}")))?; - } - Ok(()) - } } #[cfg(test)] @@ -293,17 +275,23 @@ mod tests { } #[test] - fn max_hints_drops_oldest() { + fn max_hints_rejects_when_full() { let dir = tempfile::tempdir().unwrap(); let repl = ReplicationLog::open(dir.path()).unwrap(); let hh = HintedHandoff::new(repl.db().clone(), 3).unwrap(); - for i in 0..5 { + for i in 0..3 { hh.enqueue_hint(1, &make_payload(&format!("db{i}"))) .unwrap(); } assert_eq!(hh.pending_count(1).unwrap(), 3); - let drained = hh.drain(1, 10).unwrap(); - assert_eq!(drained[0].database, "db2"); + let err = hh + .enqueue_hint(1, &make_payload("db_overflow")) + .unwrap_err(); + assert!( + err.to_string().contains("hinted handoff queue full"), + "unexpected error: {err}" + ); + assert_eq!(hh.pending_count(1).unwrap(), 3); } } diff --git a/hyperbytedb/src/adapters/cluster/peer_client.rs b/hyperbytedb/src/adapters/cluster/peer_client.rs index 4f266ec..5ab4a3c 100644 --- a/hyperbytedb/src/adapters/cluster/peer_client.rs +++ b/hyperbytedb/src/adapters/cluster/peer_client.rs @@ -175,17 +175,80 @@ impl PeerClient { } /// Fan out a line-protocol batch to all active peers (bounded queue + coalescing worker). - /// Non-blocking: if the outbound queue is full (a peer is down/slow) the batch is - /// dropped rather than stalling ingestion. Divergence is reconciled by anti-entropy - /// sync; async replication must never block the write path on an unhealthy peer. - pub fn replicate_write(self: &Arc, batch: OutboundReplicationBatch) { + /// Non-blocking: if the outbound queue is full, batches are enqueued to hinted-handoff + /// for each replication peer instead of being dropped. + pub fn replicate_write( + self: &Arc, + batch: OutboundReplicationBatch, + ) -> Result<(), HyperbytedbError> { self.start_outbound_processor(); - if let Err(e) = self.outbound_tx.try_send(batch) { - tracing::error!(error = %e, "replication outbound queue full or closed; dropping batch"); - counter!("hyperbytedb_replication_queue_drops_total").increment(1); + match self.outbound_tx.try_send(batch) { + Ok(()) => Ok(()), + Err(tokio::sync::mpsc::error::TrySendError::Full(batch)) => { + counter!("hyperbytedb_replication_queue_full_total").increment(1); + if self.hinted_handoff.is_none() { + return Err(HyperbytedbError::Internal( + "replication outbound queue full and hinted handoff is not configured" + .into(), + )); + } + let this = Arc::clone(self); + tokio::spawn(async move { + if let Err(e) = this.enqueue_batch_to_hinted_handoff(batch).await { + tracing::error!( + error = %e, + "replication outbound queue full and hinted handoff enqueue failed" + ); + counter!("hyperbytedb_replication_queue_drops_total").increment(1); + } + }); + Ok(()) + } + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + tracing::error!("replication outbound queue closed; dropping batch"); + counter!("hyperbytedb_replication_queue_drops_total").increment(1); + Err(HyperbytedbError::Internal( + "replication outbound queue closed".into(), + )) + } } } + async fn enqueue_batch_to_hinted_handoff( + self: &Arc, + batch: OutboundReplicationBatch, + ) -> Result<(), HyperbytedbError> { + let hh = self + .hinted_handoff + .as_ref() + .ok_or_else(|| { + HyperbytedbError::Internal( + "replication outbound queue full and hinted handoff is not configured".into(), + ) + })? + .clone(); + + let peers = { + let m = self.membership.read().await; + m.replication_peers(self.node_id) + .into_iter() + .map(|n| n.node_id) + .collect::>() + }; + + let hint = ReplicationHintPayload { + database: batch.database.clone(), + retention_policy: batch.retention_policy.clone(), + precision: batch.precision.clone(), + line_body: batch.body.clone(), + }; + + for peer_id in peers { + hh.enqueue_hint(peer_id, &hint)?; + } + Ok(()) + } + async fn do_replicate_write(&self, job: &OutboundReplicationBatch) { let start = std::time::Instant::now(); let peers = { @@ -490,14 +553,11 @@ impl PeerClient { } } - async fn do_replicate_mutation(&self, req: &MutationRequest) { - let mutation_seq = match self.replication_log.append_mutation(req) { - Ok(seq) => seq, - Err(e) => { - tracing::error!(error = %e, "failed to log mutation"); - return; - } - }; + async fn do_replicate_mutation(&self, req: &MutationRequest) -> Result<(), HyperbytedbError> { + let mutation_seq = self.replication_log.append_mutation(req).map_err(|e| { + tracing::error!(error = %e, "failed to log mutation"); + HyperbytedbError::Internal(format!("failed to log mutation: {e}").into()) + })?; let peers = { let m = self.membership.read().await; @@ -508,7 +568,7 @@ impl PeerClient { }; if peers.is_empty() { - return; + return Ok(()); } let wire_req = MutationReplicateRequest { @@ -552,7 +612,7 @@ impl PeerClient { if let Ok(ack) = resp.json::().await { let _ = repl_log.set_mutation_ack(pid, ack.ack_seq); } - break; + return true; } Ok(resp) => { tracing::warn!( @@ -578,7 +638,7 @@ impl PeerClient { "giving up mutation replication after {} attempts", max_attempts ); - break; + return false; } tokio::time::sleep(delay).await; delay = (delay * 2).min(Duration::from_secs(30)); @@ -587,22 +647,40 @@ impl PeerClient { }) .collect(); - futures::future::join_all(futures).await; + let outcomes = futures::future::join_all(futures).await; + if outcomes.iter().any(|ok| !ok) { + return Err(HyperbytedbError::Internal( + "mutation replication failed for one or more peers".into(), + )); + } + Ok(()) } /// Fan out a mutation to all active peers. pub fn replicate_mutation(self: &Arc, req: MutationRequest) { let this = Arc::clone(self); tokio::spawn(async move { - this.do_replicate_mutation(&req).await; + if let Err(e) = this.do_replicate_mutation(&req).await { + tracing::error!(error = %e, "background mutation replication failed"); + } }); } + + pub async fn replicate_mutation_sync( + self: &Arc, + req: MutationRequest, + ) -> Result<(), HyperbytedbError> { + self.do_replicate_mutation(&req).await + } } #[async_trait::async_trait] impl ReplicationPort for PeerClient { - fn replicate_write(self: Arc, batch: OutboundReplicationBatch) { - PeerClient::replicate_write(&self, batch); + fn replicate_write( + self: Arc, + batch: OutboundReplicationBatch, + ) -> Result<(), HyperbytedbError> { + PeerClient::replicate_write(&self, batch) } async fn replicate_write_sync( @@ -618,6 +696,13 @@ impl ReplicationPort for PeerClient { PeerClient::replicate_mutation(&self, req); } + async fn replicate_mutation_sync( + self: Arc, + req: MutationRequest, + ) -> Result<(), HyperbytedbError> { + PeerClient::replicate_mutation_sync(&self, req).await + } + async fn active_peer_count(&self, self_node_id: u64) -> usize { let m = self.membership.read().await; m.active_peers(self_node_id).len() diff --git a/hyperbytedb/src/adapters/cluster/raft/log_store.rs b/hyperbytedb/src/adapters/cluster/raft/log_store.rs index 5b0f2ad..f7ba27e 100644 --- a/hyperbytedb/src/adapters/cluster/raft/log_store.rs +++ b/hyperbytedb/src/adapters/cluster/raft/log_store.rs @@ -19,13 +19,15 @@ use rocksdb::{ }; use crate::application::materialized_view_service::MaterializedViewService; +use crate::application::schema_mutation_apply::{self, SchemaMutationDeps}; use crate::domain::cluster::membership::{NodeInfo, NodeState, SharedMembership}; use crate::ports::metadata::MetadataPort; +use crate::ports::points_sink::PointsSinkPort; +use crate::ports::wal::WalPort; use super::TypeConfig; use super::state_machine::StateMachineData; use super::types::{ClusterRequest, ClusterResponse}; -use crate::application::schema_mutation_apply; const CF_META: &str = "meta"; const CF_LOGS: &str = "logs"; @@ -92,6 +94,8 @@ pub struct RaftStore { shared_membership: SharedMembership, metadata: Option>, mv_service: Option>, + points_sink: Option>, + wal: Option>, } impl RaftStore { @@ -164,6 +168,8 @@ impl RaftStore { shared_membership, metadata: None, mv_service: None, + points_sink: None, + wal: None, }) } @@ -262,6 +268,16 @@ impl RaftStore { self } + pub fn with_points_sink(mut self, points_sink: Arc) -> Self { + self.points_sink = Some(points_sink); + self + } + + pub fn with_wal(mut self, wal: Arc) -> Self { + self.wal = Some(wal); + self + } + pub fn shared_membership(&self) -> &SharedMembership { &self.shared_membership } @@ -279,6 +295,7 @@ impl RaftStore { )) })?; db.put_cf(&cf, KEY_VOTE, &bytes).map_err(storage_io_err)?; + db.flush_wal(true).map_err(storage_io_err)?; Ok::<(), StorageError>(()) }) .await @@ -304,6 +321,7 @@ impl RaftStore { })?; db.put_cf(&cf, KEY_LAST_PURGED, &bytes) .map_err(storage_io_err)?; + db.flush_wal(true).map_err(storage_io_err)?; Ok(()) }) .await @@ -451,6 +469,8 @@ impl RaftStorage for RaftStore { shared_membership: self.shared_membership.clone(), metadata: self.metadata.clone(), mv_service: self.mv_service.clone(), + points_sink: self.points_sink.clone(), + wal: self.wal.clone(), } } @@ -481,6 +501,7 @@ impl RaftStorage for RaftStore { batch.put_cf(&cf, u64_to_be(index), val); } db.write(batch).map_err(storage_io_err)?; + db.flush_wal(true).map_err(storage_io_err)?; Ok::<(), StorageError>(()) }) .await @@ -532,6 +553,7 @@ impl RaftStorage for RaftStore { let to = u64_to_be(index + 1); db.delete_range_cf(&cf, &from, &to) .map_err(storage_io_err)?; + db.flush_wal(true).map_err(storage_io_err)?; Ok::<(), StorageError>(()) }) .await @@ -615,6 +637,8 @@ impl RaftStorage for RaftStore { shared_membership: self.shared_membership.clone(), metadata: self.metadata.clone(), mv_service: self.mv_service.clone(), + points_sink: self.points_sink.clone(), + wal: self.wal.clone(), } } @@ -837,8 +861,12 @@ impl RaftStore { ClusterRequest::SchemaMutation(mutation) => { if let Some(ref metadata) = self.metadata { match schema_mutation_apply::apply_schema_mutation( - metadata, - self.mv_service.as_deref(), + SchemaMutationDeps { + metadata, + mv_service: self.mv_service.as_deref(), + points_sink: self.points_sink.as_ref(), + wal: self.wal.as_ref(), + }, *mutation, ) .await diff --git a/hyperbytedb/src/adapters/cluster/raft/network.rs b/hyperbytedb/src/adapters/cluster/raft/network.rs index 6fda981..a2c34c7 100644 --- a/hyperbytedb/src/adapters/cluster/raft/network.rs +++ b/hyperbytedb/src/adapters/cluster/raft/network.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use openraft::BasicNode; use openraft::error::{NetworkError, RPCError, RaftError}; use openraft::network::RPCOption; @@ -12,25 +14,28 @@ use super::TypeConfig; /// HTTP-based Raft network transport using reqwest. pub struct Network { client: reqwest::Client, + default_timeout: Duration, } impl Default for Network { fn default() -> Self { - Self::new() + Self::new(10) } } impl Network { - pub fn new() -> Self { + pub fn new(default_timeout_secs: u64) -> Self { + let default_timeout = Duration::from_secs(default_timeout_secs); Self { // `Client::builder().build()` only fails if TLS init is broken, // in which case `Client::new()` (same default config) would also // be unusable. Fall back to defaults so we don't panic on a path // that's exercised on the hot bootstrap line. client: reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) + .timeout(default_timeout) .build() .unwrap_or_else(|_| reqwest::Client::new()), + default_timeout, } } } @@ -42,6 +47,7 @@ impl RaftNetworkFactory for Network { NetworkConnection { addr: node.addr.clone(), client: self.client.clone(), + default_timeout: self.default_timeout, } } } @@ -50,6 +56,7 @@ impl RaftNetworkFactory for Network { pub struct NetworkConnection { addr: String, client: reqwest::Client, + default_timeout: Duration, } impl NetworkConnection { @@ -57,15 +64,23 @@ impl NetworkConnection { format!("http://{}{}", self.addr, path) } + fn effective_timeout(option: &RPCOption, default: Duration) -> Duration { + let ttl = option.hard_ttl(); + if ttl.is_zero() { default } else { ttl } + } + async fn post_json( &self, path: &str, req: &Req, + option: &RPCOption, ) -> Result>> { + let timeout = Self::effective_timeout(option, self.default_timeout); let resp = self .client .post(self.url(path)) .json(req) + .timeout(timeout) .send() .await .map_err(|e| RPCError::Network(NetworkError::new(&e)))?; @@ -90,12 +105,15 @@ impl NetworkConnection { &self, path: &str, req: &Req, + option: &RPCOption, ) -> Result>> { + let timeout = Self::effective_timeout(option, self.default_timeout); let resp = self .client .post(self.url(path)) .json(req) + .timeout(timeout) .send() .await .map_err(|e| RPCError::Network(NetworkError::new(&e)))?; @@ -121,28 +139,53 @@ impl RaftNetwork for NetworkConnection { async fn append_entries( &mut self, rpc: AppendEntriesRequest, - _option: RPCOption, + option: RPCOption, ) -> Result, RPCError>> { - self.post_json("/internal/raft/append", &rpc).await + self.post_json("/internal/raft/append", &rpc, &option).await } async fn install_snapshot( &mut self, rpc: InstallSnapshotRequest, - _option: RPCOption, + option: RPCOption, ) -> Result< InstallSnapshotResponse, RPCError>, > { - self.post_json_snapshot("/internal/raft/snapshot", &rpc) + self.post_json_snapshot("/internal/raft/snapshot", &rpc, &option) .await } async fn vote( &mut self, rpc: VoteRequest, - _option: RPCOption, + option: RPCOption, ) -> Result, RPCError>> { - self.post_json("/internal/raft/vote", &rpc).await + self.post_json("/internal/raft/vote", &rpc, &option).await + } +} + +#[cfg(test)] +mod tests { + use super::NetworkConnection; + use openraft::network::RPCOption; + use std::time::Duration; + + #[test] + fn effective_timeout_uses_rpc_option_when_non_zero() { + let option = RPCOption::new(Duration::from_secs(5)); + assert_eq!( + NetworkConnection::effective_timeout(&option, Duration::from_secs(10)), + Duration::from_secs(5) + ); + } + + #[test] + fn effective_timeout_falls_back_to_default_when_zero() { + let option = RPCOption::new(Duration::ZERO); + assert_eq!( + NetworkConnection::effective_timeout(&option, Duration::from_secs(10)), + Duration::from_secs(10) + ); } } diff --git a/hyperbytedb/src/adapters/cluster/raft/state_machine.rs b/hyperbytedb/src/adapters/cluster/raft/state_machine.rs index f8fc5ac..4266c1c 100644 --- a/hyperbytedb/src/adapters/cluster/raft/state_machine.rs +++ b/hyperbytedb/src/adapters/cluster/raft/state_machine.rs @@ -35,5 +35,14 @@ pub async fn apply_schema_mutation( metadata: &Arc, mutation: MutationRequest, ) -> Result<(), crate::error::HyperbytedbError> { - crate::application::schema_mutation_apply::apply_schema_mutation(metadata, None, mutation).await + crate::application::schema_mutation_apply::apply_schema_mutation( + crate::application::schema_mutation_apply::SchemaMutationDeps { + metadata, + mv_service: None, + points_sink: None, + wal: None, + }, + mutation, + ) + .await } diff --git a/hyperbytedb/src/adapters/cluster/replication_log.rs b/hyperbytedb/src/adapters/cluster/replication_log.rs index c4be2f7..7ba65b9 100644 --- a/hyperbytedb/src/adapters/cluster/replication_log.rs +++ b/hyperbytedb/src/adapters/cluster/replication_log.rs @@ -23,6 +23,10 @@ fn mutation_ack_key(peer_id: u64) -> Vec { format!("mutation_ack:{}", peer_id).into_bytes() } +fn applied_mutation_key(origin_node_id: u64) -> Vec { + format!("applied_mutation:{}", origin_node_id).into_bytes() +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MutationLogEntry { pub seq: u64, @@ -49,10 +53,12 @@ impl ReplicationLog { ColumnFamilyDescriptor::new(HINTED_HANDOFF_CF, hh_opts), ]; - let db = Arc::new( - DB::open_cf_descriptors(&opts, path, cfs) - .map_err(|e| HyperbytedbError::Internal(format!("replication log open: {e}")))?, - ); + let db = Arc::new(DB::open_cf_descriptors(&opts, path, cfs).map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "replication log open", + e, + )) + })?); let mutation_seq = { let cf = db @@ -79,10 +85,38 @@ impl ReplicationLog { max_seq }; + let applied_mutation_seqs = { + let cf = db + .cf_handle(REPL_CF) + .ok_or_else(|| HyperbytedbError::Internal("replication CF not found".into()))?; + let prefix = b"applied_mutation:"; + let iter = db.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix, rocksdb::Direction::Forward), + ); + let mut map = HashMap::new(); + for (key, value) in iter.flatten() { + if !key.starts_with(prefix) { + break; + } + if let Ok(k) = std::str::from_utf8(&key) + && let Some(origin_str) = k.strip_prefix("applied_mutation:") + && let Ok(origin) = origin_str.parse::() + && value.len() == 8 + { + let mut arr = [0u8; 8]; + arr.copy_from_slice(&value); + map.insert(origin, u64::from_be_bytes(arr)); + } + } + map + }; + Ok(Self { db, mutation_seq: AtomicU64::new(mutation_seq), - applied_mutation_seqs: Mutex::new(HashMap::new()), + applied_mutation_seqs: Mutex::new(applied_mutation_seqs), }) } @@ -113,7 +147,12 @@ impl ReplicationLog { } self.db .put_cf(&cf, ack_key(peer_id), seq.to_be_bytes()) - .map_err(|e| HyperbytedbError::Internal(format!("set_wal_ack: {e}")))?; + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "set_wal_ack", + e, + )) + })?; Ok(()) } @@ -130,7 +169,9 @@ impl ReplicationLog { Ok(u64::from_be_bytes(arr)) } Ok(None) => Ok(0), - Err(e) => Err(HyperbytedbError::Internal(format!("get_wal_ack: {e}"))), + Err(e) => Err(HyperbytedbError::Internal( + crate::error::ChainedError::with_context("get_wal_ack", e), + )), } } @@ -193,11 +234,20 @@ impl ReplicationLog { seq, request: request.clone(), }; - let value = serde_json::to_vec(&entry) - .map_err(|e| HyperbytedbError::Internal(format!("serialize mutation: {e}")))?; + let value = serde_json::to_vec(&entry).map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "serialize mutation", + e, + )) + })?; self.db .put_cf(&cf, mutation_log_key(seq), value) - .map_err(|e| HyperbytedbError::Internal(format!("append_mutation: {e}")))?; + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "append_mutation", + e, + )) + })?; Ok(seq) } @@ -228,7 +278,10 @@ impl ReplicationLog { break; } let entry: MutationLogEntry = serde_json::from_slice(&value).map_err(|e| { - HyperbytedbError::Internal(format!("deserialize mutation: {e}")) + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "deserialize mutation", + e, + )) })?; results.push(entry); } @@ -236,15 +289,32 @@ impl ReplicationLog { Ok(results) } - /// Set mutation ack for a peer. + /// Record mutation ack for a peer. Only advances forward; out-of-order acks + /// are ignored so concurrent replication cannot regress the watermark. pub fn set_mutation_ack(&self, peer_id: u64, seq: u64) -> Result<(), HyperbytedbError> { let cf = self .db .cf_handle(REPL_CF) .ok_or_else(|| HyperbytedbError::Internal("replication CF not found".into()))?; + let current = match self.db.get_cf(&cf, mutation_ack_key(peer_id)) { + Ok(Some(v)) => { + let mut arr = [0u8; 8]; + arr.copy_from_slice(&v); + u64::from_be_bytes(arr) + } + _ => 0, + }; + if seq <= current { + return Ok(()); + } self.db .put_cf(&cf, mutation_ack_key(peer_id), seq.to_be_bytes()) - .map_err(|e| HyperbytedbError::Internal(format!("set_mutation_ack: {e}")))?; + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "set_mutation_ack", + e, + )) + })?; Ok(()) } @@ -261,7 +331,9 @@ impl ReplicationLog { Ok(u64::from_be_bytes(arr)) } Ok(None) => Ok(0), - Err(e) => Err(HyperbytedbError::Internal(format!("get_mutation_ack: {e}"))), + Err(e) => Err(HyperbytedbError::Internal( + crate::error::ChainedError::with_context("get_mutation_ack", e), + )), } } @@ -302,25 +374,45 @@ impl ReplicationLog { .ok_or_else(|| HyperbytedbError::Internal("replication CF not found".into()))?; let from = mutation_log_key(0); let to = mutation_log_key(seq); - self.db - .delete_range_cf(&cf, &from, &to) - .map_err(|e| HyperbytedbError::Internal(format!("truncate_mutations: {e}")))?; + self.db.delete_range_cf(&cf, &from, &to).map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "truncate_mutations", + e, + )) + })?; Ok(()) } /// Returns true if this mutation should be applied (not a duplicate). /// Returns false if it has already been applied (seq <= last seen for this origin). - pub fn check_and_record_mutation(&self, origin_node_id: u64, seq: u64) -> bool { + pub fn check_and_record_mutation( + &self, + origin_node_id: u64, + seq: u64, + ) -> Result { + let cf = self + .db + .cf_handle(REPL_CF) + .ok_or_else(|| HyperbytedbError::Internal("replication CF not found".into()))?; + let mut map = self .applied_mutation_seqs .lock() .unwrap_or_else(|e| e.into_inner()); let last = map.entry(origin_node_id).or_insert(0); if seq > *last { + self.db + .put_cf(&cf, applied_mutation_key(origin_node_id), seq.to_be_bytes()) + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::with_context( + "check_and_record_mutation", + e, + )) + })?; *last = seq; - true + Ok(true) } else { - false + Ok(false) } } @@ -430,6 +522,19 @@ mod tests { assert_eq!(entries[1].seq, 2); } + #[test] + fn test_mutation_ack_monotonic() { + let log = tmp_log(); + log.set_mutation_ack(1, 50).unwrap(); + assert_eq!(log.get_mutation_ack(1).unwrap(), 50); + log.set_mutation_ack(1, 30).unwrap(); + assert_eq!(log.get_mutation_ack(1).unwrap(), 50); + log.set_mutation_ack(1, 50).unwrap(); + assert_eq!(log.get_mutation_ack(1).unwrap(), 50); + log.set_mutation_ack(1, 60).unwrap(); + assert_eq!(log.get_mutation_ack(1).unwrap(), 60); + } + #[test] fn test_mutation_ack() { let log = tmp_log(); @@ -476,4 +581,26 @@ mod tests { assert_eq!(log.get_wal_ack(1).unwrap(), 0); assert_eq!(log.get_mutation_ack(1).unwrap(), 0); } + + #[test] + fn test_mutation_dedup_persisted_across_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path(); + + { + let log = ReplicationLog::open(path).unwrap(); + assert!(log.check_and_record_mutation(42, 10).unwrap()); + assert!(!log.check_and_record_mutation(42, 10).unwrap()); + assert!(!log.check_and_record_mutation(42, 5).unwrap()); + assert!(log.check_and_record_mutation(42, 11).unwrap()); + assert!(log.check_and_record_mutation(99, 1).unwrap()); + } + + let log = ReplicationLog::open(path).unwrap(); + assert!(!log.check_and_record_mutation(42, 10).unwrap()); + assert!(!log.check_and_record_mutation(42, 11).unwrap()); + assert!(log.check_and_record_mutation(42, 12).unwrap()); + assert!(!log.check_and_record_mutation(99, 1).unwrap()); + assert!(log.check_and_record_mutation(99, 2).unwrap()); + } } diff --git a/hyperbytedb/src/adapters/cluster/sync_client.rs b/hyperbytedb/src/adapters/cluster/sync_client.rs index 6a9ddb4..4dd13f6 100644 --- a/hyperbytedb/src/adapters/cluster/sync_client.rs +++ b/hyperbytedb/src/adapters/cluster/sync_client.rs @@ -319,7 +319,7 @@ impl SyncClient { if let Some(_db_name) = entry.key.strip_prefix("db:") { let db: Database = serde_json::from_slice(&entry.value) - .map_err(|e| HyperbytedbError::Metadata(format!("parse db: {e}")))?; + .map_err(|e| HyperbytedbError::Metadata(format!("parse db: {e}").into()))?; self.metadata.create_database(&db.name).await?; for rp in &db.retention_policies { self.metadata @@ -332,12 +332,12 @@ impl SyncClient { let db = parts[1]; let rp = parts[2]; let meta: MeasurementMeta = serde_json::from_slice(&entry.value) - .map_err(|e| HyperbytedbError::Metadata(format!("parse meas: {e}")))?; + .map_err(|e| HyperbytedbError::Metadata(format!("parse meas: {e}").into()))?; self.metadata.register_measurement(db, rp, &meta).await?; } } else if let Some(username) = entry.key.strip_prefix("user:") { let user: StoredUser = serde_json::from_slice(&entry.value) - .map_err(|e| HyperbytedbError::Metadata(format!("parse user: {e}")))?; + .map_err(|e| HyperbytedbError::Metadata(format!("parse user: {e}").into()))?; self.metadata .create_user(username, &user.password_hash, user.admin) .await?; @@ -348,9 +348,10 @@ impl SyncClient { let rp = parts[2]; let meas = parts[3]; let predicate = std::str::from_utf8(&entry.value).map_err(|e| { - HyperbytedbError::Metadata(format!( - "invalid UTF-8 in tombstone value for {db}/{rp}/{meas}: {e}" - )) + HyperbytedbError::Metadata( + format!("invalid UTF-8 in tombstone value for {db}/{rp}/{meas}: {e}") + .into(), + ) })?; self.metadata .store_tombstone(db, rp, meas, predicate) @@ -361,7 +362,7 @@ impl SyncClient { if parts.len() == 3 { let db = parts[1]; let cq: ContinuousQueryDef = serde_json::from_slice(&entry.value) - .map_err(|e| HyperbytedbError::Metadata(format!("parse cq: {e}")))?; + .map_err(|e| HyperbytedbError::Metadata(format!("parse cq: {e}").into()))?; self.metadata .store_continuous_query(db, &cq.name, &cq) .await?; @@ -372,7 +373,7 @@ impl SyncClient { let db = parts[1]; let mv: crate::ports::metadata::MaterializedViewDef = serde_json::from_slice(&entry.value) - .map_err(|e| HyperbytedbError::Metadata(format!("parse mv: {e}")))?; + .map_err(|e| HyperbytedbError::Metadata(format!("parse mv: {e}").into()))?; self.metadata .store_materialized_view(db, &mv.name, &mv) .await?; diff --git a/hyperbytedb/src/adapters/http/auth_middleware.rs b/hyperbytedb/src/adapters/http/auth_middleware.rs index 221cfbe..05ea35d 100644 --- a/hyperbytedb/src/adapters/http/auth_middleware.rs +++ b/hyperbytedb/src/adapters/http/auth_middleware.rs @@ -37,7 +37,8 @@ pub async fn auth_layer( return next.run(request).await; } - let credentials = extract_credentials(&headers, &query); + let credentials = + extract_credentials(&headers, &query, state.auth_allow_query_param_credentials); match credentials { Some((user, pass)) => match state.auth.authenticate_user(&user, &pass).await { @@ -54,9 +55,14 @@ pub async fn auth_layer( } } -fn extract_credentials(headers: &HeaderMap, query: &AuthParams) -> Option<(String, String)> { - // 1. Query parameters - if let (Some(u), Some(p)) = (&query.u, &query.p) +fn extract_credentials( + headers: &HeaderMap, + query: &AuthParams, + allow_query_params: bool, +) -> Option<(String, String)> { + // 1. Query parameters (opt-in; see `[auth] allow_query_param_credentials`) + if allow_query_params + && let (Some(u), Some(p)) = (&query.u, &query.p) && !u.is_empty() { return Some((u.clone(), p.clone())); @@ -103,7 +109,8 @@ pub async fn internal_auth_layer( return next.run(request).await; } - let credentials = extract_credentials(&headers, &query); + let credentials = + extract_credentials(&headers, &query, state.auth_allow_query_param_credentials); match credentials { Some((user, pass)) => match state.auth.authenticate_user(&user, &pass).await { @@ -166,7 +173,53 @@ pub fn hash_password(password: &str) -> Result AuthParams { + AuthParams { + u: Some(u.to_string()), + p: Some(p.to_string()), + } + } + + #[test] + fn query_params_ignored_when_not_allowed() { + let headers = HeaderMap::new(); + let query = auth_params("admin", "secret"); + assert!(extract_credentials(&headers, &query, false).is_none()); + } + + #[test] + fn query_params_used_when_allowed() { + let headers = HeaderMap::new(); + let query = auth_params("admin", "secret"); + assert_eq!( + extract_credentials(&headers, &query, true), + Some(("admin".to_string(), "secret".to_string())) + ); + } + + #[test] + fn basic_auth_works_when_query_params_disallowed() { + use base64::Engine; + let encoded = base64::engine::general_purpose::STANDARD.encode("admin:secret"); + let mut headers = HeaderMap::new(); + headers.insert( + "authorization", + HeaderValue::from_str(&format!("Basic {encoded}")).unwrap(), + ); + let query = AuthParams::default(); + assert_eq!( + extract_credentials(&headers, &query, false), + Some(("admin".to_string(), "secret".to_string())) + ); + } +} diff --git a/hyperbytedb/src/adapters/http/peer_handlers.rs b/hyperbytedb/src/adapters/http/peer_handlers.rs index dd2af41..8e36203 100644 --- a/hyperbytedb/src/adapters/http/peer_handlers.rs +++ b/hyperbytedb/src/adapters/http/peer_handlers.rs @@ -206,25 +206,31 @@ pub async fn handle_replicate_mutation( if origin != 0 && let Some(ref rl) = state.replication_log - && !rl.check_and_record_mutation(origin, sender_seq) { - tracing::debug!( - origin_node_id = origin, - seq = sender_seq, - "skipping duplicate mutation" - ); - return ( - StatusCode::OK, - Json(serde_json::json!({"ok": true, "ack_seq": sender_seq})), - ); + match rl.check_and_record_mutation(origin, sender_seq) { + Ok(false) => { + tracing::debug!( + origin_node_id = origin, + seq = sender_seq, + "skipping duplicate mutation" + ); + return ( + StatusCode::OK, + Json(serde_json::json!({"ok": true, "ack_seq": sender_seq})), + ); + } + Err(e) => { + tracing::error!(error = %e, "failed to record mutation dedup state"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": "internal error recording mutation"})), + ); + } + Ok(true) => {} + } } - let result = apply_mutation( - &state.metadata, - Some(state.mv_service.as_ref()), - req.mutation, - ) - .await; + let result = apply_mutation(&state, req.mutation).await; match result { Ok(()) => ( @@ -490,12 +496,19 @@ pub async fn handle_drain(State(state): State>) -> impl IntoRespon } async fn apply_mutation( - metadata: &Arc, - mv_service: Option<&crate::application::materialized_view_service::MaterializedViewService>, + state: &Arc, req: MutationRequest, ) -> Result<(), crate::error::HyperbytedbError> { - crate::application::schema_mutation_apply::apply_schema_mutation(metadata, mv_service, req) - .await + crate::application::schema_mutation_apply::apply_schema_mutation( + crate::application::schema_mutation_apply::SchemaMutationDeps { + metadata: &state.metadata, + mv_service: Some(state.mv_service.as_ref()), + points_sink: Some(&state.points_sink), + wal: Some(&state.wal), + }, + req, + ) + .await } async fn build_metadata_snapshot( @@ -507,8 +520,9 @@ async fn build_metadata_snapshot( for db in &databases { entries.push(MetadataEntry { key: format!("db:{}", db.name), - value: serde_json::to_vec(db) - .map_err(|e| crate::error::HyperbytedbError::Metadata(e.to_string()))?, + value: serde_json::to_vec(db).map_err(|e| { + crate::error::HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?, }); let rps = metadata.list_retention_policies(&db.name).await?; @@ -520,8 +534,11 @@ async fn build_metadata_snapshot( if let Some(meta) = metadata.get_measurement(&db.name, &rp.name, meas).await? { entries.push(MetadataEntry { key: format!("meas:{}:{}:{}", db.name, rp.name, meas), - value: serde_json::to_vec(&meta) - .map_err(|e| crate::error::HyperbytedbError::Metadata(e.to_string()))?, + value: serde_json::to_vec(&meta).map_err(|e| { + crate::error::HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + ) + })?, }); } @@ -539,8 +556,11 @@ async fn build_metadata_snapshot( for cq in cqs { entries.push(MetadataEntry { key: format!("cq:{}:{}", db.name, cq.name), - value: serde_json::to_vec(&cq) - .map_err(|e| crate::error::HyperbytedbError::Metadata(e.to_string()))?, + value: serde_json::to_vec(&cq).map_err(|e| { + crate::error::HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + ) + })?, }); } @@ -548,8 +568,11 @@ async fn build_metadata_snapshot( for mv in mvs { entries.push(MetadataEntry { key: format!("mv:{}:{}", db.name, mv.name), - value: serde_json::to_vec(&mv) - .map_err(|e| crate::error::HyperbytedbError::Metadata(e.to_string()))?, + value: serde_json::to_vec(&mv).map_err(|e| { + crate::error::HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + ) + })?, }); } } @@ -559,8 +582,11 @@ async fn build_metadata_snapshot( if let Some(u) = metadata.get_user(&user).await? { entries.push(MetadataEntry { key: format!("user:{}", user), - value: serde_json::to_vec(&u) - .map_err(|e| crate::error::HyperbytedbError::Metadata(e.to_string()))?, + value: serde_json::to_vec(&u).map_err(|e| { + crate::error::HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + ) + })?, }); } } diff --git a/hyperbytedb/src/adapters/http/query.rs b/hyperbytedb/src/adapters/http/query.rs index 0388596..edbf79b 100644 --- a/hyperbytedb/src/adapters/http/query.rs +++ b/hyperbytedb/src/adapters/http/query.rs @@ -226,7 +226,12 @@ async fn handle_query_impl( .execute_query(db, &q, epoch, rp, caller) .await .map_err(|e| { - tracing::error!(query = %q, db = db, error = %e, "query execution failed"); + tracing::error!( + query = %crate::timeseriesql::digest::redact_credentials(&q), + db = db, + error = %e, + "query execution failed" + ); counter!("hyperbytedb_query_errors_total", "db" => db.to_string(), "stmt_type" => stmt_type_label, "stmt_normalized" => normalized_query.to_string(), "stmt_digest" => digest_hex.to_string()).increment(1); e })?; @@ -286,9 +291,9 @@ async fn handle_query_impl( } else { let json = if pretty { serde_json::to_string_pretty(&result) - .map_err(|e| HyperbytedbError::Internal(e.to_string()))? + .map_err(|e| HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)))? } else { - serde_json::to_string(&result).map_err(|e| HyperbytedbError::Internal(e.to_string()))? + serde_json::to_string(&result).map_err(|e| HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)))? }; (StatusCode::OK, [("Content-Type", "application/json")], json).into_response() }; diff --git a/hyperbytedb/src/adapters/http/router.rs b/hyperbytedb/src/adapters/http/router.rs index fb6b82c..fa93cd5 100644 --- a/hyperbytedb/src/adapters/http/router.rs +++ b/hyperbytedb/src/adapters/http/router.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; +use axum::http::StatusCode; use axum::{ Router, extract::DefaultBodyLimit, @@ -8,7 +9,7 @@ use axum::{ routing::{get, post}, }; use tower::ServiceBuilder; -use tower_http::trace::TraceLayer; +use tower_http::{timeout::TimeoutLayer, trace::TraceLayer}; use crate::adapters::cluster::peer_client::PeerClient; use crate::adapters::cluster::raft::HyperbytedbRaft; @@ -45,6 +46,8 @@ pub struct AppState { pub drain_service: Option>, pub raft: Option, pub auth_enabled: bool, + /// When false (default), `?u=` / `?p=` query credentials are ignored. + pub auth_allow_query_param_credentials: bool, pub prometheus_handle: Option, pub statement_summary: Option>, /// When true and auth is enabled, `/api/v1/statements` requires credentials. @@ -70,7 +73,7 @@ pub fn build_router(state: Arc) -> Router { let auth_state = state.clone(); let body_limit = state.max_body_size_bytes; let replicate_body_limit = state.replicate_body_limit_bytes; - let _timeout_duration = std::time::Duration::from_secs(state.request_timeout_secs); + let timeout_duration = std::time::Duration::from_secs(state.request_timeout_secs); let mut statements_router = Router::new().route( "/api/v1/statements", @@ -221,9 +224,26 @@ pub fn build_router(state: Arc) -> Router { }; router - .layer(ServiceBuilder::new().layer(middleware::map_response( - http_middleware::add_version_headers, - ))) - .layer(TraceLayer::new_for_http()) + .layer( + ServiceBuilder::new() + .layer(middleware::map_response( + http_middleware::add_version_headers, + )) + .layer(TimeoutLayer::with_status_code( + StatusCode::REQUEST_TIMEOUT, + timeout_duration, + )), + ) + .layer( + TraceLayer::new_for_http().make_span_with(|request: &axum::http::Request<_>| { + tracing::span!( + tracing::Level::DEBUG, + "request", + method = %request.method(), + path = %request.uri().path(), + version = ?request.version(), + ) + }), + ) .with_state(state) } diff --git a/hyperbytedb/src/adapters/metadata/rocksdb_meta.rs b/hyperbytedb/src/adapters/metadata/rocksdb_meta.rs index d84f34a..f2b5d29 100644 --- a/hyperbytedb/src/adapters/metadata/rocksdb_meta.rs +++ b/hyperbytedb/src/adapters/metadata/rocksdb_meta.rs @@ -232,8 +232,9 @@ impl RocksDbMetadata { let cfs = vec![ColumnFamilyDescriptor::new(META_CF, cf_opts)]; let db = Arc::new( - rocksdb::DB::open_cf_descriptors(&opts, path, cfs) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?, + rocksdb::DB::open_cf_descriptors(&opts, path, cfs).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?, ); Ok(Self { @@ -253,30 +254,48 @@ impl RocksDbMetadata { } } +async fn meta_blocking(f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + tokio::task::spawn_blocking(f).await.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })? +} + #[async_trait] impl MetadataPort for RocksDbMetadata { async fn create_database(&self, name: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let name_owned = name.to_string(); let key = db_key(name); - if self - .db - .get_cf(&cf, &key) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))? - .is_some() - { - return Ok(()); - } let db = Database::new(name); let value = serde_json::to_vec(&DbValue { database: db.clone(), }) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db_cache.write().insert(name.to_string(), db); + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + if rdb + .get_cf(&cf, &key) + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))? + .is_some() + { + return Ok(()); + } + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.db_cache.write().insert(name_owned, db); Ok(()) } @@ -324,18 +343,24 @@ impl MetadataPort for RocksDbMetadata { r.is_default = r.name == db_obj.default_rp; } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let db_name = stmt.name.clone(); let key = db_key(&stmt.name); let value = serde_json::to_vec(&DbValue { database: db_obj.clone(), }) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db_cache.write().insert(stmt.name.clone(), db_obj); + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.db_cache.write().insert(db_name, db_obj); Ok(()) } @@ -380,18 +405,24 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let db_name = db.to_string(); let key = db_key(db); let value = serde_json::to_vec(&DbValue { database: db_obj.clone(), }) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db_cache.write().insert(db.to_string(), db_obj); + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.db_cache.write().insert(db_name, db_obj); Ok(()) } @@ -408,7 +439,7 @@ impl MetadataPort for RocksDbMetadata { let meas_filter = measurement.map(str::to_string); let removed = tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let prefix = if let Some(ref m) = meas_filter { format!("series:{db_s}:{rp_s}:{m}:") @@ -423,7 +454,9 @@ impl MetadataPort for RocksDbMetadata { ); let mut keys_to_delete = Vec::new(); for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; if !key.starts_with(pbytes) { break; } @@ -431,13 +464,19 @@ impl MetadataPort for RocksDbMetadata { } let count = keys_to_delete.len(); for key in keys_to_delete { - rdb.delete_cf(&cf, key) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + rdb.delete_cf(&cf, key).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; } Ok::(count) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; { let mut cache = self.series_known.write(); @@ -453,34 +492,39 @@ impl MetadataPort for RocksDbMetadata { } async fn drop_database(&self, name: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let db_prefix = format!("db:{}", name); - let meas_prefix = format!("meas:{}:", name); - let tag_prefix = format!("tag_val:{}:", name); - let series_db_prefix = format!("series:{}:", name); - - let iter = - self.db - .iterator_cf_opt(&cf, rocksdb::ReadOptions::default(), IteratorMode::Start); - let mut to_delete = Vec::new(); - for item in iter { - if let Ok((key, _)) = item - && let Ok(k) = std::str::from_utf8(&key) - && (k.starts_with(&db_prefix) - || k.starts_with(&meas_prefix) - || k.starts_with(&tag_prefix) - || k.starts_with(&series_db_prefix)) - { - to_delete.push(key.to_vec()); + let rdb = self.db.clone(); + let name_owned = name.to_string(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let db_prefix = format!("db:{}", name_owned); + let meas_prefix = format!("meas:{}:", name_owned); + let tag_prefix = format!("tag_val:{}:", name_owned); + let series_db_prefix = format!("series:{}:", name_owned); + + let iter = + rdb.iterator_cf_opt(&cf, rocksdb::ReadOptions::default(), IteratorMode::Start); + let mut to_delete = Vec::new(); + for item in iter { + if let Ok((key, _)) = item + && let Ok(k) = std::str::from_utf8(&key) + && (k.starts_with(&db_prefix) + || k.starts_with(&meas_prefix) + || k.starts_with(&tag_prefix) + || k.starts_with(&series_db_prefix)) + { + to_delete.push(key.to_vec()); + } } - } - for k in to_delete { - self.db - .delete_cf(&cf, k) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - } + for k in to_delete { + rdb.delete_cf(&cf, k).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + } + Ok(()) + }) + .await?; self.db_cache.write().remove(name); { let prefix = format!("{}:", name); @@ -495,6 +539,7 @@ impl MetadataPort for RocksDbMetadata { self.tag_count_cache .write() .retain(|k, _| !k.starts_with(&count_prefix)); + let series_db_prefix = format!("series:{}:", name); self.series_known .write() .retain(|k| !k.starts_with(&series_db_prefix)); @@ -507,27 +552,34 @@ impl MetadataPort for RocksDbMetadata { } async fn list_databases(&self) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let mut dbs = Vec::new(); - let prefix = b"db:"; - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix, rocksdb::Direction::Forward), - ); + let rdb = self.db.clone(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let mut dbs = Vec::new(); + let prefix = b"db:"; + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix, rocksdb::Direction::Forward), + ); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix) { - break; + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix) { + break; + } + let v: DbValue = serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + dbs.push(v.database); } - let v: DbValue = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - dbs.push(v.database); - } - Ok(dbs) + Ok(dbs) + }) + .await } async fn get_database(&self, name: &str) -> Result, HyperbytedbError> { @@ -542,20 +594,28 @@ impl MetadataPort for RocksDbMetadata { let name_owned = name.to_string(); let result = tokio::task::spawn_blocking(move || { let cf = db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; match db.get_cf(&cf, key) { Ok(Some(v)) => { - let dv: DbValue = serde_json::from_slice(&v) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let dv: DbValue = serde_json::from_slice(&v).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; Ok(Some(dv.database)) } Ok(None) => Ok(None), - Err(e) => Err(HyperbytedbError::Metadata(e.to_string())), + Err(e) => Err(HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + )), } }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))?; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })?; if let Ok(Some(ref database)) = result { self.db_cache.write().insert(name_owned, database.clone()); } @@ -585,18 +645,24 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let db_name = db.to_string(); let key = db_key(db); let value = serde_json::to_vec(&DbValue { database: db_opt.clone(), }) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db_cache.write().insert(db.to_string(), db_opt); + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.db_cache.write().insert(db_name, db_opt); Ok(()) } @@ -616,18 +682,24 @@ impl MetadataPort for RocksDbMetadata { .unwrap_or_else(|| "autogen".to_string()); } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let db_name = db.to_string(); let key = db_key(db); let value = serde_json::to_vec(&DbValue { database: db_obj.clone(), }) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db_cache.write().insert(db.to_string(), db_obj); + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.db_cache.write().insert(db_name, db_obj); Ok(()) } @@ -660,17 +732,23 @@ impl MetadataPort for RocksDbMetadata { let rdb = self.db.clone(); let key = meas_key(db, rp, &measurement.name); let value = serde_json::to_vec(measurement) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) })?; - rdb.put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; Ok::<(), HyperbytedbError>(()) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; self.meas_cache .write() .insert(cache_key, measurement.clone()); @@ -698,20 +776,28 @@ impl MetadataPort for RocksDbMetadata { let key = meas_key(db, rp, name); let result = tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; match rdb.get_cf(&cf, key) { Ok(Some(v)) => { - let m: MeasurementMeta = serde_json::from_slice(&v) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let m: MeasurementMeta = serde_json::from_slice(&v).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; Ok(Some(m)) } Ok(None) => Ok(None), - Err(e) => Err(HyperbytedbError::Metadata(e.to_string())), + Err(e) => Err(HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + )), } }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))?; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })?; if let Ok(Some(ref m)) = result { self.meas_cache.write().insert(cache_key, m.clone()); } @@ -729,32 +815,40 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let mut names = Vec::new(); - let prefix = meas_prefix(db); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), - ); + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let names = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let mut names = Vec::new(); + let prefix = meas_prefix(&db_owned); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), + ); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(&prefix) { - break; - } - let rest = &key[prefix.len()..]; - if let Ok(s) = std::str::from_utf8(rest) { - // Key format: meas:{db}:{rp}:{name}, prefix strips meas:{db}: - // so rest = "{rp}:{name}". Extract the last component as name. - let name = s.split(':').next_back().unwrap_or(s); - names.push(name.to_string()); + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(&prefix) { + break; + } + let rest = &key[prefix.len()..]; + if let Ok(s) = std::str::from_utf8(rest) { + // Key format: meas:{db}:{rp}:{name}, prefix strips meas:{db}: + // so rest = "{rp}:{name}". Extract the last component as name. + let name = s.split(':').next_back().unwrap_or(s); + names.push(name.to_string()); + } } - } - names.sort(); - names.dedup(); + names.sort(); + names.dedup(); + Ok(names) + }) + .await?; self.meas_list_cache .write() @@ -767,28 +861,36 @@ impl MetadataPort for RocksDbMetadata { db: &str, rp: &str, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let mut names = Vec::new(); - let prefix = meas_rp_prefix(db, rp); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), - ); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(&prefix) { - break; - } - let rest = &key[prefix.len()..]; - if let Ok(s) = std::str::from_utf8(rest) { - names.push(s.to_string()); + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let rp_owned = rp.to_string(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let mut names = Vec::new(); + let prefix = meas_rp_prefix(&db_owned, &rp_owned); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), + ); + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(&prefix) { + break; + } + let rest = &key[prefix.len()..]; + if let Ok(s) = std::str::from_utf8(rest) { + names.push(s.to_string()); + } } - } - names.sort(); - Ok(names) + names.sort(); + Ok(names) + }) + .await } async fn check_field_types( @@ -861,50 +963,60 @@ impl MetadataPort for RocksDbMetadata { tag_key: &str, measurement: Option<&str>, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let mut values: std::collections::HashSet = std::collections::HashSet::new(); - let prefix = tag_val_prefix(db, rp, measurement); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), - ); + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let rp_owned = rp.to_string(); + let tag_key_owned = tag_key.to_string(); + let meas_filter = measurement.map(str::to_string); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let mut values: std::collections::HashSet = std::collections::HashSet::new(); + let prefix = tag_val_prefix(&db_owned, &rp_owned, meas_filter.as_deref()); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), + ); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(&prefix) { - break; - } - let rest = &key[prefix.len()..]; - if let Ok(s) = std::str::from_utf8(rest) { - // Key format: tag_val:{db}:{rp}:{meas}:{tag_key}:{tag_value} - // When meas is Some: prefix = tag_val:{db}:{rp}:{meas}:, rest = "{tag_key}:{tag_value}" - // When meas is None: prefix = tag_val:{db}:{rp}:, rest = "{meas}:{tag_key}:{tag_value}" - let (k, v) = if measurement.is_some() { - let parts: Vec<&str> = s.splitn(2, ':').collect(); - if parts.len() == 2 { - (parts[0], parts[1]) - } else { - continue; - } - } else { - let parts: Vec<&str> = s.splitn(3, ':').collect(); - if parts.len() == 3 { - (parts[1], parts[2]) + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(&prefix) { + break; + } + let rest = &key[prefix.len()..]; + if let Ok(s) = std::str::from_utf8(rest) { + // Key format: tag_val:{db}:{rp}:{meas}:{tag_key}:{tag_value} + // When meas is Some: prefix = tag_val:{db}:{rp}:{meas}:, rest = "{tag_key}:{tag_value}" + // When meas is None: prefix = tag_val:{db}:{rp}:, rest = "{meas}:{tag_key}:{tag_value}" + let (k, v) = if meas_filter.is_some() { + let parts: Vec<&str> = s.splitn(2, ':').collect(); + if parts.len() == 2 { + (parts[0], parts[1]) + } else { + continue; + } } else { - continue; + let parts: Vec<&str> = s.splitn(3, ':').collect(); + if parts.len() == 3 { + (parts[1], parts[2]) + } else { + continue; + } + }; + if k == tag_key_owned { + values.insert(v.to_string()); } - }; - if k == tag_key { - values.insert(v.to_string()); } } - } - let mut result: Vec<_> = values.into_iter().collect(); - result.sort(); - Ok(result) + let mut result: Vec<_> = values.into_iter().collect(); + result.sort(); + Ok(result) + }) + .await } async fn count_tag_values( @@ -958,7 +1070,7 @@ impl MetadataPort for RocksDbMetadata { let rdb = self.db.clone(); let counts = tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let prefix = b"tag_val:"; let iter = rdb.iterator_cf_opt( @@ -968,7 +1080,9 @@ impl MetadataPort for RocksDbMetadata { ); let mut counts: HashMap = HashMap::new(); for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; if !key.starts_with(prefix) { break; } @@ -982,7 +1096,12 @@ impl MetadataPort for RocksDbMetadata { Ok::, HyperbytedbError>(counts) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; let warmed = counts.len(); *self.tag_count_cache.write() = counts; @@ -1008,8 +1127,9 @@ impl MetadataPort for RocksDbMetadata { if cache.contains(&k) { continue; } - let value = serde_json::to_vec(tags) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let value = serde_json::to_vec(tags).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; out.push((k, value)); } out @@ -1021,18 +1141,24 @@ impl MetadataPort for RocksDbMetadata { let entries = novel.clone(); tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let mut batch = rocksdb::WriteBatch::default(); for (key, value) in &entries { batch.put_cf(&cf, key.as_bytes(), value); } - rdb.write(batch) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + rdb.write(batch).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; Ok::<(), HyperbytedbError>(()) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; { let mut cache = self.series_known.write(); for (k, _) in &novel { @@ -1052,7 +1178,7 @@ impl MetadataPort for RocksDbMetadata { let prefix = series_prefix(db, rp, measurement); tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let pbytes = prefix.as_bytes(); let iter = rdb.iterator_cf_opt( @@ -1062,7 +1188,9 @@ impl MetadataPort for RocksDbMetadata { ); let mut out = Vec::new(); for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; if !key.starts_with(pbytes) { break; } @@ -1072,14 +1200,21 @@ impl MetadataPort for RocksDbMetadata { let Ok(id) = u64::from_str_radix(&s[prefix.len()..], 16) else { continue; }; - let tags: BTreeMap = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let tags: BTreeMap = + serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; out.push((id, tags)); } Ok::)>, HyperbytedbError>(out) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))? + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })? } /// Key-only variant of [`Self::list_series`]: parses the `series_id` out of @@ -1096,7 +1231,7 @@ impl MetadataPort for RocksDbMetadata { let prefix = series_prefix(db, rp, measurement); tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let pbytes = prefix.as_bytes(); let iter = rdb.iterator_cf_opt( @@ -1106,7 +1241,9 @@ impl MetadataPort for RocksDbMetadata { ); let mut out = Vec::new(); for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; if !key.starts_with(pbytes) { break; } @@ -1121,14 +1258,19 @@ impl MetadataPort for RocksDbMetadata { Ok::, HyperbytedbError>(out) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))? + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })? } async fn warm_series(&self) -> Result { let rdb = self.db.clone(); let keys = tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let prefix = b"series:"; let iter = rdb.iterator_cf_opt( @@ -1138,7 +1280,9 @@ impl MetadataPort for RocksDbMetadata { ); let mut keys: HashSet = HashSet::new(); for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; if !key.starts_with(prefix) { break; } @@ -1149,7 +1293,12 @@ impl MetadataPort for RocksDbMetadata { Ok::, HyperbytedbError>(keys) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; let warmed = keys.len(); *self.series_known.write() = keys; @@ -1200,18 +1349,24 @@ impl MetadataPort for RocksDbMetadata { let keys = novel.clone(); tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let mut batch = rocksdb::WriteBatch::default(); for key in &keys { batch.put_cf(&cf, key.as_bytes(), b"1"); } - rdb.write(batch) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + rdb.write(batch).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; Ok::<(), HyperbytedbError>(()) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; { let mut cache = self.tag_known.write(); for k in &novel { @@ -1272,8 +1427,9 @@ impl MetadataPort for RocksDbMetadata { && existing.mean_fields == merged.mean_fields ); if needs_write { - let value = serde_json::to_vec(&merged) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let value = serde_json::to_vec(&merged).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; meas_updates.push((cache_key, value, merged)); } } @@ -1304,7 +1460,7 @@ impl MetadataPort for RocksDbMetadata { tokio::task::spawn_blocking(move || { let cf = rdb.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + HyperbytedbError::Metadata("metadata column family not found".into()) })?; let mut batch = rocksdb::WriteBatch::default(); for (key, value) in &meas_keys { @@ -1313,12 +1469,18 @@ impl MetadataPort for RocksDbMetadata { for key in &tag_keys { batch.put_cf(&cf, key.as_bytes(), b"1"); } - rdb.write(batch) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + rdb.write(batch).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; Ok::<(), HyperbytedbError>(()) }) .await - .map_err(|e| HyperbytedbError::Metadata(format!("metadata task panicked: {e}")))??; + .map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::with_context( + "metadata task panicked", + e, + )) + })??; // Phase 3: update caches if !meas_updates.is_empty() { @@ -1359,9 +1521,8 @@ impl MetadataPort for RocksDbMetadata { password_hash: &str, admin: bool, ) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let username_owned = username.to_string(); let key = user_key(username); let user = StoredUser { password_hash: password_hash.to_string(), @@ -1369,23 +1530,35 @@ impl MetadataPort for RocksDbMetadata { created_at: chrono::Utc::now().to_rfc3339(), privileges: Default::default(), }; - let value = - serde_json::to_vec(&user).map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.user_cache.write().insert(username.to_string(), user); + let value = serde_json::to_vec(&user) + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.user_cache.write().insert(username_owned, user); Ok(()) } async fn drop_user(&self, username: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = user_key(username); - self.db - .delete_cf(&cf, key) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.delete_cf(&cf, key).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; self.user_cache.write().remove(username); Ok(()) } @@ -1397,45 +1570,60 @@ impl MetadataPort for RocksDbMetadata { return Ok(Some(user.clone())); } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = user_key(username); - match self.db.get_cf(&cf, key) { - Ok(Some(v)) => { - let user: StoredUser = serde_json::from_slice(&v) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.user_cache - .write() - .insert(username.to_string(), user.clone()); - Ok(Some(user)) + let username_owned = username.to_string(); + let result = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + match rdb.get_cf(&cf, key) { + Ok(Some(v)) => { + let user: StoredUser = serde_json::from_slice(&v).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(Some(user)) + } + Ok(None) => Ok(None), + Err(e) => Err(HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + )), } - Ok(None) => Ok(None), - Err(e) => Err(HyperbytedbError::Metadata(e.to_string())), + }) + .await?; + if let Some(ref user) = result { + self.user_cache.write().insert(username_owned, user.clone()); } + Ok(result) } async fn list_users(&self) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = user_prefix(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), - ); - let mut names = Vec::new(); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(&prefix) { - break; - } - if let Ok(s) = std::str::from_utf8(&key[prefix.len()..]) { - names.push(s.to_string()); + let rdb = self.db.clone(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = user_prefix(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix.as_slice(), rocksdb::Direction::Forward), + ); + let mut names = Vec::new(); + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(&prefix) { + break; + } + if let Ok(s) = std::str::from_utf8(&key[prefix.len()..]) { + names.push(s.to_string()); + } } - } - Ok(names) + Ok(names) + }) + .await } async fn grant_privilege( @@ -1444,21 +1632,26 @@ impl MetadataPort for RocksDbMetadata { database: &str, privilege: crate::domain::user::DatabasePrivilege, ) -> Result<(), HyperbytedbError> { - let mut user = self - .get_user(username) - .await? - .ok_or_else(|| HyperbytedbError::Internal(format!("user not found: {username}")))?; - user.privileges.insert(database.to_string(), privilege); - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + let mut user = self.get_user(username).await?.ok_or_else(|| { + HyperbytedbError::Internal(format!("user not found: {username}").into()) })?; + user.privileges.insert(database.to_string(), privilege); + let rdb = self.db.clone(); + let username_owned = username.to_string(); let key = user_key(username); - let value = - serde_json::to_vec(&user).map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.user_cache.write().insert(username.to_string(), user); + let value = serde_json::to_vec(&user) + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.user_cache.write().insert(username_owned, user); Ok(()) } @@ -1467,21 +1660,26 @@ impl MetadataPort for RocksDbMetadata { username: &str, database: &str, ) -> Result<(), HyperbytedbError> { - let mut user = self - .get_user(username) - .await? - .ok_or_else(|| HyperbytedbError::Internal(format!("user not found: {username}")))?; - user.privileges.remove(database); - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) + let mut user = self.get_user(username).await?.ok_or_else(|| { + HyperbytedbError::Internal(format!("user not found: {username}").into()) })?; + user.privileges.remove(database); + let rdb = self.db.clone(); + let username_owned = username.to_string(); let key = user_key(username); - let value = - serde_json::to_vec(&user).map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key, value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.user_cache.write().insert(username.to_string(), user); + let value = serde_json::to_vec(&user) + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key, value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; + self.user_cache.write().insert(username_owned, user); Ok(()) } @@ -1491,42 +1689,51 @@ impl MetadataPort for RocksDbMetadata { rp: &str, name: &str, ) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let name_owned = name.to_string(); let key = meas_key(db, rp, name); - self.db - .delete_cf(&cf, key) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - - // Sweep series rows for this measurement across all retention policies. - // The series key is rp-scoped (`series:{db}:{rp}:{meas}:{id}`) so we - // can't form a measurement-only prefix; scan `series:{db}:` and filter. - let series_db_prefix = format!("series:{}:", db); - let mut series_to_delete = Vec::new(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(series_db_prefix.as_bytes(), rocksdb::Direction::Forward), - ); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(series_db_prefix.as_bytes()) { - break; + let series_to_delete = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.delete_cf(&cf, key).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + + // Sweep series rows for this measurement across all retention policies. + // The series key is rp-scoped (`series:{db}:{rp}:{meas}:{id}`) so we + // can't form a measurement-only prefix; scan `series:{db}:` and filter. + let series_db_prefix = format!("series:{}:", db_owned); + let mut series_to_delete = Vec::new(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(series_db_prefix.as_bytes(), rocksdb::Direction::Forward), + ); + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(series_db_prefix.as_bytes()) { + break; + } + if let Ok(s) = std::str::from_utf8(&key) + && let Some((kdb, _rp, kmeas, _id)) = parse_series_storage_key(s) + && kdb == db_owned + && kmeas == name_owned + { + series_to_delete.push((key.to_vec(), s.to_string())); + } } - if let Ok(s) = std::str::from_utf8(&key) - && let Some((kdb, _rp, kmeas, _id)) = parse_series_storage_key(s) - && kdb == db - && kmeas == name - { - series_to_delete.push((key.to_vec(), s.to_string())); + for (k, _) in &series_to_delete { + rdb.delete_cf(&cf, k).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; } - } - for (k, _) in &series_to_delete { - self.db - .delete_cf(&cf, k) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - } + Ok(series_to_delete) + }) + .await?; let cache_key = format!("{}:{}:{}", db, rp, name); self.meas_cache.write().remove(&cache_key); @@ -1558,14 +1765,20 @@ impl MetadataPort for RocksDbMetadata { measurement: &str, predicate_sql: &str, ) -> Result { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let id = uuid::Uuid::new_v4().to_string(); let key = format!("tombstone:{}:{}:{}:{}", db, rp, measurement, id); - self.db - .put_cf(&cf, key.as_bytes(), predicate_sql.as_bytes()) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + let predicate = predicate_sql.as_bytes().to_vec(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key.as_bytes(), &predicate).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; let tomb_key = format!("{}:{}:{}", db, rp, measurement); self.tombstone_cache.write().remove(&tomb_key); Ok(id) @@ -1585,28 +1798,38 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = format!("tombstone:{}:{}:{}:", db, rp, measurement); - let prefix_bytes = prefix.as_bytes(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), - ); - let mut results = Vec::new(); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix_bytes) { - break; + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let rp_owned = rp.to_string(); + let meas_owned = measurement.to_string(); + let results = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = format!("tombstone:{}:{}:{}:", db_owned, rp_owned, meas_owned); + let prefix_bytes = prefix.as_bytes(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), + ); + let mut results = Vec::new(); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix_bytes) { + break; + } + let id = std::str::from_utf8(&key[prefix.len()..]) + .unwrap_or("") + .to_string(); + let predicate = std::str::from_utf8(&value).unwrap_or("").to_string(); + results.push((id, predicate)); } - let id = std::str::from_utf8(&key[prefix.len()..]) - .unwrap_or("") - .to_string(); - let predicate = std::str::from_utf8(&value).unwrap_or("").to_string(); - results.push((id, predicate)); - } + Ok(results) + }) + .await?; self.tombstone_cache .write() @@ -1615,36 +1838,44 @@ impl MetadataPort for RocksDbMetadata { } async fn remove_tombstone(&self, db: &str, tombstone_id: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = format!("tombstone:{}:", db); - let prefix_bytes = prefix.as_bytes(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), - ); - for item in iter { - let (key, _) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix_bytes) { - break; - } - if let Ok(k) = std::str::from_utf8(&key) - && k.ends_with(tombstone_id) - { - self.db - .delete_cf(&cf, &key) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - // Invalidate all tombstone cache entries for this db + let rdb = self.db.clone(); + let db_owned = db.to_string(); + let tombstone_id_owned = tombstone_id.to_string(); + let removed = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = format!("tombstone:{}:", db_owned); + let prefix_bytes = prefix.as_bytes(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), + ); + for item in iter { + let (key, _) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix_bytes) { + break; + } + if let Ok(k) = std::str::from_utf8(&key) + && k.ends_with(&tombstone_id_owned) { - let prefix = format!("{}:", db); - self.tombstone_cache - .write() - .retain(|k, _| !k.starts_with(&prefix)); + rdb.delete_cf(&cf, &key).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + return Ok(true); } - return Ok(()); } + Ok(false) + }) + .await?; + if removed { + let prefix = format!("{}:", db); + self.tombstone_cache + .write() + .retain(|k, _| !k.starts_with(&prefix)); } Ok(()) } @@ -1655,15 +1886,20 @@ impl MetadataPort for RocksDbMetadata { name: &str, definition: &ContinuousQueryDef, ) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("cq:{}:{}", db, name); let value = serde_json::to_vec(definition) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key.as_bytes(), value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key.as_bytes(), value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; *self.cq_cache.write() = None; Ok(()) } @@ -1673,46 +1909,61 @@ impl MetadataPort for RocksDbMetadata { db: &str, name: &str, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("cq:{}:{}", db, name); - match self.db.get_cf(&cf, key.as_bytes()) { - Ok(Some(v)) => { - let def: ContinuousQueryDef = serde_json::from_slice(&v) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - Ok(Some(def)) + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + match rdb.get_cf(&cf, key.as_bytes()) { + Ok(Some(v)) => { + let def: ContinuousQueryDef = serde_json::from_slice(&v).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(Some(def)) + } + Ok(None) => Ok(None), + Err(e) => Err(HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + )), } - Ok(None) => Ok(None), - Err(e) => Err(HyperbytedbError::Metadata(e.to_string())), - } + }) + .await } async fn list_continuous_queries( &self, db: &str, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = format!("cq:{}:", db); - let prefix_bytes = prefix.as_bytes(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), - ); - let mut results = Vec::new(); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix_bytes) { - break; + let rdb = self.db.clone(); + let db_owned = db.to_string(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = format!("cq:{}:", db_owned); + let prefix_bytes = prefix.as_bytes(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), + ); + let mut results = Vec::new(); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix_bytes) { + break; + } + let def: ContinuousQueryDef = serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + results.push(def); } - let def: ContinuousQueryDef = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - results.push(def); - } - Ok(results) + Ok(results) + }) + .await } async fn list_all_continuous_queries( @@ -1725,38 +1976,51 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = b"cq:"; - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix, rocksdb::Direction::Forward), - ); - let mut results = Vec::new(); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix) { - break; + let rdb = self.db.clone(); + let results = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = b"cq:"; + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix, rocksdb::Direction::Forward), + ); + let mut results = Vec::new(); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix) { + break; + } + let def: ContinuousQueryDef = serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + results.push(def); } - let def: ContinuousQueryDef = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - results.push(def); - } + Ok(results) + }) + .await?; *self.cq_cache.write() = Some(results.clone()); Ok(results) } async fn drop_continuous_query(&self, db: &str, name: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("cq:{}:{}", db, name); - self.db - .delete_cf(&cf, key.as_bytes()) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.delete_cf(&cf, key.as_bytes()).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; *self.cq_cache.write() = None; Ok(()) } @@ -1767,15 +2031,20 @@ impl MetadataPort for RocksDbMetadata { name: &str, definition: &MaterializedViewDef, ) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("mv:{}:{}", db, name); let value = serde_json::to_vec(definition) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - self.db - .put_cf(&cf, key.as_bytes(), value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + .map_err(|e| HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.put_cf(&cf, key.as_bytes(), value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; *self.mv_cache.write() = None; Ok(()) } @@ -1785,46 +2054,61 @@ impl MetadataPort for RocksDbMetadata { db: &str, name: &str, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("mv:{}:{}", db, name); - match self.db.get_cf(&cf, key.as_bytes()) { - Ok(Some(v)) => { - let def: MaterializedViewDef = serde_json::from_slice(&v) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - Ok(Some(def)) + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + match rdb.get_cf(&cf, key.as_bytes()) { + Ok(Some(v)) => { + let def: MaterializedViewDef = serde_json::from_slice(&v).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(Some(def)) + } + Ok(None) => Ok(None), + Err(e) => Err(HyperbytedbError::Metadata( + crate::error::ChainedError::from_error(e), + )), } - Ok(None) => Ok(None), - Err(e) => Err(HyperbytedbError::Metadata(e.to_string())), - } + }) + .await } async fn list_materialized_views( &self, db: &str, ) -> Result, HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = format!("mv:{}:", db); - let prefix_bytes = prefix.as_bytes(); - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), - ); - let mut results = Vec::new(); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix_bytes) { - break; + let rdb = self.db.clone(); + let db_owned = db.to_string(); + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = format!("mv:{}:", db_owned); + let prefix_bytes = prefix.as_bytes(); + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix_bytes, rocksdb::Direction::Forward), + ); + let mut results = Vec::new(); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix_bytes) { + break; + } + let def: MaterializedViewDef = serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + results.push(def); } - let def: MaterializedViewDef = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - results.push(def); - } - Ok(results) + Ok(results) + }) + .await } async fn list_all_materialized_views( @@ -1837,38 +2121,51 @@ impl MetadataPort for RocksDbMetadata { } } - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; - let prefix = b"mv:"; - let iter = self.db.iterator_cf_opt( - &cf, - rocksdb::ReadOptions::default(), - IteratorMode::From(prefix, rocksdb::Direction::Forward), - ); - let mut results = Vec::new(); - for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - if !key.starts_with(prefix) { - break; + let rdb = self.db.clone(); + let results = meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + let prefix = b"mv:"; + let iter = rdb.iterator_cf_opt( + &cf, + rocksdb::ReadOptions::default(), + IteratorMode::From(prefix, rocksdb::Direction::Forward), + ); + let mut results = Vec::new(); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + if !key.starts_with(prefix) { + break; + } + let def: MaterializedViewDef = serde_json::from_slice(&value).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + results.push(def); } - let def: MaterializedViewDef = serde_json::from_slice(&value) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; - results.push(def); - } + Ok(results) + }) + .await?; *self.mv_cache.write() = Some(results.clone()); Ok(results) } async fn drop_materialized_view(&self, db: &str, name: &str) -> Result<(), HyperbytedbError> { - let cf = self.db.cf_handle(META_CF).ok_or_else(|| { - HyperbytedbError::Metadata("metadata column family not found".to_string()) - })?; + let rdb = self.db.clone(); let key = format!("mv:{}:{}", db, name); - self.db - .delete_cf(&cf, key.as_bytes()) - .map_err(|e| HyperbytedbError::Metadata(e.to_string()))?; + meta_blocking(move || { + let cf = rdb.cf_handle(META_CF).ok_or_else(|| { + HyperbytedbError::Metadata("metadata column family not found".into()) + })?; + rdb.delete_cf(&cf, key.as_bytes()).map_err(|e| { + HyperbytedbError::Metadata(crate::error::ChainedError::from_error(e)) + })?; + Ok(()) + }) + .await?; *self.mv_cache.write() = None; Ok(()) } diff --git a/hyperbytedb/src/adapters/wal/arrow_cache.rs b/hyperbytedb/src/adapters/wal/arrow_cache.rs index c56641f..d8666fe 100644 --- a/hyperbytedb/src/adapters/wal/arrow_cache.rs +++ b/hyperbytedb/src/adapters/wal/arrow_cache.rs @@ -104,6 +104,12 @@ impl WalArrowCache { gauge!(ENTRIES_GAUGE).set(map.len() as f64); } + pub fn purge_database(&self, database: &str) { + let mut map = self.entries.write(); + map.retain(|_, slot| slot.database != database); + gauge!(ENTRIES_GAUGE).set(map.len() as f64); + } + pub fn len(&self) -> usize { self.entries.read().len() } diff --git a/hyperbytedb/src/adapters/wal/batching_wal.rs b/hyperbytedb/src/adapters/wal/batching_wal.rs index 9f19fb1..26d4572 100644 --- a/hyperbytedb/src/adapters/wal/batching_wal.rs +++ b/hyperbytedb/src/adapters/wal/batching_wal.rs @@ -146,7 +146,7 @@ impl BatchingWal { for (enqueued_at, tx) in responses { histogram!("hyperbytedb_wal_batcher_response_seconds") .record(now.duration_since(enqueued_at).as_secs_f64()); - let _ = tx.send(Err(HyperbytedbError::Wal(msg.clone()))); + let _ = tx.send(Err(HyperbytedbError::Wal(msg.clone().into()))); } } Err(e) => { @@ -154,7 +154,7 @@ impl BatchingWal { for (enqueued_at, tx) in responses { histogram!("hyperbytedb_wal_batcher_response_seconds") .record(now.duration_since(enqueued_at).as_secs_f64()); - let _ = tx.send(Err(HyperbytedbError::Internal(msg.clone()))); + let _ = tx.send(Err(HyperbytedbError::Internal(msg.clone().into()))); } } } @@ -163,9 +163,9 @@ impl BatchingWal { async fn enqueue(&self, bundle: WalAppendBundle) -> Result { if !self.writer_alive.load(Ordering::SeqCst) { - return Err(HyperbytedbError::Wal( - "WAL batcher writer unavailable".into(), - )); + return Err(HyperbytedbError::Wal(crate::error::ChainedError::new( + "WAL batcher writer unavailable", + ))); } // Serialize the durable WAL value here — on the parallel request task — @@ -174,14 +174,14 @@ impl BatchingWal { // time, so this is safe and moves the per-point serialization cost off // the writer (the measured ingest bottleneck). ArrowIpc still encodes // inline because its payload is patched with the assigned seq. - let pre_encoded = if self.inner.wal_format() == WalFormat::Bincode { - Some( - bincode::serialize(&bundle.entry) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?, - ) - } else { - None - }; + let pre_encoded = + if self.inner.wal_format() == WalFormat::Bincode { + Some(bincode::serialize(&bundle.entry).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?) + } else { + None + }; let (tx, rx) = oneshot::channel(); let req = BatchRequest { bundle, @@ -203,10 +203,17 @@ impl BatchingWal { self.sender.send(req).await }; - send_result.map_err(|_| HyperbytedbError::Wal("WAL batcher channel closed".into()))?; - - rx.await - .map_err(|_| HyperbytedbError::Wal("WAL batcher dropped response".into()))? + send_result.map_err(|_| { + HyperbytedbError::Wal(crate::error::ChainedError::new( + "WAL batcher channel closed", + )) + })?; + + rx.await.map_err(|_| { + HyperbytedbError::Wal(crate::error::ChainedError::new( + "WAL batcher dropped response", + )) + })? } } @@ -266,6 +273,10 @@ impl WalPort for BatchingWal { async fn flush_wal(&self) -> Result<(), HyperbytedbError> { self.inner.flush_wal().await } + + async fn purge_database(&self, database: &str) -> Result<(), HyperbytedbError> { + self.inner.purge_database(database).await + } } #[cfg(test)] diff --git a/hyperbytedb/src/adapters/wal/rocksdb_wal.rs b/hyperbytedb/src/adapters/wal/rocksdb_wal.rs index d5037aa..c90aec0 100644 --- a/hyperbytedb/src/adapters/wal/rocksdb_wal.rs +++ b/hyperbytedb/src/adapters/wal/rocksdb_wal.rs @@ -162,7 +162,7 @@ impl RocksDbWal { ]; let db = DB::open_cf_descriptors(&opts, path, cfs) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; Self::migrate_legacy_entries(&db)?; @@ -176,19 +176,27 @@ impl RocksDbWal { let seq = { let wal_cf = db .cf_handle(WAL_CF) - .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".to_string()))?; + .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".into()))?; let mut iter = db.iterator_cf(&wal_cf, IteratorMode::End); match iter.next() { Some(Ok((key, _))) if key.len() == 8 => be_bytes_to_u64(&key), - Some(Err(e)) => return Err(HyperbytedbError::Wal(e.to_string())), + Some(Err(e)) => { + return Err(HyperbytedbError::Wal( + crate::error::ChainedError::from_error(e), + )); + } _ => { let wal_meta_cf = db.cf_handle(WAL_META_CF).ok_or_else(|| { - HyperbytedbError::Wal("wal_meta column family not found".to_string()) + HyperbytedbError::Wal("wal_meta column family not found".into()) })?; match db.get_cf(&wal_meta_cf, LAST_SEQ_KEY) { Ok(Some(v)) if v.len() == 8 => be_bytes_to_u64(&v), Ok(_) => 0, - Err(e) => return Err(HyperbytedbError::Wal(e.to_string())), + Err(e) => { + return Err(HyperbytedbError::Wal( + crate::error::ChainedError::from_error(e), + )); + } } } } @@ -225,7 +233,7 @@ impl RocksDbWal { fn migrate_legacy_entries(db: &DB) -> Result<(), HyperbytedbError> { let wal_cf = db .cf_handle(WAL_CF) - .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".to_string()))?; + .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".into()))?; #[derive(serde::Deserialize)] struct LegacyWalEntry { @@ -244,7 +252,8 @@ impl RocksDbWal { ); for item in iter { - let (key, value) = item.map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + let (key, value) = + item.map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; if bincode::deserialize::(&value).is_ok() { continue; @@ -253,8 +262,12 @@ impl RocksDbWal { continue; } - let legacy: LegacyWalEntry = bincode::deserialize(&value) - .map_err(|e| HyperbytedbError::Wal(format!("corrupt WAL entry: {e}")))?; + let legacy: LegacyWalEntry = bincode::deserialize(&value).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "corrupt WAL entry", + e, + )) + })?; let upgraded = WalEntry { database: legacy.database, @@ -263,8 +276,12 @@ impl RocksDbWal { origin_node_id: 0, }; - let new_value = bincode::serialize(&upgraded) - .map_err(|e| HyperbytedbError::Wal(format!("re-serialize WAL entry: {e}")))?; + let new_value = bincode::serialize(&upgraded).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "re-serialize WAL entry", + e, + )) + })?; batch.put_cf(&wal_cf, &key, &new_value); migrated += 1; @@ -272,7 +289,7 @@ impl RocksDbWal { if migrated > 0 { db.write(batch) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; tracing::info!(migrated, "migrated legacy WAL entries to current schema"); } @@ -373,7 +390,7 @@ fn write_bundle_batch( } db.write(wb) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; if arrow_wal_enabled { for (wal_seq, slot) in seqs.iter().zip(prepared_slots) { @@ -415,7 +432,12 @@ impl WalPort for RocksDbWal { ) }) .await - .map_err(|e| HyperbytedbError::Wal(format!("WAL append task panicked: {e}")))?; + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "WAL append task panicked", + e, + )) + })?; match result { Ok(seqs) => { @@ -469,17 +491,15 @@ impl WalPort for RocksDbWal { from: u64, max_entries: usize, ) -> Result, HyperbytedbError> { - // RocksDB is the source of truth; reads always go to disk. (The former - // in-memory `WalMemoryCache` was removed: the prepared flush path uses - // the Arrow cache, never this, so it only ever grew unbounded while WAL - // truncation was held behind a lagging peer's replication ack.) let db = self.db.clone(); let wal_format = self.wal_format; + let arrow_cache = self.arrow_cache.clone(); + let arrow_wal_enabled = self.arrow_wal_enabled; tokio::task::spawn_blocking(move || { let wal_cf = db .cf_handle(WAL_CF) - .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".to_string()))?; + .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".into()))?; let mut results = Vec::new(); let start_key = u64_to_be_bytes(from); @@ -491,16 +511,31 @@ impl WalPort for RocksDbWal { if results.len() >= max_entries { break; } - let (key, value) = item.map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + let (key, value) = item.map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?; + if key.len() != 8 { + return Err(HyperbytedbError::Wal( + format!("invalid WAL key length: expected 8, got {}", key.len()).into(), + )); + } let seq = be_bytes_to_u64(&key); - let (_, entry) = wal_ipc::decode_wal_value(wal_format, &value)?; + let (prepared, entry) = wal_ipc::decode_wal_value(wal_format, &value)?; + if arrow_wal_enabled && let Some(slot) = prepared { + arrow_cache.insert(seq, slot); + } results.push((seq, entry)); } Ok(results) }) .await - .map_err(|e| HyperbytedbError::Wal(format!("WAL read task panicked: {e}")))? + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "WAL read task panicked", + e, + )) + })? } async fn truncate_before(&self, sequence: u64) -> Result<(), HyperbytedbError> { @@ -514,10 +549,10 @@ impl WalPort for RocksDbWal { tokio::task::spawn_blocking(move || { let wal_cf = db .cf_handle(WAL_CF) - .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".to_string()))?; - let wal_meta_cf = db.cf_handle(WAL_META_CF).ok_or_else(|| { - HyperbytedbError::Wal("wal_meta column family not found".to_string()) - })?; + .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".into()))?; + let wal_meta_cf = db + .cf_handle(WAL_META_CF) + .ok_or_else(|| HyperbytedbError::Wal("wal_meta column family not found".into()))?; let from = u64_to_be_bytes(0); let to = u64_to_be_bytes(sequence); @@ -525,11 +560,16 @@ impl WalPort for RocksDbWal { batch.delete_range_cf(&wal_cf, &from, &to); batch.put_cf(&wal_meta_cf, LAST_SEQ_KEY, u64_to_be_bytes(last_seq)); db.write(batch) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; Ok(()) }) .await - .map_err(|e| HyperbytedbError::Wal(format!("WAL truncate task panicked: {e}")))? + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "WAL truncate task panicked", + e, + )) + })? } async fn last_sequence(&self) -> Result { @@ -540,10 +580,59 @@ impl WalPort for RocksDbWal { let db = self.db.clone(); tokio::task::spawn_blocking(move || { db.flush_wal(true) - .map_err(|e| HyperbytedbError::Wal(e.to_string())) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e))) + }) + .await + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "WAL flush panicked", + e, + )) + })? + } + + async fn purge_database(&self, database: &str) -> Result<(), HyperbytedbError> { + self.arrow_cache.purge_database(database); + let db = self.db.clone(); + let wal_format = self.wal_format; + let database = database.to_string(); + tokio::task::spawn_blocking(move || { + let wal_cf = db + .cf_handle(WAL_CF) + .ok_or_else(|| HyperbytedbError::Wal("wal column family not found".into()))?; + + let mut batch = WriteBatch::default(); + let iter = db.iterator_cf_opt( + &wal_cf, + rocksdb::ReadOptions::default(), + IteratorMode::Start, + ); + for item in iter { + let (key, value) = item.map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?; + if key.len() != 8 { + continue; + } + let (_, entry) = wal_ipc::decode_wal_value(wal_format, &value)?; + if entry.database == database { + batch.delete_cf(&wal_cf, key); + } + } + if !batch.is_empty() { + db.write(batch).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?; + } + Ok(()) }) .await - .map_err(|e| HyperbytedbError::Wal(format!("WAL flush panicked: {e}")))? + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "WAL purge task panicked", + e, + )) + })? } } @@ -676,4 +765,32 @@ mod tests { .unwrap(); assert_absolute_seqs(&append_and_take(&wal).await); } + + #[tokio::test] + async fn purge_database_removes_matching_wal_entries_and_cache() { + let tmp = TempDir::new().unwrap(); + let wal = RocksDbWal::open(tmp.path()).unwrap(); + + let mut other = test_entry(); + other.database = "keep".into(); + wal.append(other).await.unwrap(); + wal.append_bundle(WalAppendBundle { + entry: test_entry(), + prepared: Some(relative_slot()), + }) + .await + .unwrap(); + + wal.purge_database("db").await.unwrap(); + + let remaining = wal.read_from(1).await.unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].1.database, "keep"); + assert!( + wal.take_prepared_range(1, u64::MAX, 10) + .await + .unwrap() + .is_none() + ); + } } diff --git a/hyperbytedb/src/adapters/wal/wal_ipc.rs b/hyperbytedb/src/adapters/wal/wal_ipc.rs index 54dbc2c..04c225c 100644 --- a/hyperbytedb/src/adapters/wal/wal_ipc.rs +++ b/hyperbytedb/src/adapters/wal/wal_ipc.rs @@ -21,46 +21,60 @@ fn write_string(w: &mut Vec, s: &str) -> Result<(), HyperbytedbError> { return Err(HyperbytedbError::Wal("WAL string too long".into())); } w.write_all(&(bytes.len() as u16).to_le_bytes()) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; w.write_all(bytes) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; Ok(()) } fn read_string(r: &mut Cursor<&[u8]>) -> Result { let mut len_buf = [0u8; 2]; r.read_exact(&mut len_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let len = u16::from_le_bytes(len_buf) as usize; let mut buf = vec![0u8; len]; r.read_exact(&mut buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; - String::from_utf8(buf).map_err(|e| HyperbytedbError::Wal(e.to_string())) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; + String::from_utf8(buf) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e))) } fn encode_record_batch(batch: &RecordBatch) -> Result, HyperbytedbError> { let mut buf = Vec::new(); { - let mut writer = StreamWriter::try_new(&mut buf, &batch.schema()) - .map_err(|e| HyperbytedbError::Wal(format!("IPC encode: {e}")))?; - writer - .write(batch) - .map_err(|e| HyperbytedbError::Wal(format!("IPC encode write: {e}")))?; - writer - .finish() - .map_err(|e| HyperbytedbError::Wal(format!("IPC encode finish: {e}")))?; + let mut writer = StreamWriter::try_new(&mut buf, &batch.schema()).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context("IPC encode", e)) + })?; + writer.write(batch).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "IPC encode write", + e, + )) + })?; + writer.finish().map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "IPC encode finish", + e, + )) + })?; } Ok(buf) } fn decode_record_batch(bytes: &[u8]) -> Result, HyperbytedbError> { let cursor = Cursor::new(bytes); - let mut reader = StreamReader::try_new(cursor, None) - .map_err(|e| HyperbytedbError::Wal(format!("IPC decode: {e}")))?; + let mut reader = StreamReader::try_new(cursor, None).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context("IPC decode", e)) + })?; let batch = reader .next() .transpose() - .map_err(|e| HyperbytedbError::Wal(format!("IPC decode batch: {e}")))? + .map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::with_context( + "IPC decode batch", + e, + )) + })? .ok_or_else(|| HyperbytedbError::Wal("IPC stream empty".into()))?; Ok(Arc::new(batch)) } @@ -104,7 +118,7 @@ pub fn encode_prepared_slot( let legacy = legacy_entry .map(bincode::serialize) .transpose() - .map_err(|e| HyperbytedbError::Wal(e.to_string()))? + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))? .unwrap_or_default(); out.extend_from_slice(&(legacy.len() as u32).to_le_bytes()); if !legacy.is_empty() { @@ -122,9 +136,9 @@ pub fn decode_prepared_slot( } let version = bytes[4]; if version != VERSION { - return Err(HyperbytedbError::Wal(format!( - "unsupported prepared WAL version {version}" - ))); + return Err(HyperbytedbError::Wal( + format!("unsupported prepared WAL version {version}").into(), + )); } let mut cursor = Cursor::new(&bytes[5..]); @@ -133,13 +147,13 @@ pub fn decode_prepared_slot( let mut origin_buf = [0u8; 8]; cursor .read_exact(&mut origin_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let origin_node_id = u64::from_le_bytes(origin_buf); let mut mc_buf = [0u8; 4]; cursor .read_exact(&mut mc_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let measurement_count = u32::from_le_bytes(mc_buf) as usize; let mut measurements = Vec::with_capacity(measurement_count); @@ -151,61 +165,63 @@ pub fn decode_prepared_slot( let mut rc_buf = [0u8; 4]; cursor .read_exact(&mut rc_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let row_count = u32::from_le_bytes(rc_buf) as usize; let mut time_buf = [0u8; 8]; cursor .read_exact(&mut time_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let min_time = i64::from_le_bytes(time_buf); cursor .read_exact(&mut time_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let max_time = i64::from_le_bytes(time_buf); let mut len_buf = [0u8; 4]; cursor .read_exact(&mut len_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let fact_len = u32::from_le_bytes(len_buf) as usize; let remaining = cursor .get_ref() .len() .saturating_sub(cursor.position() as usize); if fact_len > remaining { - return Err(HyperbytedbError::Wal(format!( - "invalid fact_ipc length {fact_len} (only {remaining} bytes remaining)" - ))); + return Err(HyperbytedbError::Wal( + format!("invalid fact_ipc length {fact_len} (only {remaining} bytes remaining)") + .into(), + )); } let mut fact_ipc = vec![0u8; fact_len]; cursor .read_exact(&mut fact_ipc) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let batch = decode_record_batch(&fact_ipc)?; cursor .read_exact(&mut len_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let series_len = u32::from_le_bytes(len_buf) as usize; - let new_series_batch = if series_len > 0 { - let remaining = cursor - .get_ref() - .len() - .saturating_sub(cursor.position() as usize); - if series_len > remaining { - return Err(HyperbytedbError::Wal(format!( + let new_series_batch = + if series_len > 0 { + let remaining = cursor + .get_ref() + .len() + .saturating_sub(cursor.position() as usize); + if series_len > remaining { + return Err(HyperbytedbError::Wal(format!( "invalid series_ipc length {series_len} (only {remaining} bytes remaining)" - ))); - } - let mut series_ipc = vec![0u8; series_len]; - cursor - .read_exact(&mut series_ipc) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; - Some(decode_record_batch(&series_ipc)?) - } else { - None - }; + ).into())); + } + let mut series_ipc = vec![0u8; series_len]; + cursor.read_exact(&mut series_ipc).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?; + Some(decode_record_batch(&series_ipc)?) + } else { + None + }; measurements.push(PreparedMeasurementBatch { measurement, @@ -221,26 +237,31 @@ pub fn decode_prepared_slot( cursor .read_exact(&mut mc_buf) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; let legacy_len = u32::from_le_bytes(mc_buf) as usize; - let legacy_entry = if legacy_len > 0 { - let remaining = cursor - .get_ref() - .len() - .saturating_sub(cursor.position() as usize); - if legacy_len > remaining { - return Err(HyperbytedbError::Wal(format!( + let legacy_entry = + if legacy_len > 0 { + let remaining = cursor + .get_ref() + .len() + .saturating_sub(cursor.position() as usize); + if legacy_len > remaining { + return Err(HyperbytedbError::Wal(format!( "invalid legacy entry length {legacy_len} (only {remaining} bytes remaining)" - ))); - } - let mut legacy = vec![0u8; legacy_len]; - cursor - .read_exact(&mut legacy) - .map_err(|e| HyperbytedbError::Wal(e.to_string()))?; - Some(bincode::deserialize(&legacy).map_err(|e| HyperbytedbError::Wal(e.to_string()))?) - } else { - None - }; + ).into())); + } + let mut legacy = vec![0u8; legacy_len]; + cursor + .read_exact(&mut legacy) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; + Some( + bincode::deserialize(&legacy).map_err(|e| { + HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)) + })?, + ) + } else { + None + }; Ok(( PreparedWalSlot { @@ -259,9 +280,8 @@ pub fn encode_wal_value( entry: &WalEntry, ) -> Result, HyperbytedbError> { match format { - WalFormat::Bincode => { - bincode::serialize(entry).map_err(|e| HyperbytedbError::Wal(e.to_string())) - } + WalFormat::Bincode => bincode::serialize(entry) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e))), WalFormat::ArrowIpc => { let slot = slot.ok_or_else(|| { HyperbytedbError::Wal("arrow IPC WAL requires prepared slot".into()) @@ -277,8 +297,8 @@ pub fn decode_wal_value( ) -> Result<(Option, WalEntry), HyperbytedbError> { match format { WalFormat::Bincode => { - let entry: WalEntry = - bincode::deserialize(bytes).map_err(|e| HyperbytedbError::Wal(e.to_string()))?; + let entry: WalEntry = bincode::deserialize(bytes) + .map_err(|e| HyperbytedbError::Wal(crate::error::ChainedError::from_error(e)))?; Ok((None, entry)) } WalFormat::ArrowIpc => { diff --git a/hyperbytedb/src/application/arrow_ingest/columnar.rs b/hyperbytedb/src/application/arrow_ingest/columnar.rs index 1b97c7b..2a8feeb 100644 --- a/hyperbytedb/src/application/arrow_ingest/columnar.rs +++ b/hyperbytedb/src/application/arrow_ingest/columnar.rs @@ -1,7 +1,7 @@ use crate::adapters::chdb::native_adapter::ChdbNativeAdapter; -use crate::application::arrow_ingest::points::points_to_prepared_bundle; use crate::application::columnar_msgpack::ColumnarMsgpackBatch; -use crate::application::columnar_msgpack::columnar_batch_to_points; +use crate::application::columnar_msgpack::columnar_batch_to_record_batch; +use crate::domain::wal::WalEntry; use crate::error::HyperbytedbError; use crate::ports::wal::WalAppendBundle; @@ -13,6 +13,21 @@ pub async fn columnar_to_prepared_slot( precision: Option<&str>, origin_node_id: u64, ) -> Result { - let points = columnar_batch_to_points(wire, precision)?; - points_to_prepared_bundle(sink, db, rp, origin_node_id, points).await + // Validate the wire batch can be converted to Arrow before building the + // chDB fact-table prepared slot (shared timestamp/field semantics). + let _ = columnar_batch_to_record_batch(wire, precision)?; + + let prepared = sink + .build_prepared_wal_slot_from_columnar(db, rp, origin_node_id, wire, precision) + .await?; + let entry = WalEntry { + database: db.to_string(), + retention_policy: rp.to_string(), + points: Vec::new(), + origin_node_id, + }; + Ok(WalAppendBundle { + entry, + prepared: Some(prepared), + }) } diff --git a/hyperbytedb/src/application/cluster/bootstrap.rs b/hyperbytedb/src/application/cluster/bootstrap.rs index 2cb1ed6..6b6c874 100644 --- a/hyperbytedb/src/application/cluster/bootstrap.rs +++ b/hyperbytedb/src/application/cluster/bootstrap.rs @@ -118,12 +118,9 @@ impl ClusterBootstrap { self.peer_addrs.clone(), ); - let has_data = metadata - .list_databases() - .await - .map(|dbs| !dbs.is_empty()) - .unwrap_or(false); - let wal_seq = wal.last_sequence().await.unwrap_or(0); + let dbs = metadata.list_databases().await?; + let has_data = !dbs.is_empty(); + let wal_seq = wal.last_sequence().await?; const MAX_RETRIES: u32 = 5; let is_new_node = !has_data && wal_seq == 0; @@ -237,19 +234,20 @@ impl ClusterBootstrap { config: &ClusterConfig, metadata: Arc, mv_service: Arc, - ) -> Option { + points_sink: Arc, + wal: Arc, + ) -> anyhow::Result { use crate::adapters::cluster::raft::log_store::RaftStore; use crate::adapters::cluster::raft::network::Network; use openraft::Config as RaftConfig; use openraft::storage::Adaptor; - let raft_store = match RaftStore::open(&config.raft_dir, self.membership.clone()) { - Ok(store) => store.with_metadata(metadata).with_mv_service(mv_service), - Err(e) => { - tracing::error!(error = %e, "failed to open raft store"); - return None; - } - }; + let raft_store = RaftStore::open(&config.raft_dir, self.membership.clone()) + .map_err(|e| anyhow::anyhow!("failed to open raft store: {e}"))? + .with_metadata(metadata) + .with_mv_service(mv_service) + .with_points_sink(points_sink) + .with_wal(wal); // Push the persisted Raft membership into the data-plane SharedMembership // BEFORE handing the store to openraft. Without this, a restarted leader's // log is fully applied so no Membership entries replay during catch-up @@ -260,27 +258,23 @@ impl ClusterBootstrap { .await; let (log_store, state_machine) = Adaptor::::new(raft_store); - let network = Network::new(); + let network = Network::new(config.raft_rpc_timeout_secs); - let raft_config = match (RaftConfig { - heartbeat_interval: config.raft_heartbeat_interval_ms.unwrap_or(1000), - election_timeout_min: config.raft_election_timeout_ms.unwrap_or(1000), - election_timeout_max: config.raft_election_timeout_ms.unwrap_or(1000) * 2, - snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast( - config.raft_snapshot_threshold.unwrap_or(1000) as u64, - ), - ..Default::default() - }) - .validate() - { - Ok(c) => Arc::new(c), - Err(e) => { - tracing::error!(error = %e, "invalid raft config; cluster will not start"); - return None; - } - }; + let raft_config = Arc::new( + (RaftConfig { + heartbeat_interval: config.raft_heartbeat_interval_ms.unwrap_or(1000), + election_timeout_min: config.raft_election_timeout_ms.unwrap_or(1000), + election_timeout_max: config.raft_election_timeout_ms.unwrap_or(1000) * 2, + snapshot_policy: openraft::SnapshotPolicy::LogsSinceLast( + config.raft_snapshot_threshold.unwrap_or(1000) as u64, + ), + ..Default::default() + }) + .validate() + .map_err(|e| anyhow::anyhow!("invalid raft config: {e}"))?, + ); - match HyperbytedbRaft::new( + let raft = HyperbytedbRaft::new( config.node_id, raft_config, network, @@ -288,31 +282,25 @@ impl ClusterBootstrap { state_machine, ) .await - { - Ok(raft) => { - tracing::info!( - node_id = config.node_id, - "raft consensus engine initialized" - ); + .map_err(|e| anyhow::anyhow!("failed to initialize raft: {e}"))?; - if config.node_id == 1 { - use std::collections::BTreeMap; - let mut members = BTreeMap::new(); - members.insert( - config.node_id, - openraft::BasicNode::new(config.cluster_addr.clone()), - ); - if let Err(e) = raft.initialize(members).await { - tracing::debug!(error = %e, "raft already initialized (expected on restart)"); - } - } + tracing::info!( + node_id = config.node_id, + "raft consensus engine initialized" + ); - Some(raft) - } - Err(e) => { - tracing::error!(error = %e, "failed to initialize raft"); - None + if config.node_id == 1 { + use std::collections::BTreeMap; + let mut members = BTreeMap::new(); + members.insert( + config.node_id, + openraft::BasicNode::new(config.cluster_addr.clone()), + ); + if let Err(e) = raft.initialize(members).await { + tracing::debug!(error = %e, "raft already initialized (expected on restart)"); } } + + Ok(raft) } } diff --git a/hyperbytedb/src/application/cluster/drain.rs b/hyperbytedb/src/application/cluster/drain.rs index 3472744..ab16d1e 100644 --- a/hyperbytedb/src/application/cluster/drain.rs +++ b/hyperbytedb/src/application/cluster/drain.rs @@ -80,7 +80,9 @@ impl DrainService { loop { if start.elapsed() > max_wait { tracing::warn!("timed out waiting for replication acks"); - break; + return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "timed out waiting for peer replication acks during drain", + ))); } let peers = { diff --git a/hyperbytedb/src/application/cluster/raft_formation.rs b/hyperbytedb/src/application/cluster/raft_formation.rs index c22c044..6b525dd 100644 --- a/hyperbytedb/src/application/cluster/raft_formation.rs +++ b/hyperbytedb/src/application/cluster/raft_formation.rs @@ -4,6 +4,39 @@ use std::time::Duration; use crate::adapters::http::router::AppState; use crate::domain::cluster::membership::{NodeInfo, NodeState}; +use openraft::BasicNode; +use openraft::RaftMetrics; + +/// Returns true when every non-leader peer in `peer_ids` has replicated the +/// leader's current log tail (per Raft replication metrics). +fn learners_caught_up( + metrics: &RaftMetrics, + peer_ids: &BTreeSet, + leader_id: u64, +) -> bool { + let target_index = metrics.last_log_index.unwrap_or(0); + let learner_ids: Vec = peer_ids + .iter() + .copied() + .filter(|id| *id != leader_id) + .collect(); + if learner_ids.is_empty() { + return true; + } + + let replication = match &metrics.replication { + Some(r) => r, + None => return false, + }; + + learner_ids + .iter() + .all(|learner_id| match replication.get(learner_id) { + Some(Some(matched)) => matched.index >= target_index, + Some(None) => target_index == 0, + None => false, + }) +} /// Background task that forms the Raft cluster from the static peer list. /// @@ -136,24 +169,35 @@ pub async fn run_raft_cluster_formation( } } - // ── Phase 2 & 3: Promote when all peers discovered ────────── + // ── Phase 2 & 3: Promote when all peers discovered and caught up ── if discovered_ids.len() >= expected_size { - match raft.change_membership(discovered_ids.clone(), false).await { - Ok(_) => { - tracing::info!( - voters = ?discovered_ids, - "raft membership promotion succeeded" - ); - break; - } - Err(e) => { - tracing::debug!( - error = %e, - attempt = attempt, - discovered = ?discovered_ids, - "membership promotion not ready yet (learners catching up)" - ); + let metrics = raft.metrics().borrow().clone(); + if learners_caught_up(&metrics, &discovered_ids, node_id) { + match raft.change_membership(discovered_ids.clone(), false).await { + Ok(_) => { + tracing::info!( + voters = ?discovered_ids, + "raft membership promotion succeeded" + ); + break; + } + Err(e) => { + tracing::debug!( + error = %e, + attempt = attempt, + discovered = ?discovered_ids, + "membership promotion not ready yet" + ); + } } + } else { + tracing::debug!( + attempt = attempt, + discovered = ?discovered_ids, + last_log_index = ?metrics.last_log_index, + replication = ?metrics.replication, + "waiting for learners to catch up before promotion" + ); } } else { tracing::debug!( diff --git a/hyperbytedb/src/application/columnar_msgpack.rs b/hyperbytedb/src/application/columnar_msgpack.rs index 663582a..7837642 100644 --- a/hyperbytedb/src/application/columnar_msgpack.rs +++ b/hyperbytedb/src/application/columnar_msgpack.rs @@ -4,7 +4,10 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use std::fmt::Write as _; +use crate::application::ingest_metadata::validate_point_count; +use crate::application::msgpack_limits::peek_columnar_values_len; use crate::domain::column_mapping::tag_col_name_for_columnar; use crate::domain::database::Precision; use crate::domain::point::{FieldValue, Point}; @@ -36,37 +39,70 @@ pub fn decode_columnar_batch(body: &[u8]) -> Result` (`precision` matches `/write` query param). -pub fn parse_columnar_msgpack_to_points( +/// Like [`decode_columnar_batch`] but rejects batches above `max_points` before +/// allocating the `values` vector (`0` = default cap). +pub fn decode_columnar_batch_limited( body: &[u8], - precision: Option<&str>, -) -> Result, HyperbytedbError> { - let wire = decode_columnar_batch(body)?; - columnar_batch_to_points(&wire, precision) + max_points: usize, +) -> Result { + if body.is_empty() { + return Err(HyperbytedbError::ColumnarMsgpackParse { + reason: "empty body".into(), + }); + } + if let Some(count) = peek_columnar_values_len(body)? { + validate_point_count(count, max_points)?; + } + decode_columnar_batch(body) } -/// Expand a decoded columnar batch into `Vec`. -/// -/// Shares the measurement and tags allocation across all points using -/// clone-on-first then reuse, reducing heap churn vs the naive per-point clone. -pub fn columnar_batch_to_points( +/// Nanosecond timestamps for every row in a columnar batch. +pub fn columnar_timestamps_ns( wire: &ColumnarMsgpackBatch, precision: Option<&str>, -) -> Result, HyperbytedbError> { +) -> Result, HyperbytedbError> { + let n = wire.values.len(); + if n == 0 { + return Ok(Vec::new()); + } + + let precision_val = Precision::from_str_opt(precision); + + if let Some(ref ts) = wire.timestamps { + if ts.len() != n { + return Err(HyperbytedbError::ColumnarMsgpackParse { + reason: format!( + "timestamps length {} does not match values length {}", + ts.len(), + n + ), + }); + } + return Ok(ts.iter().map(|t| precision_val.to_nanos(*t)).collect()); + } + + let now = chrono::Utc::now() + .timestamp_nanos_opt() + .ok_or(HyperbytedbError::WallClockTimestampUnavailable)?; + Ok(vec![now; n]) +} + +/// Encode columnar wire data as line protocol without expanding to `Vec`. +pub fn columnar_wire_to_line_protocol( + wire: &ColumnarMsgpackBatch, + precision: Precision, +) -> Result, HyperbytedbError> { if wire.field.is_empty() { return Err(HyperbytedbError::ColumnarMsgpackParse { reason: "field name must be non-empty".into(), }); } - let n = wire.values.len(); if n == 0 { return Ok(Vec::new()); } - let precision_val = Precision::from_str_opt(precision); - - let ts_ns_vec: Vec = if let Some(ref ts) = wire.timestamps { + let ts_raw: Vec = if let Some(ref ts) = wire.timestamps { if ts.len() != n { return Err(HyperbytedbError::ColumnarMsgpackParse { reason: format!( @@ -76,14 +112,64 @@ pub fn columnar_batch_to_points( ), }); } - ts.iter().map(|t| precision_val.to_nanos(*t)).collect() + ts.clone() } else { let now = chrono::Utc::now() .timestamp_nanos_opt() .ok_or(HyperbytedbError::WallClockTimestampUnavailable)?; - vec![now; n] + let raw = precision.from_nanos(now); + vec![raw; n] }; + let mut tags = String::new(); + for (k, v) in &wire.tags { + let _ = write!(tags, ",{k}={v}"); + } + + let mut out = String::with_capacity(n.saturating_mul(64 + tags.len() + wire.field.len())); + for (i, value) in wire.values.iter().enumerate() { + if i > 0 { + out.push('\n'); + } + let _ = write!( + out, + "{}{tags} {}={value} {}", + wire.measurement, wire.field, ts_raw[i] + ); + } + Ok(out.into_bytes()) +} + +/// Parses columnar msgpack map into `Vec` (`precision` matches `/write` query param). +pub fn parse_columnar_msgpack_to_points( + body: &[u8], + precision: Option<&str>, +) -> Result, HyperbytedbError> { + let wire = decode_columnar_batch(body)?; + columnar_batch_to_points(&wire, precision) +} + +/// Expand a decoded columnar batch into `Vec`. +/// +/// Shares the measurement and tags allocation across all points using +/// clone-on-first then reuse, reducing heap churn vs the naive per-point clone. +pub fn columnar_batch_to_points( + wire: &ColumnarMsgpackBatch, + precision: Option<&str>, +) -> Result, HyperbytedbError> { + if wire.field.is_empty() { + return Err(HyperbytedbError::ColumnarMsgpackParse { + reason: "field name must be non-empty".into(), + }); + } + + let n = wire.values.len(); + if n == 0 { + return Ok(Vec::new()); + } + + let ts_ns_vec = columnar_timestamps_ns(wire, precision)?; + let mut points = Vec::with_capacity(n); for (i, v) in wire.values.iter().enumerate() { let mut fields = BTreeMap::new(); @@ -118,25 +204,7 @@ pub fn columnar_batch_to_record_batch( }); } - let precision_val = Precision::from_str_opt(precision); - - let ts_ns: Vec = if let Some(ref ts) = wire.timestamps { - if ts.len() != n { - return Err(HyperbytedbError::ColumnarMsgpackParse { - reason: format!( - "timestamps length {} does not match values length {}", - ts.len(), - n - ), - }); - } - ts.iter().map(|t| precision_val.to_nanos(*t)).collect() - } else { - let now = chrono::Utc::now() - .timestamp_nanos_opt() - .ok_or(HyperbytedbError::WallClockTimestampUnavailable)?; - vec![now; n] - }; + let ts_ns = columnar_timestamps_ns(wire, precision)?; let mut fields = vec![Field::new( "time", diff --git a/hyperbytedb/src/application/database_drop.rs b/hyperbytedb/src/application/database_drop.rs new file mode 100644 index 0000000..d64fc20 --- /dev/null +++ b/hyperbytedb/src/application/database_drop.rs @@ -0,0 +1,87 @@ +//! Full local teardown for `DROP DATABASE` (metadata + MV + WAL + chDB). + +use std::sync::Arc; + +use crate::application::materialized_view_service::MaterializedViewService; +use crate::error::HyperbytedbError; +use crate::ports::metadata::MetadataPort; +use crate::ports::points_sink::PointsSinkPort; +use crate::ports::wal::WalPort; + +/// Drop a database and await full local cleanup. +/// +/// MV and chDB table drops run concurrently but the caller waits for every task +/// to finish before returning. WAL purge and metadata removal are awaited inline. +pub async fn drop_database( + metadata: &Arc, + mv_service: Option<&MaterializedViewService>, + points_sink: Option<&Arc>, + wal: Option<&Arc>, + name: &str, +) -> Result<(), HyperbytedbError> { + metadata + .get_database(name) + .await? + .ok_or_else(|| HyperbytedbError::DatabaseNotFound(name.to_string()))?; + + let to_drop: Vec<(String, String)> = { + let rps = metadata.list_retention_policies(name).await?; + let mut pairs = Vec::new(); + for rp in &rps { + let measurements = metadata.list_measurements_for_rp(name, &rp.name).await?; + for m in measurements { + pairs.push((rp.name.clone(), m)); + } + } + pairs + }; + + if let Some(mv) = mv_service + && let Err(e) = mv.drop_all_in_database(name).await + { + tracing::warn!( + db = name, + error = %e, + "failed to cascade-drop materialized views for database" + ); + } + + if let Some(wal) = wal { + wal.purge_database(name).await?; + } + + metadata.drop_database(name).await?; + + if let Some(sink) = points_sink { + let db = name.to_string(); + let mut handles = Vec::with_capacity(to_drop.len()); + for (rp, measurement) in to_drop { + let sink = sink.clone(); + let db = db.clone(); + handles.push(tokio::spawn(async move { + sink.drop_measurement(&db, &rp, &measurement).await + })); + } + for handle in handles { + match handle.await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + tracing::warn!( + db = name, + error = %e, + "failed to drop chDB native table during DROP DATABASE" + ); + } + Err(e) => { + tracing::warn!( + db = name, + error = %e, + "chDB drop task panicked during DROP DATABASE" + ); + } + } + } + } + + Ok(()) +} diff --git a/hyperbytedb/src/application/flush_service.rs b/hyperbytedb/src/application/flush_service.rs index 017ddf5..d52d05a 100644 --- a/hyperbytedb/src/application/flush_service.rs +++ b/hyperbytedb/src/application/flush_service.rs @@ -423,9 +423,9 @@ impl FlushServiceImpl { if any_failed { // Stop this flush cycle (retry next tick). Returning Err also prevents // drain() from looping forever on a persistently-failing chunk. - return Err(HyperbytedbError::Internal( - "prepared flush sink write failed; holding WAL for retry".into(), - )); + return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "prepared flush sink write failed; holding WAL for retry", + ))); } Ok(()) @@ -458,7 +458,11 @@ impl FlushServiceImpl { return Ok(()); } - loop { + // Only read sequences captured at flush start (`snapshot_seq`). Entries + // appended mid-flush stay for the next tick; without this bound the + // prepared path would `continue` past the snapshot and fall through to a + // native `read_range` that always returns empty but still seeks RocksDB. + while cursor < snapshot_seq { let from_seq = cursor + 1; let wal_read_start = std::time::Instant::now(); @@ -477,7 +481,7 @@ impl FlushServiceImpl { { if !prepared.is_empty() { let wal_read_elapsed = wal_read_start.elapsed(); - histogram!("hyperbytedb_flush_wal_read_seconds") + histogram!("hyperbytedb_flush_wal_read_seconds", "path" => "prepared") .record(wal_read_elapsed.as_secs_f64()); self.flush_prepared_chunk( prepared, @@ -498,7 +502,8 @@ impl FlushServiceImpl { let entries = self.wal.read_range(from_seq, native_limit).await?; let wal_read_elapsed = wal_read_start.elapsed(); - histogram!("hyperbytedb_flush_wal_read_seconds").record(wal_read_elapsed.as_secs_f64()); + histogram!("hyperbytedb_flush_wal_read_seconds", "path" => "native") + .record(wal_read_elapsed.as_secs_f64()); if entries.is_empty() { break; } @@ -667,9 +672,9 @@ impl FlushServiceImpl { if any_failed { // Stop this flush cycle (retry next tick). Returning Err also prevents // drain() from looping forever on a persistently-failing chunk. - return Err(HyperbytedbError::Internal( - "native flush sink write failed; holding WAL for retry".into(), - )); + return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "native flush sink write failed; holding WAL for retry", + ))); } } @@ -752,9 +757,9 @@ mod tests { _ingest_seq_base: u64, _points: &[Point], ) -> Result { - Err(HyperbytedbError::Internal( - "simulated chDB write failure".into(), - )) + Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "simulated chDB write failure", + ))) } } diff --git a/hyperbytedb/src/application/ingest_metadata.rs b/hyperbytedb/src/application/ingest_metadata.rs index 19854de..673de2d 100644 --- a/hyperbytedb/src/application/ingest_metadata.rs +++ b/hyperbytedb/src/application/ingest_metadata.rs @@ -271,7 +271,9 @@ pub async fn register_series_from_series_table( } let parts: Vec<&str> = line.split('\t').collect(); let sid: u64 = parts[0].parse().map_err(|e| { - HyperbytedbError::Internal(format!("invalid series_id in {series_table_quoted}: {e}")) + HyperbytedbError::Internal( + format!("invalid series_id in {series_table_quoted}: {e}").into(), + ) })?; let mut tags = BTreeMap::new(); for (i, key) in tag_keys.iter().enumerate() { @@ -376,12 +378,10 @@ pub async fn prepare_columnar_metadata( for (tag_key, tag_value) in &batch.tags { let count = metadata .count_tag_values(db, rp, tag_key, Some(&batch.measurement)) - .await - .unwrap_or(0); + .await?; let total = if metadata .tag_value_is_known(db, rp, &batch.measurement, tag_key, tag_value) - .await - .unwrap_or(false) + .await? { count } else { @@ -599,8 +599,7 @@ pub async fn prepare_batch_metadata( for tag_key in tag_keys.iter() { let count = metadata .count_tag_values(db, rp, tag_key, Some(meas_name)) - .await - .unwrap_or(0); + .await?; let new_values: std::collections::BTreeSet<&String> = points .iter() .filter(|p| p.measurement == *meas_name) @@ -610,8 +609,7 @@ pub async fn prepare_batch_metadata( for v in &new_values { if !metadata .tag_value_is_known(db, rp, meas_name, tag_key, v) - .await - .unwrap_or(false) + .await? { novel.insert(v); } diff --git a/hyperbytedb/src/application/ingestion_service.rs b/hyperbytedb/src/application/ingestion_service.rs index f03a332..7b05521 100644 --- a/hyperbytedb/src/application/ingestion_service.rs +++ b/hyperbytedb/src/application/ingestion_service.rs @@ -8,8 +8,10 @@ use crate::application::ingest_metadata::{ IngestCardinalityLimits, IngestSchemaCache, prepare_batch_metadata, validate_point_count, }; use crate::application::line_protocol::parse_line_body_to_points_limited; -use crate::application::msgpack_ingest::parse_msgpack_body_to_points; -use crate::application::wal_append::append_points_with_prepared; +use crate::application::msgpack_ingest::parse_msgpack_body_to_points_limited; +use crate::application::wal_append::{ + ColumnarWalAppend, append_columnar_with_prepared, append_points_with_prepared, +}; use crate::domain::point::Point; use crate::error::HyperbytedbError; use crate::ports::ingestion::{IngestionPort, WritePayloadFormat}; @@ -111,7 +113,10 @@ impl IngestionPort for IngestionServiceImpl { #[cfg(feature = "columnar-ingest")] if matches!(format, WritePayloadFormat::ColumnarMsgpack) { - let wire = crate::application::columnar_msgpack::decode_columnar_batch(body)?; + let wire = crate::application::columnar_msgpack::decode_columnar_batch_limited( + body, + self.max_points_per_request, + )?; if wire.values.is_empty() { return Ok(()); } @@ -153,9 +158,19 @@ impl IngestionPort for IngestionServiceImpl { .record((t3 - t2).as_secs_f64()); let point_count = wire.values.len() as u64; - let points = - crate::application::columnar_msgpack::columnar_batch_to_points(&wire, precision)?; - self.append_points(db, &retention_policy, points, 0).await?; + append_columnar_with_prepared( + self.wal.as_ref(), + self.sink.as_ref(), + &ColumnarWalAppend { + db, + rp: &retention_policy, + wire: &wire, + precision, + origin_node_id: 0, + max_points_per_request: self.max_points_per_request, + }, + ) + .await?; let t4 = std::time::Instant::now(); histogram!("hyperbytedb_ingest_wal_append_seconds").record((t4 - t3).as_secs_f64()); @@ -171,7 +186,9 @@ impl IngestionPort for IngestionServiceImpl { WritePayloadFormat::LineProtocol => { parse_line_body_to_points_limited(body, precision, self.max_points_per_request)? } - WritePayloadFormat::Msgpack => parse_msgpack_body_to_points(body, precision)?, + WritePayloadFormat::Msgpack => { + parse_msgpack_body_to_points_limited(body, precision, self.max_points_per_request)? + } #[cfg(feature = "columnar-ingest")] WritePayloadFormat::ColumnarMsgpack => { unreachable!("handled by fast path above") diff --git a/hyperbytedb/src/application/materialized_view_service.rs b/hyperbytedb/src/application/materialized_view_service.rs index 062142c..aa8d887 100644 --- a/hyperbytedb/src/application/materialized_view_service.rs +++ b/hyperbytedb/src/application/materialized_view_service.rs @@ -37,6 +37,10 @@ impl MaterializedViewService { } } + pub fn points_sink(&self) -> &Arc { + &self.points_sink + } + pub async fn create( &self, mv: &CreateMaterializedViewStatement, @@ -86,8 +90,9 @@ impl MaterializedViewService { dest_db, dest_rp: dest_rp.clone(), dest_measurement, - ch_fact_mv_name: unquoted_fact_mv_name(&mv.database, &dest_rp, &mv.name), - ch_series_mv_name: unquoted_series_mv_name(&mv.database, &dest_rp, &mv.name), + ch_fact_mv_name: unquoted_fact_mv_name(&mv.database, &dest_rp, &mv.name).to_string(), + ch_series_mv_name: unquoted_series_mv_name(&mv.database, &dest_rp, &mv.name) + .to_string(), created_at: chrono::Utc::now().to_rfc3339(), backfill_on_create: mv.backfill_on_create, }; @@ -122,7 +127,8 @@ impl MaterializedViewService { let fact_mv = quoted_fact_mv_name(db, &def.dest_rp, name); let series_mv = quoted_series_mv_name(db, &def.dest_rp, name); - self.drop_ch_mv_objects(&fact_mv, &series_mv).await?; + self.drop_ch_mv_objects(fact_mv.as_str(), series_mv.as_str()) + .await?; // Drop the destination fact + series tables to avoid orphaned tables. if let Err(e) = self @@ -256,7 +262,8 @@ impl MaterializedViewService { ) -> Result<(), HyperbytedbError> { let fact_mv = quoted_fact_mv_name(&def.database, &def.dest_rp, &def.name); let series_mv = quoted_series_mv_name(&def.database, &def.dest_rp, &def.name); - self.drop_ch_mv_objects(&fact_mv, &series_mv).await + self.drop_ch_mv_objects(fact_mv.as_str(), series_mv.as_str()) + .await } async fn reconcile_one(&self, def: &MaterializedViewDef) -> Result { @@ -408,7 +415,7 @@ impl MaterializedViewService { .ensure_measurement_schema(&dest_db, &dest_rp, &dest_meta) .await?; - self.drop_ch_mv_objects(&fact_mv_quoted, &series_mv_quoted) + self.drop_ch_mv_objects(fact_mv_quoted.as_str(), series_mv_quoted.as_str()) .await?; self.query_port.execute_sql(&create_fact_mv).await?; @@ -438,7 +445,7 @@ impl MaterializedViewService { &dest_rp, &dest_measurement, &dest_meta, - &dest_series, + dest_series.as_str(), ) .await { @@ -457,7 +464,7 @@ impl MaterializedViewService { if let Err(e) = result { let _ = self - .drop_ch_mv_objects(&fact_mv_quoted, &series_mv_quoted) + .drop_ch_mv_objects(fact_mv_quoted.as_str(), series_mv_quoted.as_str()) .await; let _ = self .points_sink @@ -503,8 +510,8 @@ pub fn def_from_statement( dest_db, dest_rp: dest_rp.to_string(), dest_measurement, - ch_fact_mv_name: unquoted_fact_mv_name(&mv.database, dest_rp, &mv.name), - ch_series_mv_name: unquoted_series_mv_name(&mv.database, dest_rp, &mv.name), + ch_fact_mv_name: unquoted_fact_mv_name(&mv.database, dest_rp, &mv.name).to_string(), + ch_series_mv_name: unquoted_series_mv_name(&mv.database, dest_rp, &mv.name).to_string(), created_at: chrono::Utc::now().to_rfc3339(), backfill_on_create: mv.backfill_on_create, }) diff --git a/hyperbytedb/src/application/mod.rs b/hyperbytedb/src/application/mod.rs index 85c8321..da14e34 100644 --- a/hyperbytedb/src/application/mod.rs +++ b/hyperbytedb/src/application/mod.rs @@ -3,6 +3,7 @@ pub mod backup; pub mod cluster; pub mod columnar_msgpack; pub mod continuous_query_service; +pub mod database_drop; pub mod disk_monitor; pub mod flush_service; pub mod ingest_metadata; @@ -10,6 +11,7 @@ pub mod ingestion_service; pub mod line_protocol; pub mod materialized_view_service; pub mod msgpack_ingest; +pub mod msgpack_limits; pub mod peer_ingestion_service; pub mod peer_query_service; pub mod predicate_sql; diff --git a/hyperbytedb/src/application/msgpack_ingest.rs b/hyperbytedb/src/application/msgpack_ingest.rs index b6d9e95..7b5f93f 100644 --- a/hyperbytedb/src/application/msgpack_ingest.rs +++ b/hyperbytedb/src/application/msgpack_ingest.rs @@ -8,6 +8,8 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use crate::application::ingest_metadata::validate_point_count; +use crate::application::msgpack_limits::peek_top_level_array_len; use crate::domain::database::Precision; use crate::domain::point::{FieldValue, Point}; use crate::error::HyperbytedbError; @@ -25,11 +27,24 @@ struct MsgpackPointWire { pub fn parse_msgpack_body_to_points( body: &[u8], precision: Option<&str>, +) -> Result, HyperbytedbError> { + parse_msgpack_body_to_points_limited(body, precision, 0) +} + +/// Like [`parse_msgpack_body_to_points`] but rejects batches above `max_points` +/// before deserializing point maps (`0` = default cap). +pub fn parse_msgpack_body_to_points_limited( + body: &[u8], + precision: Option<&str>, + max_points: usize, ) -> Result, HyperbytedbError> { if body.is_empty() { return Ok(Vec::new()); } + let count = peek_top_level_array_len(body)?; + validate_point_count(count, max_points)?; + let wire: Vec = rmp_serde::from_slice(body).map_err(|e| HyperbytedbError::MsgpackParse { reason: e.to_string(), diff --git a/hyperbytedb/src/application/msgpack_limits.rs b/hyperbytedb/src/application/msgpack_limits.rs new file mode 100644 index 0000000..c22ef03 --- /dev/null +++ b/hyperbytedb/src/application/msgpack_limits.rs @@ -0,0 +1,203 @@ +//! MessagePack size guards before full deserialization. + +use std::io::Cursor; + +use rmp::Marker; +use rmp::decode::{self, RmpRead}; + +use crate::error::HyperbytedbError; + +fn msgpack_parse_err(e: E) -> HyperbytedbError { + HyperbytedbError::MsgpackParse { + reason: format!("{e:?}"), + } +} + +fn skip_msgpack_value(rd: &mut R) -> Result<(), HyperbytedbError> { + match decode::read_marker(rd).map_err(msgpack_parse_err)? { + Marker::Null | Marker::True | Marker::False => Ok(()), + Marker::FixPos(_) | Marker::FixNeg(_) => Ok(()), + Marker::U8 | Marker::I8 => skip_bytes(rd, 1), + Marker::U16 | Marker::I16 => skip_bytes(rd, 2), + Marker::U32 | Marker::I32 | Marker::F32 => skip_bytes(rd, 4), + Marker::U64 | Marker::I64 | Marker::F64 => skip_bytes(rd, 8), + Marker::FixStr(len) => skip_bytes(rd, u32::from(len)), + Marker::Str8 => { + let len = u32::from(decode::read_u8(rd).map_err(msgpack_parse_err)?); + skip_bytes(rd, len) + } + Marker::Str16 => { + let len = u32::from(decode::read_u16(rd).map_err(msgpack_parse_err)?); + skip_bytes(rd, len) + } + Marker::Str32 => { + let len = decode::read_u32(rd).map_err(msgpack_parse_err)?; + skip_bytes(rd, len) + } + Marker::Bin8 => { + let len = u32::from(decode::read_u8(rd).map_err(msgpack_parse_err)?); + skip_bytes(rd, len) + } + Marker::Bin16 => { + let len = u32::from(decode::read_u16(rd).map_err(msgpack_parse_err)?); + skip_bytes(rd, len) + } + Marker::Bin32 => { + let len = decode::read_u32(rd).map_err(msgpack_parse_err)?; + skip_bytes(rd, len) + } + Marker::FixArray(len) => skip_array_elements(rd, u32::from(len)), + Marker::Array16 => { + let len = decode::read_u16(rd).map_err(msgpack_parse_err)?; + skip_array_elements(rd, u32::from(len)) + } + Marker::Array32 => { + let len = decode::read_u32(rd).map_err(msgpack_parse_err)?; + skip_array_elements(rd, len) + } + Marker::FixMap(len) => skip_map_entries(rd, u32::from(len)), + Marker::Map16 => { + let len = decode::read_u16(rd).map_err(msgpack_parse_err)?; + skip_map_entries(rd, u32::from(len)) + } + Marker::Map32 => { + let len = decode::read_u32(rd).map_err(msgpack_parse_err)?; + skip_map_entries(rd, len) + } + Marker::FixExt1 => skip_bytes(rd, 2), + Marker::FixExt2 => skip_bytes(rd, 3), + Marker::FixExt4 => skip_bytes(rd, 5), + Marker::FixExt8 => skip_bytes(rd, 9), + Marker::FixExt16 => skip_bytes(rd, 17), + Marker::Ext8 => { + let len = decode::read_u8(rd).map_err(msgpack_parse_err)?; + skip_bytes(rd, u32::from(len) + 1) + } + Marker::Ext16 => { + let len = decode::read_u16(rd).map_err(msgpack_parse_err)?; + skip_bytes(rd, u32::from(len) + 1) + } + Marker::Ext32 => { + let len = decode::read_u32(rd).map_err(msgpack_parse_err)?; + skip_bytes(rd, len.saturating_add(1)) + } + Marker::Reserved => Err(HyperbytedbError::MsgpackParse { + reason: "reserved msgpack marker".into(), + }), + } +} + +fn skip_bytes(rd: &mut R, n: u32) -> Result<(), HyperbytedbError> { + let mut buf = [0u8; 4096]; + let mut remaining = n as u64; + while remaining > 0 { + let chunk = remaining.min(buf.len() as u64) as usize; + rd.read_exact_buf(&mut buf[..chunk]) + .map_err(msgpack_parse_err)?; + remaining -= chunk as u64; + } + Ok(()) +} + +fn skip_array_elements(rd: &mut R, len: u32) -> Result<(), HyperbytedbError> { + for _ in 0..len { + skip_msgpack_value(rd)?; + } + Ok(()) +} + +fn skip_map_entries(rd: &mut R, len: u32) -> Result<(), HyperbytedbError> { + for _ in 0..len { + skip_msgpack_value(rd)?; + skip_msgpack_value(rd)?; + } + Ok(()) +} + +fn read_msgpack_str(rd: &mut R) -> Result { + let len = match decode::read_marker(rd).map_err(msgpack_parse_err)? { + Marker::FixStr(len) => u32::from(len), + Marker::Str8 => u32::from(decode::read_u8(rd).map_err(msgpack_parse_err)?), + Marker::Str16 => u32::from(decode::read_u16(rd).map_err(msgpack_parse_err)?), + Marker::Str32 => decode::read_u32(rd).map_err(msgpack_parse_err)?, + marker => { + return Err(HyperbytedbError::MsgpackParse { + reason: format!("expected msgpack string, got {marker:?}"), + }); + } + }; + let mut buf = vec![0u8; len as usize]; + rd.read_exact_buf(&mut buf).map_err(msgpack_parse_err)?; + String::from_utf8(buf).map_err(msgpack_parse_err) +} + +/// Return the element count of a top-level msgpack array without deserializing elements. +pub fn peek_top_level_array_len(body: &[u8]) -> Result { + if body.is_empty() { + return Ok(0); + } + let mut cur = Cursor::new(body); + let len = decode::read_array_len(&mut cur).map_err(msgpack_parse_err)?; + Ok(len as usize) +} + +/// Return the `values` array length from a columnar msgpack map without full deserialization. +pub fn peek_columnar_values_len(body: &[u8]) -> Result, HyperbytedbError> { + if body.is_empty() { + return Ok(None); + } + let mut cur = Cursor::new(body); + let map_len = decode::read_map_len(&mut cur).map_err(msgpack_parse_err)?; + for _ in 0..map_len { + let key = read_msgpack_str(&mut cur)?; + if key == "values" { + let len = decode::read_array_len(&mut cur).map_err(msgpack_parse_err)?; + return Ok(Some(len as usize)); + } + skip_msgpack_value(&mut cur)?; + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::application::columnar_msgpack::ColumnarMsgpackBatch; + use crate::domain::point::FieldValue; + use std::collections::BTreeMap; + + #[test] + fn peek_top_level_array_len_matches_decode() { + #[derive(serde::Serialize)] + struct Wire { + measurement: String, + #[serde(default)] + tags: BTreeMap, + fields: BTreeMap, + timestamp: Option, + } + let mut fields = BTreeMap::new(); + fields.insert("idle".into(), FieldValue::Float(0.5)); + let wire = vec![Wire { + measurement: "cpu".into(), + tags: BTreeMap::new(), + fields, + timestamp: Some(1), + }]; + let body = rmp_serde::to_vec_named(&wire).unwrap(); + assert_eq!(peek_top_level_array_len(&body).unwrap(), 1); + } + + #[test] + fn peek_columnar_values_len_matches_wire() { + let batch = ColumnarMsgpackBatch { + measurement: "cpu".into(), + tags: BTreeMap::new(), + field: "idle".into(), + values: vec![1.0, 2.0, 3.0], + timestamps: None, + }; + let body = rmp_serde::to_vec_named(&batch).unwrap(); + assert_eq!(peek_columnar_values_len(&body).unwrap(), Some(3)); + } +} diff --git a/hyperbytedb/src/application/peer_ingestion_service.rs b/hyperbytedb/src/application/peer_ingestion_service.rs index 4159284..fa19e63 100644 --- a/hyperbytedb/src/application/peer_ingestion_service.rs +++ b/hyperbytedb/src/application/peer_ingestion_service.rs @@ -11,9 +11,11 @@ use crate::application::ingest_metadata::{ use crate::application::line_protocol::{ encode_points_to_line_protocol, parse_line_body_to_points_limited, }; -use crate::application::msgpack_ingest::parse_msgpack_body_to_points; +use crate::application::msgpack_ingest::parse_msgpack_body_to_points_limited; use crate::application::replication_dispatch::dispatch_outbound_replication; -use crate::application::wal_append::append_points_with_prepared; +use crate::application::wal_append::{ + ColumnarWalAppend, append_columnar_with_prepared, append_points_with_prepared, +}; use crate::config::{ReplicationConfig, ReplicationMode}; use crate::domain::database::Precision; use crate::error::HyperbytedbError; @@ -159,7 +161,10 @@ impl IngestionPort for PeerIngestionService { // Columnar fast path: decode once, metadata from batch, then expand for WAL/replication #[cfg(feature = "columnar-ingest")] if matches!(format, WritePayloadFormat::ColumnarMsgpack) { - let wire = crate::application::columnar_msgpack::decode_columnar_batch(body)?; + let wire = crate::application::columnar_msgpack::decode_columnar_batch_limited( + body, + self.max_points_per_request, + )?; if wire.values.is_empty() { return Ok(()); } @@ -201,19 +206,24 @@ impl IngestionPort for PeerIngestionService { .record((t3 - t2).as_secs_f64()); let point_count = wire.values.len() as u64; - let points = - crate::application::columnar_msgpack::columnar_batch_to_points(&wire, precision)?; let precision_val = Precision::from_str_opt(precision); - let replication_body = encode_points_to_line_protocol(&points, precision_val)?; + let replication_body = + crate::application::columnar_msgpack::columnar_wire_to_line_protocol( + &wire, + precision_val, + )?; - let wal_seq = append_points_with_prepared( + let wal_seq = append_columnar_with_prepared( self.wal.as_ref(), self.sink.as_ref(), - db, - &retention_policy, - points, - self.node_id, - self.max_points_per_request, + &ColumnarWalAppend { + db, + rp: &retention_policy, + wire: &wire, + precision, + origin_node_id: self.node_id, + max_points_per_request: self.max_points_per_request, + }, ) .await?; @@ -240,7 +250,9 @@ impl IngestionPort for PeerIngestionService { WritePayloadFormat::LineProtocol => { parse_line_body_to_points_limited(body, precision, self.max_points_per_request)? } - WritePayloadFormat::Msgpack => parse_msgpack_body_to_points(body, precision)?, + WritePayloadFormat::Msgpack => { + parse_msgpack_body_to_points_limited(body, precision, self.max_points_per_request)? + } #[cfg(feature = "columnar-ingest")] WritePayloadFormat::ColumnarMsgpack => { unreachable!("handled by fast path above") diff --git a/hyperbytedb/src/application/peer_query_service.rs b/hyperbytedb/src/application/peer_query_service.rs index 96f869a..4aed776 100644 --- a/hyperbytedb/src/application/peer_query_service.rs +++ b/hyperbytedb/src/application/peer_query_service.rs @@ -85,11 +85,30 @@ impl PeerQueryService { let cluster_req = ClusterRequest::SchemaMutation(Box::new(req)); self.client_write_with_forward(raft, cluster_req).await } else { - self.replication_port.clone().replicate_mutation(req); - Ok(()) + self.replication_port + .clone() + .replicate_mutation_sync(req) + .await } } + async fn execute_inner_statement( + &self, + db: &str, + stmt_query: &str, + epoch: Option<&str>, + retention_policy: Option<&str>, + caller: Option<&crate::domain::user::StoredUser>, + statement_id: u32, + ) -> StatementResult { + execute_or_error( + self.inner + .execute_query(db, stmt_query, epoch, retention_policy, caller) + .await, + statement_id, + ) + } + async fn client_write_with_forward( &self, raft: &HyperbytedbRaft, @@ -102,11 +121,11 @@ impl PeerQueryService { if resp.data.ok { return Ok(()); } - return Err(HyperbytedbError::Internal( + return Err(HyperbytedbError::Internal(crate::error::ChainedError::new( resp.data .message - .unwrap_or_else(|| "raft schema mutation apply failed".into()), - )); + .unwrap_or_else(|| "raft schema mutation apply failed".to_string()), + ))); } Err(e) => { if let Some(forward) = e.forward_to_leader::() { @@ -136,15 +155,15 @@ impl PeerQueryService { } } tracing::error!(error = %e, "failed to replicate mutation via raft"); - return Err(HyperbytedbError::Internal(format!( - "raft replication failed: {e}" - ))); + return Err(HyperbytedbError::Internal( + format!("raft replication failed: {e}").into(), + )); } } } - Err(HyperbytedbError::Internal( - "raft replication failed after retries".into(), - )) + Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "raft replication failed after retries", + ))) } async fn execute_raft_mutation( @@ -232,17 +251,17 @@ fn resolve_leader_addr_with_lookup( let leader_id = forward.leader_id.or(fallback_leader_id); let leader_id = leader_id.ok_or_else(|| { - HyperbytedbError::Internal( - "raft replication failed: forward to leader but leader id is unknown".into(), - ) + HyperbytedbError::Internal(crate::error::ChainedError::new( + "raft replication failed: forward to leader but leader id is unknown", + )) })?; raft_lookup(leader_id) .or_else(|| membership_lookup(leader_id)) .ok_or_else(|| { - HyperbytedbError::Internal(format!( - "raft replication failed: leader {leader_id} has no known address" - )) + HyperbytedbError::Internal( + format!("raft replication failed: leader {leader_id} has no known address").into(), + ) }) } @@ -261,36 +280,38 @@ async fn forward_client_write( .send() .await .map_err(|e| { - HyperbytedbError::Internal(format!( - "raft replication failed: forward to leader {url}: {e}" - )) + HyperbytedbError::Internal( + format!("raft replication failed: forward to leader {url}: {e}").into(), + ) })?; let status = resp.status(); let body_text = resp.text().await.map_err(|e| { - HyperbytedbError::Internal(format!( - "raft replication failed: read leader response: {e}" - )) + HyperbytedbError::Internal( + format!("raft replication failed: read leader response: {e}").into(), + ) })?; if !status.is_success() { - return Err(HyperbytedbError::Internal(format!( - "raft replication failed: leader returned HTTP {status}: {body_text}" - ))); + return Err(HyperbytedbError::Internal( + format!("raft replication failed: leader returned HTTP {status}: {body_text}").into(), + )); } let body: ClusterResponse = serde_json::from_str(&body_text).map_err(|e| { - HyperbytedbError::Internal(format!( - "raft replication failed: invalid leader response: {e}: {body_text}" - )) + HyperbytedbError::Internal( + format!("raft replication failed: invalid leader response: {e}: {body_text}").into(), + ) })?; if body.ok { Ok(()) } else { - Err(HyperbytedbError::Internal(body.message.unwrap_or_else( - || "raft leader rejected schema mutation".into(), - ))) + Err(HyperbytedbError::Internal( + body.message + .unwrap_or_else(|| "raft leader rejected schema mutation".into()) + .into(), + )) } } @@ -445,8 +466,7 @@ async fn mutation_request_from_statement( password, admin, } => { - let password_hash = - crate::adapters::http::auth_middleware::hash_password(password).unwrap_or_default(); + let password_hash = hash_password_for_replication(password)?; Ok(MutationRequest::CreateUser { username: username.clone(), password_hash, @@ -455,8 +475,7 @@ async fn mutation_request_from_statement( } Statement::DropUser(username) => Ok(MutationRequest::DropUser(username.clone())), Statement::SetPassword { username, password } => { - let password_hash = - crate::adapters::http::auth_middleware::hash_password(password).unwrap_or_default(); + let password_hash = hash_password_for_replication(password)?; Ok(MutationRequest::SetPassword { username: username.clone(), password_hash, @@ -525,9 +544,9 @@ async fn mutation_request_from_statement( username: username.clone(), database: database.clone(), }), - _ => Err(HyperbytedbError::Internal( - "not a cluster mutation statement".into(), - )), + _ => Err(HyperbytedbError::Internal(crate::error::ChainedError::new( + "not a cluster mutation statement", + ))), } } @@ -564,6 +583,11 @@ impl QueryService for PeerQueryService { retention_policy: Option<&str>, caller: Option<&crate::domain::user::StoredUser>, ) -> Result { + let stmt_queries: Vec = crate::timeseriesql::lexer::split_statements(query)? + .into_iter() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); let stmts = crate::timeseriesql::parse(query)?; if !stmts.iter().any(is_cluster_mutation) { @@ -577,6 +601,7 @@ impl QueryService for PeerQueryService { let use_raft = self.raft.get().is_some(); for (i, stmt) in stmts.into_iter().enumerate() { let statement_id = i as u32; + let stmt_query = stmt_queries.get(i).map(String::as_str).unwrap_or(query); if use_raft && is_cluster_mutation(&stmt) { results.push( self.execute_raft_mutation(db, caller, statement_id, &stmt) @@ -586,22 +611,16 @@ impl QueryService for PeerQueryService { } let result = match stmt { Statement::CreateDatabase(ref stmt) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::CreateDatabase { @@ -615,22 +634,16 @@ impl QueryService for PeerQueryService { result } Statement::DropDatabase(ref name) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::DropDatabase(name.clone())) @@ -641,22 +654,16 @@ impl QueryService for PeerQueryService { result } Statement::Delete(ref del) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let del_rp = if let Some(rp) = retention_policy { rp.to_string() @@ -684,22 +691,16 @@ impl QueryService for PeerQueryService { result } Statement::CreateContinuousQuery(ref cq) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let def = match ContinuousQueryDef::from_create(cq) { Ok(def) => def, @@ -726,22 +727,16 @@ impl QueryService for PeerQueryService { ref name, db: ref cq_db, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let target_db = if cq_db.is_empty() { db } else { cq_db }; if let Err(e) = self @@ -757,22 +752,16 @@ impl QueryService for PeerQueryService { result } Statement::CreateMaterializedView(ref mv) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let source_rp = match resolve_mv_source_rp(&self.metadata, mv).await { Ok(rp) => rp, @@ -808,22 +797,16 @@ impl QueryService for PeerQueryService { ref name, db: ref mv_db, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let target_db = if mv_db.is_empty() { db } else { mv_db }; if let Err(e) = self @@ -846,11 +829,16 @@ impl QueryService for PeerQueryService { ref shard_duration, is_default, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() { let dur = duration .as_ref() @@ -881,11 +869,16 @@ impl QueryService for PeerQueryService { result } Statement::DropRetentionPolicyStmt { ref name, ref db } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::DropRetentionPolicy { @@ -903,34 +896,46 @@ impl QueryService for PeerQueryService { ref password, admin, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() { - let password_hash = - crate::adapters::http::auth_middleware::hash_password(password) - .unwrap_or_default(); - if let Err(e) = self - .replicate_mutation(MutationRequest::CreateUser { - username: username.clone(), - password_hash, - admin, - }) - .await - { - result.error = Some(e.to_string()); + match hash_password_for_replication(password) { + Ok(password_hash) => { + if let Err(e) = self + .replicate_mutation(MutationRequest::CreateUser { + username: username.clone(), + password_hash, + admin, + }) + .await + { + result.error = Some(e.to_string()); + } + } + Err(e) => result.error = Some(e.to_string()), } } result } Statement::DropUser(ref username) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::DropUser(username.clone())) @@ -944,23 +949,30 @@ impl QueryService for PeerQueryService { ref username, ref password, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() { - let password_hash = - crate::adapters::http::auth_middleware::hash_password(password) - .unwrap_or_default(); - if let Err(e) = self - .replicate_mutation(MutationRequest::SetPassword { - username: username.clone(), - password_hash, - }) - .await - { - result.error = Some(e.to_string()); + match hash_password_for_replication(password) { + Ok(password_hash) => { + if let Err(e) = self + .replicate_mutation(MutationRequest::SetPassword { + username: username.clone(), + password_hash, + }) + .await + { + result.error = Some(e.to_string()); + } + } + Err(e) => result.error = Some(e.to_string()), } } result @@ -973,11 +985,16 @@ impl QueryService for PeerQueryService { ref shard_duration, is_default, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() { let change = RetentionPolicyChange { duration: duration.as_ref().map(|d| { @@ -1005,11 +1022,16 @@ impl QueryService for PeerQueryService { result } Statement::DropSeries(ref s) => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, + statement_id, + ) .await; - let mut result = execute_or_error(resp, statement_id); if result.error.is_none() { let target_db = s.database.as_deref().unwrap_or(db); let measurement = s.from.as_ref().and_then(|n| match n { @@ -1046,22 +1068,16 @@ impl QueryService for PeerQueryService { ref name, rp: ref stmt_rp, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() { let dm_rp = if let Some(rp) = stmt_rp { rp.clone() @@ -1085,22 +1101,16 @@ impl QueryService for PeerQueryService { ref username, ref database, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { - statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::Grant { @@ -1117,22 +1127,16 @@ impl QueryService for PeerQueryService { ref username, ref database, } => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await; - let mut result = match resp { - Ok(r) => r.results.into_iter().next().unwrap_or(StatementResult { - statement_id, - series: Some(vec![]), - error: None, - }), - Err(e) => StatementResult { + let mut result = self + .execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: None, - error: Some(e.to_string()), - }, - }; + ) + .await; if result.error.is_none() && let Err(e) = self .replicate_mutation(MutationRequest::Revoke { @@ -1146,15 +1150,15 @@ impl QueryService for PeerQueryService { result } _ => { - let resp = self - .inner - .execute_query(db, query, epoch, retention_policy, caller) - .await?; - resp.results.into_iter().next().unwrap_or(StatementResult { + self.execute_inner_statement( + db, + stmt_query, + epoch, + retention_policy, + caller, statement_id, - series: Some(vec![]), - error: None, - }) + ) + .await } }; results.push(result); @@ -1172,6 +1176,11 @@ impl QueryService for PeerQueryService { } } +fn hash_password_for_replication(password: &str) -> Result { + crate::adapters::http::auth_middleware::hash_password(password) + .map_err(|e| HyperbytedbError::Internal(crate::error::ChainedError::from_error(e))) +} + fn execute_or_error( resp: Result, statement_id: u32, diff --git a/hyperbytedb/src/application/predicate_sql.rs b/hyperbytedb/src/application/predicate_sql.rs index a759127..777d0bf 100644 --- a/hyperbytedb/src/application/predicate_sql.rs +++ b/hyperbytedb/src/application/predicate_sql.rs @@ -15,11 +15,16 @@ pub async fn build_predicate_sql( measurement: &str, cond: &Expr, ) -> Result { - let mapping = metadata + let meta = metadata .get_measurement(db, rp, measurement) .await? - .map(|m| ColumnMapping::from_measurement_meta(&m)); + .ok_or_else(|| { + HyperbytedbError::QueryParse(format!( + "measurement \"{measurement}\" not found in database \"{db}\"" + )) + })?; + let mapping = ColumnMapping::from_measurement_meta(&meta); let mut sql = String::new(); - to_clickhouse::translate_condition_with_mapping(cond, mapping.as_ref(), &mut sql)?; + to_clickhouse::translate_condition(cond, &mapping, &mut sql)?; Ok(sql) } diff --git a/hyperbytedb/src/application/query_service.rs b/hyperbytedb/src/application/query_service.rs index 917c2f1..1954d99 100644 --- a/hyperbytedb/src/application/query_service.rs +++ b/hyperbytedb/src/application/query_service.rs @@ -127,15 +127,23 @@ impl QueryServiceImpl { self } + async fn with_query_timeout( + &self, + fut: impl std::future::Future>, + ) -> Result { + let timeout = std::time::Duration::from_secs(self.query_timeout_secs); + match tokio::time::timeout(timeout, fut).await { + Ok(result) => result, + Err(_) => Err(HyperbytedbError::QueryTimeout), + } + } + /// Execute one InfluxDB v1-style continuous query run at `now`. pub async fn execute_continuous_query( &self, cq: &mut ContinuousQueryDef, now: chrono::DateTime, ) -> Result { - use metrics::{counter, histogram}; - use std::time::Instant; - cq.normalize()?; if !should_run(now, cq) { return Err(HyperbytedbError::QueryParse( @@ -143,8 +151,20 @@ impl QueryServiceImpl { )); } + self.with_query_timeout(self.execute_continuous_query_inner(cq, now)) + .await + } + + async fn execute_continuous_query_inner( + &self, + cq: &mut ContinuousQueryDef, + now: chrono::DateTime, + ) -> Result { + use metrics::{counter, histogram}; + use std::time::Instant; + let started = Instant::now(); - let window = coverage_window(now, cq); + let window = coverage_window(now, cq)?; let start_nanos = window.start.timestamp_nanos_opt().unwrap_or(0); let end_nanos = window.end.timestamp_nanos_opt().unwrap_or(0); @@ -388,7 +408,6 @@ impl QueryService for QueryServiceImpl { retention_policy: Option<&str>, caller: Option<&crate::domain::user::StoredUser>, ) -> Result { - let timeout = std::time::Duration::from_secs(self.query_timeout_secs); let caller_owned = caller.cloned(); let fut = async { let stmts = crate::timeseriesql::parse(query)?; @@ -455,10 +474,9 @@ impl QueryService for QueryServiceImpl { Ok(QueryResponse { results }) }; - match tokio::time::timeout(timeout, fut).await { - Ok(result) => result, - Err(_) => Err(HyperbytedbError::QueryTimeout), - } + // Fail the whole HTTP response on timeout; mutating statements executed before + // expiry remain committed (Influx-style best-effort batch semantics). + self.with_query_timeout(fut).await } async fn execute_continuous_query( @@ -534,8 +552,9 @@ async fn execute_statement( let rp = resolve_retention_policy_for_select(svc.metadata.as_ref(), db, None, query_rp) .await?; let names = list_measurements_for_rp(svc, db, &rp).await?; + let filtered = filter_names_by_selector(&names, s.measurement_filter.as_ref()); let columns = vec!["name".to_string()]; - let values: Vec> = names + let values: Vec> = filtered .iter() .map(|n| vec![serde_json::Value::String(n.clone())]) .collect(); @@ -628,25 +647,8 @@ async fn execute_statement( .await? }; - let matching_keys: Vec = match &s.tag_key { - TagKeySelector::All => all_tag_keys, - TagKeySelector::Eq(k) => vec![k.clone()], - TagKeySelector::Neq(k) => all_tag_keys.into_iter().filter(|tk| tk != k).collect(), - TagKeySelector::Regex(pattern) => match regex::Regex::new(pattern) { - Ok(re) => all_tag_keys - .into_iter() - .filter(|tk| re.is_match(tk)) - .collect(), - Err(_) => vec![], - }, - TagKeySelector::In(keys) => { - let key_set: std::collections::HashSet<&String> = keys.iter().collect(); - all_tag_keys - .into_iter() - .filter(|tk| key_set.contains(tk)) - .collect() - } - }; + let matching_keys: Vec = + filter_names_by_selector(&all_tag_keys, Some(&s.tag_key)); let mut all_values = Vec::new(); for tag_key in &matching_keys { @@ -744,51 +746,14 @@ async fn execute_statement( }) } Statement::DropDatabase(name) => { - svc.metadata - .get_database(name) - .await? - .ok_or_else(|| HyperbytedbError::DatabaseNotFound(name.clone()))?; - // Snapshot measurements + retention policies before - // metadata drops them so the native sink can DROP TABLE - // each backing chDB table. - let to_drop: Vec<(String, String)> = { - let rps = svc - .metadata - .list_retention_policies(name) - .await - .unwrap_or_default(); - let measurements = svc - .metadata - .list_measurements(name) - .await - .unwrap_or_default(); - let mut pairs = Vec::with_capacity(rps.len() * measurements.len()); - for rp in &rps { - for m in &measurements { - pairs.push((rp.name.clone(), m.clone())); - } - } - pairs - }; - if let Err(e) = svc.mv_service.drop_all_in_database(name).await { - tracing::warn!( - db = name, - error = %e, - "failed to cascade-drop materialized views for database" - ); - } - svc.metadata.drop_database(name).await?; - for (rp, m) in &to_drop { - if let Err(e) = svc.points_sink.drop_measurement(name, rp, m).await { - tracing::warn!( - db = name, - rp = %rp, - measurement = %m, - error = %e, - "failed to drop chDB native table during DROP DATABASE" - ); - } - } + crate::application::database_drop::drop_database( + &svc.metadata, + Some(&svc.mv_service), + Some(&svc.points_sink), + Some(&svc.wal), + name, + ) + .await?; Ok(StatementResult { statement_id, series: Some(vec![]), @@ -1248,7 +1213,9 @@ async fn execute_statement( admin, } => { let password_hash = crate::adapters::http::auth_middleware::hash_password(password) - .map_err(|e| HyperbytedbError::Internal(e.to_string()))?; + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) + })?; svc.metadata .create_user(username, &password_hash, *admin) .await?; @@ -1270,7 +1237,9 @@ async fn execute_statement( let existing = svc.metadata.get_user(username).await?; let is_admin = existing.map(|u| u.admin).unwrap_or(false); let password_hash = crate::adapters::http::auth_middleware::hash_password(password) - .map_err(|e| HyperbytedbError::Internal(e.to_string()))?; + .map_err(|e| { + HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) + })?; svc.metadata .create_user(username, &password_hash, is_admin) .await?; @@ -1426,9 +1395,9 @@ fn parse_chdb_json_line(line: &str) -> Result, pattern: &str) -> Vec { + match Regex::new(pattern) { + Ok(re) => items.into_iter().filter(|s| re.is_match(s)).collect(), + Err(_) => Vec::new(), + } +} + +fn filter_names_by_selector(names: &[String], selector: Option<&TagKeySelector>) -> Vec { + match selector { + None | Some(TagKeySelector::All) => names.to_vec(), + Some(TagKeySelector::Eq(k)) => names.iter().filter(|n| *n == k).cloned().collect(), + Some(TagKeySelector::Neq(k)) => names.iter().filter(|n| *n != k).cloned().collect(), + Some(TagKeySelector::Regex(pattern)) => { + filter_strings_by_show_regex(names.to_vec(), pattern) + } + Some(TagKeySelector::In(keys)) => { + let key_set: std::collections::HashSet<&String> = keys.iter().collect(); + names + .iter() + .filter(|n| key_set.contains(n)) + .cloned() + .collect() + } + } +} + fn regex_pattern_matches(pattern: &str) -> Box bool + '_> { let anchored = if pattern.starts_with('^') { pattern.to_string() @@ -2037,7 +2033,8 @@ async fn list_measurements_for_rp( let all = svc.metadata.list_measurements(db).await?; let mut names = Vec::new(); for measurement in all { - let table = unquoted_table_name(db, rp, &measurement); + let table_ident = unquoted_table_name(db, rp, &measurement); + let table = table_ident.as_str(); let sql = format!( "SELECT count() FROM system.tables WHERE database = 'default' AND name = '{table}' FORMAT TabSeparated" ); @@ -2083,7 +2080,7 @@ async fn tag_keys_from_series_table( let mut keys: Vec = mapping .tag_keys .iter() - .filter(|logical| phys_cols.contains(&mapping.tag_column_name(logical))) + .filter(|logical| phys_cols.contains(&mapping.physical_tag_column_name(logical))) .cloned() .collect(); keys.sort(); @@ -2114,7 +2111,7 @@ async fn tag_values_for_measurement( let Some(mapping) = mapping else { return Ok(Vec::new()); }; - let phys = mapping.tag_column_name(tag_key); + let phys = mapping.physical_tag_column_name(tag_key); if !series_table_columns(svc, db, rp, measurement) .await? .contains(&phys) @@ -2145,7 +2142,8 @@ async fn fact_table_columns( measurement: &str, ) -> Result, HyperbytedbError> { use crate::domain::chdb_naming::unquoted_table_name; - let table = unquoted_table_name(db, rp, measurement); + let table_ident = unquoted_table_name(db, rp, measurement); + let table = table_ident.as_str(); let sql = format!( "SELECT name FROM system.columns WHERE table = '{}' AND name NOT IN ('series_id', 'time', 'origin_node_id', 'ingest_seq') FORMAT TabSeparated", table.replace('\'', "''") @@ -2165,7 +2163,8 @@ async fn series_table_columns( rp: &str, measurement: &str, ) -> Result, HyperbytedbError> { - let table = unquoted_series_table_name(db, rp, measurement); + let table_ident = unquoted_series_table_name(db, rp, measurement); + let table = table_ident.as_str(); let sql = format!( "SELECT name FROM system.columns WHERE table = '{}' AND name != 'series_id' FORMAT TabSeparated", table.replace('\'', "''") @@ -2491,3 +2490,20 @@ mod auth_tests { ); } } + +#[cfg(test)] +mod show_regex_tests { + use super::filter_strings_by_show_regex; + + #[test] + fn invalid_show_regex_returns_empty() { + assert!(filter_strings_by_show_regex(vec!["host".into()], "[invalid").is_empty()); + } + + #[test] + fn valid_show_regex_filters() { + let keys = vec!["host".into(), "region".into()]; + let out = filter_strings_by_show_regex(keys, "^ho"); + assert_eq!(out, vec!["host".to_string()]); + } +} diff --git a/hyperbytedb/src/application/replication_apply.rs b/hyperbytedb/src/application/replication_apply.rs index a720770..ff5146e 100644 --- a/hyperbytedb/src/application/replication_apply.rs +++ b/hyperbytedb/src/application/replication_apply.rs @@ -95,21 +95,16 @@ impl ReplicationApplyQueue { ) -> Arc { let depth = depth.max(1); let num_workers = num_workers.max(1); - let (tx, rx) = mpsc::channel::(depth); - let rx = Arc::new(tokio::sync::Mutex::new(rx)); + let (tx, mut rx) = mpsc::channel::(depth); let sem = Arc::new(Semaphore::new(num_workers)); let schema_cache = Arc::new(IngestSchemaCache::new()); - let dispatch_rx = rx.clone(); let dispatch_sem = sem.clone(); tokio::spawn(async move { loop { - let job = { - let mut guard = dispatch_rx.lock().await; - match guard.recv().await { - Some(j) => j, - None => break, - } + let job = match rx.recv().await { + Some(j) => j, + None => break, }; let permit = match dispatch_sem.clone().acquire_owned().await { diff --git a/hyperbytedb/src/application/replication_dispatch.rs b/hyperbytedb/src/application/replication_dispatch.rs index 12b0404..28c10af 100644 --- a/hyperbytedb/src/application/replication_dispatch.rs +++ b/hyperbytedb/src/application/replication_dispatch.rs @@ -13,10 +13,7 @@ pub async fn dispatch_outbound_replication( batch: OutboundReplicationBatch, ) -> Result<(), HyperbytedbError> { match replication_config.mode { - ReplicationMode::Async => { - replication.replicate_write(batch); - Ok(()) - } + ReplicationMode::Async => replication.replicate_write(batch), ReplicationMode::SyncQuorum => { let peer_count = replication.active_peer_count(node_id).await; let min_acks: SyncQuorumMinAcks = replication_config.sync_quorum.min_acks; diff --git a/hyperbytedb/src/application/retention_service.rs b/hyperbytedb/src/application/retention_service.rs index 3f586ef..17af715 100644 --- a/hyperbytedb/src/application/retention_service.rs +++ b/hyperbytedb/src/application/retention_service.rs @@ -2,7 +2,8 @@ use metrics::counter; use std::sync::Arc; use tokio::sync::watch; -use crate::domain::chdb_naming::quoted_table_name; +use crate::adapters::cluster::raft::HyperbytedbRaft; +use crate::domain::chdb_naming::{quoted_series_table_name, quoted_table_name}; use crate::error::HyperbytedbError; use crate::ports::metadata::MetadataPort; use crate::ports::query::QueryPort; @@ -10,18 +11,45 @@ use crate::ports::query::QueryPort; pub struct RetentionService { metadata: Arc, query: Arc, + raft: Option, + node_id: u64, } impl RetentionService { - pub fn new(metadata: Arc, query: Arc) -> Self { - Self { metadata, query } + pub fn new( + metadata: Arc, + query: Arc, + raft: Option, + node_id: u64, + ) -> Self { + Self { + metadata, + query, + raft, + node_id, + } + } + + fn is_raft_leader(&self) -> bool { + match &self.raft { + Some(raft) => { + let metrics = raft.metrics().borrow().clone(); + metrics.current_leader == Some(self.node_id) + } + None => true, + } } pub async fn run(&self, interval: std::time::Duration, mut shutdown_rx: watch::Receiver) { let mut ticker = tokio::time::interval(interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - tracing::info!("retention service started, interval = {:?}", interval); + tracing::info!( + interval = ?interval, + raft_gated = self.raft.is_some(), + node_id = self.node_id, + "retention service started" + ); loop { tokio::select! { _ = ticker.tick() => { @@ -31,7 +59,7 @@ impl RetentionService { counter!("hyperbytedb_retention_runs_total").increment(1); } Err(e) => { - tracing::error!("retention enforcement error: {}", e); + tracing::error!(error = %e, "retention enforcement error"); } } } @@ -46,6 +74,14 @@ impl RetentionService { } async fn enforce(&self) -> Result<(), HyperbytedbError> { + if !self.is_raft_leader() { + tracing::debug!( + node_id = self.node_id, + "skipping retention tick: not raft leader" + ); + return Ok(()); + } + let now_nanos = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0); let databases = self.metadata.list_databases().await?; @@ -81,35 +117,50 @@ impl RetentionService { let cutoff_nanos = now_nanos - (duration.as_nanos() as i64); - let measurements = match self.metadata.list_measurements(&db.name).await { + let measurements = match self + .metadata + .list_measurements_for_rp(&db.name, &rp.name) + .await + { Ok(m) => m, Err(e) => { - tracing::error!(db = %db.name, error = %e, "retention: failed to list measurements, skipping database"); + tracing::error!( + db = %db.name, + rp = %rp.name, + error = %e, + "retention: failed to list measurements for retention policy, skipping" + ); continue; } }; for meas in &measurements { - let table = quoted_table_name(&db.name, &rp.name, meas); - let sql = format!("ALTER TABLE {table} DELETE WHERE time < {cutoff_nanos}"); - match self.query.execute_sql(&sql).await { - Ok(_) => { - tracing::debug!( - db = %db.name, - rp = %rp.name, - measurement = %meas, - "retention ALTER DELETE issued" - ); - counter!("hyperbytedb_retention_delete_mutations_total").increment(1); - } - Err(e) => { - tracing::warn!( - db = %db.name, - rp = %rp.name, - measurement = %meas, - error = %e, - "retention: ALTER DELETE failed (table may not exist yet)" - ); + let fact_table = quoted_table_name(&db.name, &rp.name, meas); + let series_table = quoted_series_table_name(&db.name, &rp.name, meas); + for table in [fact_table, series_table] { + let sql = format!("ALTER TABLE {table} DELETE WHERE time < {cutoff_nanos}"); + match self.query.execute_sql(&sql).await { + Ok(_) => { + tracing::debug!( + db = %db.name, + rp = %rp.name, + measurement = %meas, + table = %table, + "retention ALTER DELETE issued" + ); + counter!("hyperbytedb_retention_delete_mutations_total") + .increment(1); + } + Err(e) => { + tracing::warn!( + db = %db.name, + rp = %rp.name, + measurement = %meas, + table = %table, + error = %e, + "retention: ALTER DELETE failed (table may not exist yet)" + ); + } } } } diff --git a/hyperbytedb/src/application/runtime/mod.rs b/hyperbytedb/src/application/runtime/mod.rs index cfe547e..633db39 100644 --- a/hyperbytedb/src/application/runtime/mod.rs +++ b/hyperbytedb/src/application/runtime/mod.rs @@ -44,34 +44,6 @@ pub async fn serve(config: HyperbytedbConfig) -> anyhow::Result<()> { }) }; - // Spawn retention enforcement service. Interval and toggle live in - // [retention] in config.toml — operator-driven via the - // HyperbytedbCluster CRD's `spec.retention` field. When disabled, - // we skip spawning entirely so the loop has zero footprint. - let retention_handle = if config.retention.enabled { - let retention_interval = config.retention.interval_duration(); - if retention_interval == RetentionConfig::FALLBACK_INTERVAL - && config.retention.interval.trim() != "60s" - { - tracing::warn!( - configured = %config.retention.interval, - fallback_secs = retention_interval.as_secs(), - "retention.interval is invalid or zero, falling back to default" - ); - } - let retention_service = Arc::new(RetentionService::new( - app_state.metadata.clone(), - app_state.query_port.clone(), - )); - let rx = service_shutdown_rx.clone(); - Some(tokio::spawn(async move { - retention_service.run(retention_interval, rx).await; - })) - } else { - tracing::info!("retention service disabled by config"); - None - }; - // Spawn uptime gauge updater let uptime_handle = { let rx = service_shutdown_rx.clone(); @@ -121,26 +93,31 @@ pub async fn serve(config: HyperbytedbConfig) -> anyhow::Result<()> { }; // Run startup sync and initialize Raft if cluster mode is enabled. - let raft_instance = if let Some(ref c) = cluster { - let meta_port: Arc = app_state.metadata.clone(); - let wal_port: Arc = app_state.wal.clone(); - let sink_port: Arc = app_state.points_sink.clone(); - c.run_startup_sync( - &config.cluster, - &meta_port, - &wal_port, - Some(sink_port), - config.server.max_points_per_request, - ) - .await?; - c.start_raft( - &config.cluster, - app_state.metadata.clone(), - app_state.mv_service.clone(), - ) - .await - } else { - None + let raft_instance = match &cluster { + Some(c) => { + let meta_port: Arc = app_state.metadata.clone(); + let wal_port: Arc = app_state.wal.clone(); + let sink_port: Arc = app_state.points_sink.clone(); + c.run_startup_sync( + &config.cluster, + &meta_port, + &wal_port, + Some(sink_port), + config.server.max_points_per_request, + ) + .await?; + Some( + c.start_raft( + &config.cluster, + app_state.metadata.clone(), + app_state.mv_service.clone(), + app_state.points_sink.clone(), + app_state.wal.clone(), + ) + .await?, + ) + } + None => None, }; // Wire Raft to PeerQueryService for consensus-based schema replication @@ -196,6 +173,34 @@ pub async fn serve(config: HyperbytedbConfig) -> anyhow::Result<()> { }) }; + // Retention enforcement runs on the Raft leader when cluster mode is + // enabled, otherwise the sole local instance (mirrors CQ service). + let retention_handle = if config.retention.enabled { + let retention_interval = config.retention.interval_duration(); + if retention_interval == RetentionConfig::FALLBACK_INTERVAL + && config.retention.interval.trim() != "60s" + { + tracing::warn!( + configured = %config.retention.interval, + fallback_secs = retention_interval.as_secs(), + "retention.interval is invalid or zero, falling back to default" + ); + } + let retention_service = Arc::new(RetentionService::new( + app_state.metadata.clone(), + app_state.query_port.clone(), + app_state.raft.clone(), + config.cluster.node_id, + )); + let rx = service_shutdown_rx.clone(); + Some(tokio::spawn(async move { + retention_service.run(retention_interval, rx).await; + })) + } else { + tracing::info!("retention service disabled by config"); + None + }; + let disk_monitor_handle = if disk_config.enabled { let paths = disk_paths; let cfg = disk_config; @@ -326,13 +331,14 @@ pub async fn serve(config: HyperbytedbConfig) -> anyhow::Result<()> { .await?; } - // ── Phase 2: API is down — drain if cluster mode ──────────────────── - // The HTTP server has fully stopped; no new external requests will arrive. - // Flush the RocksDB WAL to physical media before drain, so all acknowledged - // writes are durable even if the process crashes during drain. - // Run the drain procedure so WAL is flushed and peers acknowledge replication - // before we tear down background services. - tracing::info!("API server stopped, flushing WAL to disk"); + // ── Phase 2: API is down — stop background flush before drain ─────── + tracing::info!("API server stopped, stopping background services before drain"); + let _ = service_shutdown_tx.send(true); + if let Err(e) = flush_handle.await { + tracing::warn!(error = %e, "flush service task join error"); + } + + tracing::info!("flushing WAL to disk"); if let Err(e) = shutdown_wal.flush_wal().await { tracing::error!(error = %e, "WAL flush before drain failed"); } @@ -359,11 +365,9 @@ pub async fn serve(config: HyperbytedbConfig) -> anyhow::Result<()> { } } - // ── Phase 3: Stop background services ─────────────────────────────── - tracing::info!("stopping background services"); - let _ = service_shutdown_tx.send(true); + // ── Phase 3: Stop remaining background services ───────────────────── + tracing::info!("stopping remaining background services"); - flush_handle.await?; cq_handle.await?; if let Some(h) = disk_monitor_handle { h.await?; diff --git a/hyperbytedb/src/application/schema_mutation_apply.rs b/hyperbytedb/src/application/schema_mutation_apply.rs index f758bb5..988be38 100644 --- a/hyperbytedb/src/application/schema_mutation_apply.rs +++ b/hyperbytedb/src/application/schema_mutation_apply.rs @@ -10,13 +10,28 @@ use crate::application::materialized_view_service::MaterializedViewService; use crate::domain::cluster::types::MutationRequest; use crate::error::HyperbytedbError; use crate::ports::metadata::MetadataPort; +use crate::ports::points_sink::PointsSinkPort; +use crate::ports::wal::WalPort; + +/// Dependencies required to apply schema mutations with full local side effects. +pub struct SchemaMutationDeps<'a> { + pub metadata: &'a Arc, + pub mv_service: Option<&'a MaterializedViewService>, + pub points_sink: Option<&'a Arc>, + pub wal: Option<&'a Arc>, +} /// Apply a schema mutation locally, including chDB DDL where required. pub async fn apply_schema_mutation( - metadata: &Arc, - mv_service: Option<&MaterializedViewService>, + deps: SchemaMutationDeps<'_>, mutation: MutationRequest, ) -> Result<(), HyperbytedbError> { + let SchemaMutationDeps { + metadata, + mv_service, + points_sink, + wal, + } = deps; match mutation { MutationRequest::CreateDatabase { name, rp } => { crate::adapters::cluster::raft::state_machine::apply_create_database( @@ -24,7 +39,11 @@ pub async fn apply_schema_mutation( ) .await } - MutationRequest::DropDatabase(name) => metadata.drop_database(&name).await, + MutationRequest::DropDatabase(name) => { + let sink = points_sink.or_else(|| mv_service.map(|mv| mv.points_sink())); + crate::application::database_drop::drop_database(metadata, mv_service, sink, wal, &name) + .await + } MutationRequest::CreateRetentionPolicy { db, rp } => { metadata.create_retention_policy(&db, rp).await } @@ -40,7 +59,13 @@ pub async fn apply_schema_mutation( MutationRequest::SetPassword { username, password_hash, - } => metadata.create_user(&username, &password_hash, false).await, + } => { + let admin = match metadata.get_user(&username).await? { + Some(user) => user.admin, + None => false, + }; + metadata.create_user(&username, &password_hash, admin).await + } MutationRequest::Delete { database, rp, diff --git a/hyperbytedb/src/application/wal_append.rs b/hyperbytedb/src/application/wal_append.rs index 5d8f145..c340010 100644 --- a/hyperbytedb/src/application/wal_append.rs +++ b/hyperbytedb/src/application/wal_append.rs @@ -58,3 +58,79 @@ pub async fn append_points_with_prepared( }; wal.append(entry).await } + +/// Parameters for appending a columnar wire batch to the WAL. +#[cfg(feature = "columnar-ingest")] +pub struct ColumnarWalAppend<'a> { + pub db: &'a str, + pub rp: &'a str, + pub wire: &'a crate::application::columnar_msgpack::ColumnarMsgpackBatch, + pub precision: Option<&'a str>, + pub origin_node_id: u64, + pub max_points_per_request: usize, +} + +/// Append a columnar wire batch, building a prepared WAL slot without expanding +/// to `Vec` on the hot path. Falls back to point expansion when prepared +/// Arrow WAL is unavailable or the columnar prepared build fails. +#[cfg(feature = "columnar-ingest")] +pub async fn append_columnar_with_prepared( + wal: &dyn WalPort, + sink: Option<&Arc>, + req: &ColumnarWalAppend<'_>, +) -> Result { + use crate::application::columnar_msgpack::columnar_batch_to_points; + + crate::application::ingest_metadata::validate_point_count( + req.wire.values.len(), + req.max_points_per_request, + )?; + + let build_start = std::time::Instant::now(); + if wal.arrow_wal_enabled() + && let Some(sink) = sink + { + match sink + .build_prepared_wal_slot_from_columnar( + req.db, + req.rp, + req.origin_node_id, + req.wire, + req.precision, + ) + .await + { + Ok(prepared) => { + histogram!("hyperbytedb_ingest_arrow_build_seconds") + .record(build_start.elapsed().as_secs_f64()); + let entry = WalEntry { + database: req.db.to_string(), + retention_policy: req.rp.to_string(), + points: Vec::new(), + origin_node_id: req.origin_node_id, + }; + return wal + .append_bundle(WalAppendBundle { + entry, + prepared: Some(prepared), + }) + .await; + } + Err(e) => { + tracing::debug!(error = %e, "columnar prepared WAL build failed; falling back"); + } + } + } + + let points = columnar_batch_to_points(req.wire, req.precision)?; + append_points_with_prepared( + wal, + sink, + req.db, + req.rp, + points, + req.origin_node_id, + req.max_points_per_request, + ) + .await +} diff --git a/hyperbytedb/src/bootstrap.rs b/hyperbytedb/src/bootstrap.rs index d788052..5cf9baf 100644 --- a/hyperbytedb/src/bootstrap.rs +++ b/hyperbytedb/src/bootstrap.rs @@ -396,6 +396,7 @@ pub async fn build_services(config: &HyperbytedbConfig) -> anyhow::Result String { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct FlushConfig { pub interval_secs: u64, pub wal_size_threshold_mb: u64, @@ -114,6 +119,7 @@ fn default_wal_batch_delay_us() -> u64 { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ChdbConfig { pub session_data_path: String, /// Number of chDB connections opened to the same `session_data_path`. @@ -130,17 +136,24 @@ pub struct ChdbConfig { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct CardinalityConfig { pub max_tag_values_per_measurement: usize, pub max_measurements_per_database: usize, } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct AuthConfig { pub enabled: bool, + /// InfluxDB v1-style `?u=` / `?p=` credentials on the query string. + /// Disabled by default: passwords can leak into access logs and Referer headers. + #[serde(default)] + pub allow_query_param_credentials: bool, } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ClusterConfig { pub enabled: bool, pub node_id: u64, @@ -199,6 +212,9 @@ pub struct ClusterConfig { pub raft_election_timeout_ms: Option, /// Number of log entries since last snapshot before a new snapshot is taken (default: 1000). pub raft_snapshot_threshold: Option, + /// Default HTTP timeout for outbound Raft RPCs when no per-RPC deadline is supplied (seconds). + #[serde(default = "default_raft_rpc_timeout_secs")] + pub raft_rpc_timeout_secs: u64, /// Per-node replication mode and tuning. When the entire `[cluster.replication]` /// block is omitted, the resolved mode is `async`, exactly preserving today's /// fire-and-forget behavior. @@ -233,6 +249,7 @@ impl ReplicationMode { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ReplicationConfig { #[serde(default)] pub mode: ReplicationMode, @@ -256,6 +273,7 @@ impl Default for ReplicationConfig { } #[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(deny_unknown_fields)] pub struct SyncQuorumConfig { /// Number of peer acks required for `sync_quorum`. The local WAL append /// always happens before fan-out, so self-durability is implicit and the @@ -339,6 +357,10 @@ fn default_replication_truncate_stale_peer_multiplier() -> u64 { 2 } +fn default_raft_rpc_timeout_secs() -> u64 { + 10 +} + impl ClusterConfig { pub fn peer_list(&self) -> Vec { if self.peers.is_empty() { @@ -364,12 +386,14 @@ impl ClusterConfig { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct LoggingConfig { pub level: String, pub format: String, } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct StatementSummaryConfig { pub enabled: bool, pub max_entries: usize, @@ -383,6 +407,7 @@ fn default_true() -> bool { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct DiskConfig { #[serde(default = "default_true")] pub enabled: bool, @@ -418,6 +443,7 @@ impl Default for DiskConfig { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct HintedHandoffConfig { pub enabled: bool, /// Maximum queued hints per unreachable peer before oldest are dropped. @@ -434,6 +460,7 @@ pub struct HintedHandoffConfig { /// only how often that scan runs; the per-policy `duration` is metadata /// stored alongside each retention policy (`CREATE/ALTER RETENTION POLICY`). #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct RetentionConfig { /// When `false`, the retention enforcement loop is not spawned and /// expired data stays in chDB until removed manually or via DDL. @@ -538,7 +565,10 @@ impl HyperbytedbConfig { pool_size: default_chdb_pool_size(), schema_cache_max_entries: default_schema_cache_max_entries(), }, - auth: AuthConfig { enabled: false }, + auth: AuthConfig { + enabled: false, + allow_query_param_credentials: false, + }, cardinality: CardinalityConfig { max_tag_values_per_measurement: 100_000, max_measurements_per_database: 10_000, @@ -566,6 +596,7 @@ impl HyperbytedbConfig { raft_heartbeat_interval_ms: None, raft_election_timeout_ms: None, raft_snapshot_threshold: None, + raft_rpc_timeout_secs: default_raft_rpc_timeout_secs(), replication: ReplicationConfig::default(), }, logging: LoggingConfig { @@ -594,7 +625,29 @@ impl HyperbytedbConfig { #[cfg(test)] mod replication_config_tests { - use super::{ReplicationConfig, ReplicationMode, SyncQuorumMinAcks, SyncQuorumMinAcksKeyword}; + use super::{ + ReplicationConfig, ReplicationMode, ServerConfig, SyncQuorumMinAcks, + SyncQuorumMinAcksKeyword, + }; + + #[test] + fn rejects_unknown_server_config_field() { + let err = serde_json::from_str::( + r#"{ + "bind_address": "0.0.0.0", + "port": 8086, + "max_body_size_bytes": 1, + "request_timeout_secs": 30, + "query_timeout_secs": 30, + "max_concurrent_queries": 0, + "tls_enabled": false, + "tls_cert_path": "", + "tls_key_path": "", + "timout_secs": 30 + }"#, + ); + assert!(err.is_err(), "unknown section keys should be rejected"); + } #[test] fn replication_defaults_to_async_when_block_missing() { @@ -746,6 +799,7 @@ mod replicate_body_limit_tests { raft_heartbeat_interval_ms: None, raft_election_timeout_ms: None, raft_snapshot_threshold: None, + raft_rpc_timeout_secs: super::default_raft_rpc_timeout_secs(), replication: super::ReplicationConfig::default(), } } diff --git a/hyperbytedb/src/domain/arrow_coalesce.rs b/hyperbytedb/src/domain/arrow_coalesce.rs index 71244cd..0ad0ef1 100644 --- a/hyperbytedb/src/domain/arrow_coalesce.rs +++ b/hyperbytedb/src/domain/arrow_coalesce.rs @@ -27,7 +27,7 @@ pub fn coalesce_prepared_batches( let schema = template.batch.schema(); let refs: Vec<&RecordBatch> = batches.iter().map(|b| b.batch.as_ref()).collect(); let merged = concat_batches(&schema, refs) - .map_err(|e| HyperbytedbError::Internal(format!("concat prepared batches: {e}")))?; + .map_err(|e| HyperbytedbError::Internal(format!("concat prepared batches: {e}").into()))?; let min_time = batches.iter().map(|b| b.min_time).min().unwrap_or(0); let max_time = batches.iter().map(|b| b.max_time).max().unwrap_or(0); @@ -65,8 +65,9 @@ fn merge_series_batches( _ => { let schema = batches[0].schema(); let refs: Vec<&RecordBatch> = batches.iter().map(|b| b.as_ref()).collect(); - let merged = concat_batches(&schema, refs) - .map_err(|e| HyperbytedbError::Internal(format!("concat series batches: {e}")))?; + let merged = concat_batches(&schema, refs).map_err(|e| { + HyperbytedbError::Internal(format!("concat series batches: {e}").into()) + })?; Ok(Some(Arc::new(normalize_dictionary_columns(merged)?))) } } @@ -100,10 +101,10 @@ fn normalize_dictionary_columns(batch: RecordBatch) -> Result { let target = col.data_type().clone(); let decoded = cast(col, value_type).map_err(|e| { - HyperbytedbError::Internal(format!("decode dictionary column: {e}")) + HyperbytedbError::Internal(format!("decode dictionary column: {e}").into()) })?; let reencoded = cast(&decoded, &target).map_err(|e| { - HyperbytedbError::Internal(format!("re-encode dictionary column: {e}")) + HyperbytedbError::Internal(format!("re-encode dictionary column: {e}").into()) })?; columns.push(reencoded); } @@ -112,7 +113,7 @@ fn normalize_dictionary_columns(batch: RecordBatch) -> Result Self { + Self(sanitise_ident(input)) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Backtick-quoted form suitable for splicing into SQL. + #[must_use] + pub fn quoted(&self) -> QuotedTableName { + QuotedTableName::new_quoted(quote_backticks(self.as_str())) + } +} + +impl fmt::Display for SanitizedIdent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// Backtick-quoted table or object name safe to splice into generated SQL. +/// +/// Construct only via [`quoted_table_name`], [`quoted_series_table_name`], or +/// related helpers in this module — do not build from raw user input elsewhere. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct QuotedTableName(String); + +impl QuotedTableName { + pub(crate) fn new_quoted(s: String) -> Self { + Self(s) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for QuotedTableName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// Reject control characters in identifiers before quoting. +fn reject_control_chars(ident: &str) -> Result<(), HyperbytedbError> { + if ident.chars().any(char::is_control) { + return Err(HyperbytedbError::QueryParse(format!( + "identifier contains control characters: {ident:?}" + ))); + } + Ok(()) +} /// Replace every byte that isn't `[A-Za-z0-9_]` with `_` and prefix /// `_` if the result starts with a digit. Empty input becomes `_`. @@ -39,13 +104,13 @@ fn sanitise_ident(input: &str) -> String { /// Build the unquoted table identifier for a `(db, rp, measurement)` /// tuple, using `db_rp_measurement` after sanitisation. #[must_use] -pub fn unquoted_table_name(db: &str, rp: &str, measurement: &str) -> String { - format!( +pub fn unquoted_table_name(db: &str, rp: &str, measurement: &str) -> SanitizedIdent { + SanitizedIdent(format!( "{}_{}_{}", sanitise_ident(db), sanitise_ident(rp), sanitise_ident(measurement) - ) + )) } /// Quote an identifier with backticks, escaping embedded backticks. @@ -57,52 +122,64 @@ pub fn quote_backticks(ident: &str) -> String { format!("`{escaped}`") } +/// Like [`quote_backticks`], but rejects control characters first. +pub fn quote_backticks_validated(ident: &str) -> Result { + reject_control_chars(ident)?; + Ok(quote_backticks(ident)) +} + /// Backtick-quoted, sanitised table name suitable for splicing into /// `CREATE TABLE`, `INSERT INTO`, `DROP TABLE`, and `FROM` clauses. #[must_use] -pub fn quoted_table_name(db: &str, rp: &str, measurement: &str) -> String { - quote_backticks(&unquoted_table_name(db, rp, measurement)) +pub fn quoted_table_name(db: &str, rp: &str, measurement: &str) -> QuotedTableName { + unquoted_table_name(db, rp, measurement).quoted() } /// Unquoted name of the per-measurement series (tag dimension) table: /// `___series`. The `_series` suffix is appended after /// sanitisation (it is already valid `[A-Za-z0-9_]`). #[must_use] -pub fn unquoted_series_table_name(db: &str, rp: &str, measurement: &str) -> String { - format!("{}_series", unquoted_table_name(db, rp, measurement)) +pub fn unquoted_series_table_name(db: &str, rp: &str, measurement: &str) -> SanitizedIdent { + SanitizedIdent(format!( + "{}_series", + unquoted_table_name(db, rp, measurement).0 + )) } /// Backtick-quoted series (tag dimension) table name. See /// [`unquoted_series_table_name`]. #[must_use] -pub fn quoted_series_table_name(db: &str, rp: &str, measurement: &str) -> String { - quote_backticks(&unquoted_series_table_name(db, rp, measurement)) +pub fn quoted_series_table_name(db: &str, rp: &str, measurement: &str) -> QuotedTableName { + unquoted_series_table_name(db, rp, measurement).quoted() } /// Unquoted ClickHouse object name for a fact-table materialized view: /// `___mv`. #[must_use] -pub fn unquoted_fact_mv_name(db: &str, rp: &str, mv_name: &str) -> String { - format!("{}_mv", unquoted_table_name(db, rp, mv_name)) +pub fn unquoted_fact_mv_name(db: &str, rp: &str, mv_name: &str) -> SanitizedIdent { + SanitizedIdent(format!("{}_mv", unquoted_table_name(db, rp, mv_name).0)) } /// Backtick-quoted fact MV object name. #[must_use] -pub fn quoted_fact_mv_name(db: &str, rp: &str, mv_name: &str) -> String { - quote_backticks(&unquoted_fact_mv_name(db, rp, mv_name)) +pub fn quoted_fact_mv_name(db: &str, rp: &str, mv_name: &str) -> QuotedTableName { + unquoted_fact_mv_name(db, rp, mv_name).quoted() } /// Unquoted ClickHouse object name for a series-dimension MV: /// `___series_mv`. #[must_use] -pub fn unquoted_series_mv_name(db: &str, rp: &str, mv_name: &str) -> String { - format!("{}_series_mv", unquoted_table_name(db, rp, mv_name)) +pub fn unquoted_series_mv_name(db: &str, rp: &str, mv_name: &str) -> SanitizedIdent { + SanitizedIdent(format!( + "{}_series_mv", + unquoted_table_name(db, rp, mv_name).0 + )) } /// Backtick-quoted series MV object name. #[must_use] -pub fn quoted_series_mv_name(db: &str, rp: &str, mv_name: &str) -> String { - quote_backticks(&unquoted_series_mv_name(db, rp, mv_name)) +pub fn quoted_series_mv_name(db: &str, rp: &str, mv_name: &str) -> QuotedTableName { + unquoted_series_mv_name(db, rp, mv_name).quoted() } /// Resolve the physical column name for a tag key, taking field-name @@ -145,11 +222,11 @@ mod tests { #[test] fn quoted_table_name_matches_db_rp_measurement() { assert_eq!( - quoted_table_name("mydb", "autogen", "cpu"), + quoted_table_name("mydb", "autogen", "cpu").as_str(), "`mydb_autogen_cpu`" ); assert_eq!( - quoted_table_name("my-db", "autogen", "cpu.load"), + quoted_table_name("my-db", "autogen", "cpu.load").as_str(), "`my_db_autogen_cpu_load`" ); } @@ -159,14 +236,20 @@ mod tests { assert_eq!(quote_backticks("a`b"), "`a``b`"); } + #[test] + fn quote_backticks_validated_rejects_control_chars() { + assert!(quote_backticks_validated("a\nb").is_err()); + assert!(quote_backticks_validated("ok_name").is_ok()); + } + #[test] fn series_table_name_appends_suffix() { assert_eq!( - quoted_series_table_name("mydb", "autogen", "cpu"), + quoted_series_table_name("mydb", "autogen", "cpu").as_str(), "`mydb_autogen_cpu_series`" ); assert_eq!( - quoted_series_table_name("my-db", "autogen", "cpu.load"), + quoted_series_table_name("my-db", "autogen", "cpu.load").as_str(), "`my_db_autogen_cpu_load_series`" ); } diff --git a/hyperbytedb/src/domain/cluster/replication_wire.rs b/hyperbytedb/src/domain/cluster/replication_wire.rs index 507806c..0bfd302 100644 --- a/hyperbytedb/src/domain/cluster/replication_wire.rs +++ b/hyperbytedb/src/domain/cluster/replication_wire.rs @@ -34,7 +34,7 @@ impl ReplicationHintPayload { let rp = self.retention_policy.as_bytes(); if db.len() > u32::MAX as usize || rp.len() > u32::MAX as usize { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: db/rp name too long".into(), + crate::error::ChainedError::new("replication hint: db/rp name too long"), )); } let prec_bytes = self @@ -44,12 +44,12 @@ impl ReplicationHintPayload { .unwrap_or_default(); if prec_bytes.len() > u16::MAX as usize { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: precision too long".into(), + crate::error::ChainedError::new("replication hint: precision too long"), )); } if self.line_body.len() > u64::MAX as usize { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: body too large".into(), + crate::error::ChainedError::new("replication hint: body too large"), )); } @@ -77,14 +77,16 @@ impl ReplicationHintPayload { pub fn decode_hint_value(data: &[u8]) -> Result { if data.len() < 4 || &data[0..4] != HH_MAGIC { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: bad magic or unsupported legacy format".into(), + crate::error::ChainedError::new( + "replication hint: bad magic or unsupported legacy format", + ), )); } let mut i = 4; let read_u32 = |buf: &[u8], i: &mut usize| -> Result { if *i + 4 > buf.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated".into(), + crate::error::ChainedError::new("replication hint: truncated"), )); } let mut arr = [0u8; 4]; @@ -95,7 +97,7 @@ impl ReplicationHintPayload { let read_u16 = |buf: &[u8], i: &mut usize| -> Result { if *i + 2 > buf.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated".into(), + crate::error::ChainedError::new("replication hint: truncated"), )); } let mut arr = [0u8; 2]; @@ -106,7 +108,7 @@ impl ReplicationHintPayload { let read_u64 = |buf: &[u8], i: &mut usize| -> Result { if *i + 8 > buf.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated".into(), + crate::error::ChainedError::new("replication hint: truncated"), )); } let mut arr = [0u8; 8]; @@ -118,28 +120,32 @@ impl ReplicationHintPayload { let dlen = read_u32(data, &mut i)? as usize; if i + dlen > data.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated db".into(), + crate::error::ChainedError::new("replication hint: truncated db"), )); } let database = std::str::from_utf8(&data[i..i + dlen]) - .map_err(|e| crate::error::HyperbytedbError::Internal(e.to_string()))? + .map_err(|e| { + crate::error::HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) + })? .to_string(); i += dlen; let rlen = read_u32(data, &mut i)? as usize; if i + rlen > data.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated rp".into(), + crate::error::ChainedError::new("replication hint: truncated rp"), )); } let retention_policy = std::str::from_utf8(&data[i..i + rlen]) - .map_err(|e| crate::error::HyperbytedbError::Internal(e.to_string()))? + .map_err(|e| { + crate::error::HyperbytedbError::Internal(crate::error::ChainedError::from_error(e)) + })? .to_string(); i += rlen; if i >= data.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: missing flags".into(), + crate::error::ChainedError::new("replication hint: missing flags"), )); } let prec_flag = data[i]; @@ -150,18 +156,22 @@ impl ReplicationHintPayload { let plen = read_u16(data, &mut i)? as usize; if i + plen > data.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated precision".into(), + crate::error::ChainedError::new("replication hint: truncated precision"), )); } let p = std::str::from_utf8(&data[i..i + plen]) - .map_err(|e| crate::error::HyperbytedbError::Internal(e.to_string()))? + .map_err(|e| { + crate::error::HyperbytedbError::Internal( + crate::error::ChainedError::from_error(e), + ) + })? .to_string(); i += plen; Some(p) } _ => { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: bad precision flag".into(), + crate::error::ChainedError::new("replication hint: bad precision flag"), )); } }; @@ -169,7 +179,7 @@ impl ReplicationHintPayload { let blen = read_u64(data, &mut i)? as usize; if i + blen > data.len() { return Err(crate::error::HyperbytedbError::Internal( - "replication hint: truncated body".into(), + crate::error::ChainedError::new("replication hint: truncated body"), )); } let line_body = data[i..i + blen].to_vec(); diff --git a/hyperbytedb/src/domain/cluster/types.rs b/hyperbytedb/src/domain/cluster/types.rs index 28921b0..2410404 100644 --- a/hyperbytedb/src/domain/cluster/types.rs +++ b/hyperbytedb/src/domain/cluster/types.rs @@ -1,7 +1,8 @@ use serde::{Deserialize, Serialize}; +use crate::domain::continuous_query::ContinuousQueryDef; use crate::domain::database::RetentionPolicy; -use crate::ports::metadata::{ContinuousQueryDef, MaterializedViewDef}; +use crate::domain::materialized_view::MaterializedViewDef; use crate::timeseriesql::ast::RetentionPolicyChange; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/hyperbytedb/src/domain/column_mapping.rs b/hyperbytedb/src/domain/column_mapping.rs index b274371..8a87b3a 100644 --- a/hyperbytedb/src/domain/column_mapping.rs +++ b/hyperbytedb/src/domain/column_mapping.rs @@ -86,6 +86,14 @@ impl ColumnMapping { let fields: HashSet<&str> = self.field_names.iter().map(|s| s.as_str()).collect(); tag_column_name(tag_key, &fields) } + + /// Physical (sanitized) series-table column name for a logical tag key. + #[must_use] + pub fn physical_tag_column_name(&self, tag_key: &str) -> String { + let fields: HashSet<&str> = self.field_names.iter().map(|s| s.as_str()).collect(); + crate::domain::chdb_naming::tag_column_name(tag_key, &fields) + } + /// SELECT / aggregate: prefer field column when tag and field share a name. #[must_use] pub fn physical_select_identifier(&self, name: &str) -> String { diff --git a/hyperbytedb/src/domain/cq_schedule.rs b/hyperbytedb/src/domain/cq_schedule.rs index 8e7edf9..6a4270d 100644 --- a/hyperbytedb/src/domain/cq_schedule.rs +++ b/hyperbytedb/src/domain/cq_schedule.rs @@ -2,7 +2,7 @@ //! //! See: https://docs.influxdata.com/influxdb/v1/query_language/continuous_queries/ -use chrono::{DateTime, TimeZone, Utc}; +use chrono::{DateTime, LocalResult, TimeZone, Utc}; use crate::domain::continuous_query::ContinuousQueryDef; use crate::error::HyperbytedbError; @@ -125,7 +125,10 @@ fn last_run_time(cq: &ContinuousQueryDef, fallback: DateTime) -> DateTime, cq: &ContinuousQueryDef) -> CqWindow { +pub fn coverage_window( + now: DateTime, + cq: &ContinuousQueryDef, +) -> Result { let meta = schedule_from_def(cq); coverage_window_from_meta(now, &meta) } @@ -142,7 +145,10 @@ fn schedule_from_def(cq: &ContinuousQueryDef) -> ScheduleMeta { } } -fn coverage_window_from_meta(now: DateTime, meta: &ScheduleMeta) -> CqWindow { +fn coverage_window_from_meta( + now: DateTime, + meta: &ScheduleMeta, +) -> Result { let offset = meta.group_by_offset_secs; let group_secs = meta.group_by_interval_secs as i64; let exec_secs = meta.execution_interval_secs as i64; @@ -176,13 +182,22 @@ fn coverage_window_from_meta(now: DateTime, meta: &ScheduleMeta) -> CqWindo window_from_unix(start, end) } -fn window_from_unix(start_secs: i64, end_secs: i64) -> CqWindow { - CqWindow { - start: Utc.timestamp_opt(start_secs, 0).unwrap(), - end: Utc.timestamp_opt(end_secs, 0).unwrap(), +fn unix_timestamp(secs: i64) -> Result, HyperbytedbError> { + match Utc.timestamp_opt(secs, 0) { + LocalResult::Single(dt) => Ok(dt), + LocalResult::None | LocalResult::Ambiguous(_, _) => Err(HyperbytedbError::Internal( + format!("invalid unix timestamp for CQ coverage window: {secs}s").into(), + )), } } +fn window_from_unix(start_secs: i64, end_secs: i64) -> Result { + Ok(CqWindow { + start: unix_timestamp(start_secs)?, + end: unix_timestamp(end_secs)?, + }) +} + fn group_by_interval_secs( query: &crate::timeseriesql::ast::SelectStatement, ) -> Result { @@ -302,7 +317,7 @@ mod tests { assert_eq!(def.execution_interval_secs, 3600); assert_eq!(def.coverage_interval_secs, 3600); - let w = coverage_window(ts(8, 0), &def); + let w = coverage_window(ts(8, 0), &def).unwrap(); assert_eq!(w.start, ts(7, 0)); assert_eq!(w.end, ts(8, 0)); } @@ -315,7 +330,7 @@ mod tests { let def = def_from_cq(&cq); assert_eq!(def.group_by_offset_secs, 900); - let w = coverage_window(ts(8, 15), &def); + let w = coverage_window(ts(8, 15), &def).unwrap(); assert_eq!(w.start, ts(7, 15)); assert_eq!(w.end, ts(8, 15)); } @@ -329,7 +344,7 @@ mod tests { let def = def_from_cq(&cq); assert_eq!(def.execution_interval_secs, 1800); - let w = coverage_window(ts(8, 30), &def); + let w = coverage_window(ts(8, 30), &def).unwrap(); assert_eq!(w.start, ts(8, 0)); assert_eq!(w.end, ts(9, 0)); } @@ -344,7 +359,7 @@ mod tests { assert_eq!(def.execution_interval_secs, 1800); assert_eq!(def.resample_for_secs, Some(3600)); - let w = coverage_window(ts(8, 0), &def); + let w = coverage_window(ts(8, 0), &def).unwrap(); assert_eq!(w.start, ts(7, 0)); assert_eq!(w.end, ts(8, 0)); } @@ -357,7 +372,7 @@ mod tests { ); let def = def_from_cq(&cq); - let w = coverage_window(ts(9, 0), &def); + let w = coverage_window(ts(9, 0), &def).unwrap(); assert_eq!(w.start, ts(7, 30)); assert_eq!(w.end, ts(9, 0)); } @@ -374,7 +389,7 @@ mod tests { let mut def = def_from_cq(&cq); meta.apply_to(&mut def); - let w = coverage_window(ts(8, 0), &def); + let w = coverage_window(ts(8, 0), &def).unwrap(); assert_eq!(w.end.timestamp() - w.start.timestamp(), 600); } diff --git a/hyperbytedb/src/domain/prepared_wal.rs b/hyperbytedb/src/domain/prepared_wal.rs index bf6b7ef..fb19e9f 100644 --- a/hyperbytedb/src/domain/prepared_wal.rs +++ b/hyperbytedb/src/domain/prepared_wal.rs @@ -53,15 +53,15 @@ pub fn patch_ingest_seq( .iter() .position(|f| f.name() == "ingest_seq") .ok_or_else(|| { - crate::error::HyperbytedbError::Internal( - "prepared batch missing ingest_seq column".into(), - ) + crate::error::HyperbytedbError::Internal(crate::error::ChainedError::new( + "prepared batch missing ingest_seq column", + )) })?; let seq_col = batch.column(ingest_idx); if seq_col.data_type() != &DataType::UInt64 { return Err(crate::error::HyperbytedbError::Internal( - "ingest_seq column has unexpected type".into(), + crate::error::ChainedError::new("ingest_seq column has unexpected type"), )); } @@ -79,8 +79,9 @@ pub fn patch_ingest_seq( let mut columns: Vec = batch.columns().to_vec(); columns[ingest_idx] = Arc::new(UInt64Array::from(seqs)); - let patched = RecordBatch::try_new(schema, columns) - .map_err(|e| crate::error::HyperbytedbError::Internal(format!("patch ingest_seq: {e}")))?; + let patched = RecordBatch::try_new(schema, columns).map_err(|e| { + crate::error::HyperbytedbError::Internal(format!("patch ingest_seq: {e}").into()) + })?; Ok(Arc::new(patched)) } diff --git a/hyperbytedb/src/error.rs b/hyperbytedb/src/error.rs index 34e699e..91564b7 100644 --- a/hyperbytedb/src/error.rs +++ b/hyperbytedb/src/error.rs @@ -1,5 +1,64 @@ +use std::fmt; + use thiserror::Error; +/// Error payload that preserves an optional `source()` chain for ops debugging. +#[derive(Debug)] +pub struct ChainedError { + message: String, + source: Option>, +} + +impl ChainedError { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + source: None, + } + } + + pub fn from_error(source: E) -> Self { + Self { + message: source.to_string(), + source: Some(Box::new(source)), + } + } + + pub fn with_context( + context: impl Into, + source: E, + ) -> Self { + Self { + message: format!("{}: {source}", context.into()), + source: Some(Box::new(source)), + } + } +} + +impl fmt::Display for ChainedError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ChainedError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source.as_ref().map(|e| e.as_ref() as _) + } +} + +impl From for ChainedError { + fn from(message: String) -> Self { + Self::new(message) + } +} + +impl From<&str> for ChainedError { + fn from(message: &str) -> Self { + Self::new(message) + } +} + #[derive(Error, Debug)] pub enum HyperbytedbError { #[error("database not found: \"{0}\"")] @@ -46,16 +105,16 @@ pub enum HyperbytedbError { MissingParameter(String), #[error("WAL error: {0}")] - Wal(String), + Wal(#[source] ChainedError), #[error("storage error: {0}")] - Storage(String), + Storage(#[source] ChainedError), #[error("chdb error: {0}")] - Chdb(String), + Chdb(#[source] ChainedError), #[error("metadata error: {0}")] - Metadata(String), + Metadata(#[source] ChainedError), #[error( "cardinality limit exceeded: measurement \"{measurement}\" tag \"{tag_key}\" has {current} values (limit: {limit})" @@ -79,7 +138,9 @@ pub enum HyperbytedbError { #[error("WAL backpressure: write queue full for {timeout_ms}ms")] WalBackpressure { timeout_ms: u64 }, - #[error("query timeout exceeded")] + #[error( + "query timeout exceeded; earlier statements in a multi-statement batch may already be committed" + )] QueryTimeout, #[error("cluster unavailable: {0}")] @@ -104,7 +165,7 @@ pub enum HyperbytedbError { }, #[error("internal error: {0}")] - Internal(String), + Internal(#[source] ChainedError), } // RocksDB errors are mapped per subsystem (e.g. `Wal` in [`crate::adapters::wal::rocksdb_wal`], @@ -112,18 +173,40 @@ pub enum HyperbytedbError { impl From for HyperbytedbError { fn from(e: std::fmt::Error) -> Self { - HyperbytedbError::Internal(e.to_string()) + HyperbytedbError::Internal(ChainedError::from_error(e)) } } impl From for HyperbytedbError { fn from(e: std::io::Error) -> Self { - HyperbytedbError::Storage(e.to_string()) + HyperbytedbError::Storage(ChainedError::from_error(e)) } } impl From for HyperbytedbError { fn from(e: bincode::Error) -> Self { - HyperbytedbError::Internal(e.to_string()) + HyperbytedbError::Internal(ChainedError::from_error(e)) + } +} + +#[cfg(test)] +mod tests { + use super::{ChainedError, HyperbytedbError}; + use std::error::Error; + + #[test] + fn chained_error_preserves_io_source() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "missing file"); + let chained = ChainedError::from_error(io_err); + let wal = HyperbytedbError::Wal(chained); + assert!(wal.source().is_some()); + assert!(wal.source().unwrap().source().is_some()); + } + + #[test] + fn message_only_chained_error_has_no_source() { + let err = HyperbytedbError::Wal(ChainedError::new("wal column family not found")); + assert!(err.source().is_some()); + assert!(err.source().unwrap().source().is_none()); } } diff --git a/hyperbytedb/src/ports/points_sink.rs b/hyperbytedb/src/ports/points_sink.rs index 5ff1613..5e324cc 100644 --- a/hyperbytedb/src/ports/points_sink.rs +++ b/hyperbytedb/src/ports/points_sink.rs @@ -53,6 +53,21 @@ pub trait PointsSinkPort: Send + Sync { )) } + /// Build a prepared WAL slot directly from a columnar wire batch. + #[cfg(feature = "columnar-ingest")] + async fn build_prepared_wal_slot_from_columnar( + &self, + _db: &str, + _rp: &str, + _origin_node_id: u64, + _wire: &crate::application::columnar_msgpack::ColumnarMsgpackBatch, + _precision: Option<&str>, + ) -> Result { + Err(HyperbytedbError::Internal( + "columnar prepared WAL build not supported".into(), + )) + } + async fn drop_measurement( &self, db: &str, diff --git a/hyperbytedb/src/ports/replication.rs b/hyperbytedb/src/ports/replication.rs index c78743c..1151cca 100644 --- a/hyperbytedb/src/ports/replication.rs +++ b/hyperbytedb/src/ports/replication.rs @@ -18,7 +18,10 @@ pub struct OutboundReplicationBatch { /// Outbound write/mutation replication to cluster peers. #[async_trait] pub trait ReplicationPort: Send + Sync { - fn replicate_write(self: Arc, batch: OutboundReplicationBatch); + fn replicate_write( + self: Arc, + batch: OutboundReplicationBatch, + ) -> Result<(), HyperbytedbError>; async fn replicate_write_sync( self: Arc, @@ -29,5 +32,10 @@ pub trait ReplicationPort: Send + Sync { fn replicate_mutation(self: Arc, req: MutationRequest); + async fn replicate_mutation_sync( + self: Arc, + req: MutationRequest, + ) -> Result<(), HyperbytedbError>; + async fn active_peer_count(&self, self_node_id: u64) -> usize; } diff --git a/hyperbytedb/src/ports/wal.rs b/hyperbytedb/src/ports/wal.rs index e19a2d0..9c2decf 100644 --- a/hyperbytedb/src/ports/wal.rs +++ b/hyperbytedb/src/ports/wal.rs @@ -74,6 +74,12 @@ pub trait WalPort: Send + Sync { async fn truncate_before(&self, sequence: u64) -> Result<(), HyperbytedbError>; async fn last_sequence(&self) -> Result; + /// Remove all durable and in-memory WAL entries for `database`. + async fn purge_database(&self, database: &str) -> Result<(), HyperbytedbError> { + let _ = database; + Ok(()) + } + /// Flush (fsync) the RocksDB WAL to physical media. /// Default no-op for non-RocksDB implementations. async fn flush_wal(&self) -> Result<(), HyperbytedbError> { diff --git a/hyperbytedb/src/timeseriesql/ast.rs b/hyperbytedb/src/timeseriesql/ast.rs index fedcf81..b66da8e 100644 --- a/hyperbytedb/src/timeseriesql/ast.rs +++ b/hyperbytedb/src/timeseriesql/ast.rs @@ -121,6 +121,7 @@ pub struct RetentionPolicyChange { #[derive(Debug, Clone)] pub struct ShowMeasurementsStatement { pub database: Option, + pub measurement_filter: Option, pub condition: Option, pub limit: Option, pub offset: Option, diff --git a/hyperbytedb/src/timeseriesql/ddl_parser.rs b/hyperbytedb/src/timeseriesql/ddl_parser.rs index 596fe29..21c98d3 100644 --- a/hyperbytedb/src/timeseriesql/ddl_parser.rs +++ b/hyperbytedb/src/timeseriesql/ddl_parser.rs @@ -72,19 +72,12 @@ fn parse_show(cur: &mut TokenCursor<'_>) -> Result cur.bump(); let mut stmt = ShowMeasurementsStatement { database: parse_optional_on_db(cur)?, + measurement_filter: None, condition: None, limit: None, offset: None, }; - // Previously this clause was silently dropped, listing every - // measurement unfiltered. Reject loudly until implemented. - if cur.match_keyword("WITH") { - return Err(HyperbytedbError::QueryParse( - "SHOW MEASUREMENTS WITH MEASUREMENT is not supported; \ - use a WHERE clause instead" - .to_string(), - )); - } + stmt.measurement_filter = parse_with_measurement(cur)?; parse_show_tail( cur, &mut stmt.database, @@ -851,32 +844,141 @@ fn parse_measurement_name(cur: &mut TokenCursor<'_>) -> Result, +) -> Result, HyperbytedbError> { + if cur.match_keyword("WITH") { + cur.expect_keyword("MEASUREMENT")?; + Ok(Some(parse_key_selector(cur)?)) + } else { + Ok(None) + } +} + fn parse_with_key(cur: &mut TokenCursor<'_>) -> Result { if cur.match_keyword("WITH") { cur.expect_keyword("KEY")?; - if matches!( - cur.peek(), - Some(Token { - kind: TokenKind::Eq, - .. - }) - ) { - cur.bump(); - } - if matches!( - cur.peek(), - Some(Token { - kind: TokenKind::Star, - .. - }) - ) { + parse_key_selector(cur) + } else { + Ok(TagKeySelector::All) + } +} + +fn parse_key_selector(cur: &mut TokenCursor<'_>) -> Result { + if matches!( + cur.peek(), + Some(Token { + kind: TokenKind::Eq, + .. + }) + ) { + cur.bump(); + } + + if matches!( + cur.peek(), + Some(Token { + kind: TokenKind::Star, + .. + }) + ) { + cur.bump(); + return Ok(TagKeySelector::All); + } + + if matches!( + cur.peek(), + Some(Token { + kind: TokenKind::Ne, + .. + }) + ) { + cur.bump(); + return Ok(TagKeySelector::Neq(cur.take_ident()?)); + } + + if matches!( + cur.peek(), + Some(Token { + kind: TokenKind::MatchRegex, + .. + }) + ) { + cur.bump(); + return Ok(TagKeySelector::Regex(take_regex_token(cur)?)); + } + + if cur.match_keyword("IN") { + return Ok(TagKeySelector::In(parse_ident_list(cur)?)); + } + + Ok(TagKeySelector::Eq(cur.take_ident()?)) +} + +fn take_regex_token(cur: &mut TokenCursor<'_>) -> Result { + match cur.bump() { + Some(Token { + kind: TokenKind::Regex(p), + .. + }) => Ok(p), + Some(t) => Err(HyperbytedbError::QueryParse(format!( + "expected regex literal, found {:?}", + t.kind + ))), + None => Err(HyperbytedbError::QueryParse( + "expected regex literal, found EOF".to_string(), + )), + } +} + +fn parse_ident_list(cur: &mut TokenCursor<'_>) -> Result, HyperbytedbError> { + match cur.peek() { + Some(Token { + kind: TokenKind::LParen, + .. + }) => { cur.bump(); - return Ok(TagKeySelector::All); + let mut keys = Vec::new(); + loop { + if matches!( + cur.peek(), + Some(Token { + kind: TokenKind::RParen, + .. + }) + ) { + cur.bump(); + break; + } + if !keys.is_empty() { + match cur.peek() { + Some(Token { + kind: TokenKind::Comma, + .. + }) => { + cur.bump(); + } + Some(t) => { + return Err(HyperbytedbError::QueryParse(format!( + "expected comma in IN list, found {:?}", + t.kind + ))); + } + None => { + return Err(HyperbytedbError::QueryParse( + "unclosed IN list".to_string(), + )); + } + } + } + keys.push(cur.take_ident()?); + } + Ok(keys) } - let key = cur.take_ident()?; - return Ok(TagKeySelector::Eq(key)); + _ => Err(HyperbytedbError::QueryParse( + "expected '(' after IN".to_string(), + )), } - Ok(TagKeySelector::All) } fn parse_duration_token(cur: &mut TokenCursor<'_>) -> Result { @@ -1220,14 +1322,62 @@ mod tests { } } + #[test] + fn show_measurements_with_measurement_filter() { + let stmt = parse_ddl_statement("SHOW MEASUREMENTS WITH MEASUREMENT =~ /cpu/").unwrap(); + match stmt { + Statement::ShowMeasurements(s) => { + assert!(matches!( + s.measurement_filter, + Some(TagKeySelector::Regex(ref p)) if p == "cpu" + )); + } + other => panic!("expected ShowMeasurements, got {other:?}"), + } + let stmt = parse_ddl_statement(r#"SHOW MEASUREMENTS WITH MEASUREMENT = "cpu""#).unwrap(); + match stmt { + Statement::ShowMeasurements(s) => { + assert!(matches!( + s.measurement_filter, + Some(TagKeySelector::Eq(ref n)) if n == "cpu" + )); + } + other => panic!("expected ShowMeasurements, got {other:?}"), + } + } + + #[test] + fn show_tag_values_with_key_operators() { + let stmt = parse_ddl_statement(r#"SHOW TAG VALUES FROM cpu WITH KEY != "host""#).unwrap(); + match stmt { + Statement::ShowTagValues(s) => { + assert!(matches!(s.tag_key, TagKeySelector::Neq(ref k) if k == "host")); + } + other => panic!("expected ShowTagValues, got {other:?}"), + } + let stmt = parse_ddl_statement(r#"SHOW TAG VALUES FROM cpu WITH KEY =~ /^host/"#).unwrap(); + match stmt { + Statement::ShowTagValues(s) => { + assert!(matches!(s.tag_key, TagKeySelector::Regex(ref p) if p == "^host")); + } + other => panic!("expected ShowTagValues, got {other:?}"), + } + let stmt = + parse_ddl_statement(r#"SHOW TAG VALUES FROM cpu WITH KEY IN ("host", "region")"#) + .unwrap(); + match stmt { + Statement::ShowTagValues(s) => { + assert!( + matches!(s.tag_key, TagKeySelector::In(ref keys) if keys == &["host", "region"]) + ); + } + other => panic!("expected ShowTagValues, got {other:?}"), + } + } + #[test] fn trailing_garbage_rejected() { assert!(parse_ddl_statement("DROP DATABASE foo bar baz").is_err()); - let err = parse_ddl_statement("SHOW MEASUREMENTS WITH MEASUREMENT =~ /cpu/").unwrap_err(); - assert!( - err.to_string().contains("not supported"), - "WITH MEASUREMENT must fail loudly, got: {err}" - ); } #[test] diff --git a/hyperbytedb/src/timeseriesql/digest.rs b/hyperbytedb/src/timeseriesql/digest.rs index 875e7b4..aee53be 100644 --- a/hyperbytedb/src/timeseriesql/digest.rs +++ b/hyperbytedb/src/timeseriesql/digest.rs @@ -86,6 +86,9 @@ fn normalize_statement(stmt: &Statement) -> String { Statement::ShowMeasurements(s) => { out.push_str("show measurements"); normalize_on_db(&mut out, &s.database); + if let Some(filter) = &s.measurement_filter { + normalize_key_selector(&mut out, "measurement", filter); + } normalize_show_tail(&mut out, &s.condition, &s.limit, &s.offset); } Statement::ShowTagKeys(s) => { @@ -104,21 +107,7 @@ fn normalize_statement(stmt: &Statement) -> String { out.push_str(" from "); normalize_measurement(&mut out, m); } - match &s.tag_key { - TagKeySelector::All => out.push_str(" with key = *"), - TagKeySelector::Eq(k) => { - write!(out, " with key = {}", k).ok(); - } - TagKeySelector::Neq(k) => { - write!(out, " with key != {}", k).ok(); - } - TagKeySelector::Regex(r) => { - write!(out, " with key =~ /{}/", r).ok(); - } - TagKeySelector::In(keys) => { - write!(out, " with key in ({})", keys.join(", ")).ok(); - } - } + normalize_key_selector(&mut out, "key", &s.tag_key); normalize_show_tail(&mut out, &s.condition, &s.limit, &s.offset); } Statement::ShowFieldKeys(s) => { @@ -210,6 +199,16 @@ fn normalize_statement(stmt: &Statement) -> String { out } +fn normalize_key_selector(out: &mut String, label: &str, selector: &TagKeySelector) { + match selector { + TagKeySelector::All => write!(out, " with {label} = *").ok(), + TagKeySelector::Eq(k) => write!(out, " with {label} = {k}").ok(), + TagKeySelector::Neq(k) => write!(out, " with {label} != {k}").ok(), + TagKeySelector::Regex(r) => write!(out, " with {label} =~ /{r}/").ok(), + TagKeySelector::In(keys) => write!(out, " with {label} in ({})", keys.join(", ")).ok(), + }; +} + fn normalize_on_db(out: &mut String, database: &Option) { if let Some(db) = database { write!(out, " on {}", db).ok(); diff --git a/hyperbytedb/src/timeseriesql/lexer.rs b/hyperbytedb/src/timeseriesql/lexer.rs index 41f53bd..8fc92a6 100644 --- a/hyperbytedb/src/timeseriesql/lexer.rs +++ b/hyperbytedb/src/timeseriesql/lexer.rs @@ -227,6 +227,11 @@ pub fn parse_duration_text(input: &str) -> Result, HyperbytedbError> let value: i64 = num_str .parse() .map_err(|_| HyperbytedbError::QueryParse(format!("invalid duration: {input}")))?; + if value < 0 { + return Err(HyperbytedbError::QueryParse( + "duration must not be negative".to_string(), + )); + } let mut matched = false; for (suffix, mult) in units { diff --git a/hyperbytedb/src/timeseriesql/parser.rs b/hyperbytedb/src/timeseriesql/parser.rs index 21daf35..f2ce43e 100644 --- a/hyperbytedb/src/timeseriesql/parser.rs +++ b/hyperbytedb/src/timeseriesql/parser.rs @@ -109,7 +109,7 @@ fn parse_select(input: &str) -> Result { // Parse WHERE — strip any trailing fill(...) that Grafana may attach without GROUP BY if let Some(where_str) = parts.get("where") { - let (where_clean, standalone_fill) = strip_trailing_fill(where_str); + let (where_clean, standalone_fill) = strip_trailing_fill(where_str)?; stmt.condition = Some(parse_expr(&where_clean)?); if standalone_fill.is_some() { stmt.fill = standalone_fill; @@ -199,10 +199,10 @@ struct ScannedChar { /// - parenthesis depth. /// /// The output has exactly one entry per input char, in order. -fn scan_chars(input: &str) -> Vec { +fn scan_chars(input: &str) -> Result, HyperbytedbError> { let chars: Vec<(usize, char)> = input.char_indices().collect(); let mut out = Vec::with_capacity(chars.len()); - let mut depth: i32 = 0; + let mut depth: u32 = 0; let mut i = 0usize; while i < chars.len() { let (idx, ch) = chars[i]; @@ -212,7 +212,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx, ch, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -221,7 +221,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx: jdx, ch: c, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -231,7 +231,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx: kdx, ch: k, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -242,7 +242,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx: kdx, ch: k, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -256,7 +256,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx, ch, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -265,7 +265,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx: jdx, ch: c, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -274,7 +274,7 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx: kdx, ch: k, - depth, + depth: depth as i32, masked: true, }); i += 1; @@ -287,18 +287,23 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx, ch, - depth, + depth: depth as i32, masked: false, }); depth += 1; i += 1; } ')' => { + if depth == 0 { + return Err(HyperbytedbError::QueryParse(format!( + "unbalanced ')' in expression: {input}" + ))); + } depth -= 1; out.push(ScannedChar { idx, ch, - depth, + depth: depth as i32, masked: false, }); i += 1; @@ -307,14 +312,19 @@ fn scan_chars(input: &str) -> Vec { out.push(ScannedChar { idx, ch, - depth, + depth: depth as i32, masked: false, }); i += 1; } } } - out + if depth != 0 { + return Err(HyperbytedbError::QueryParse(format!( + "unclosed '(' in expression: {input}" + ))); + } + Ok(out) } /// Whether a `/` at `chars[pos]` begins a regex literal rather than division. @@ -443,7 +453,7 @@ fn split_clauses( ("TZ", "tz"), ]; - let scan = scan_chars(input); + let scan = scan_chars(input)?; // (keyword, key, keyword start byte, value start byte) let mut found: Vec<(&str, &str, usize, usize)> = Vec::new(); let mut i = 0; @@ -496,7 +506,7 @@ fn parse_field_list(input: &str) -> Result, HyperbytedbError> { }]); } - let parts = split_top_level_commas(input); + let parts = split_top_level_commas(input)?; let mut fields = Vec::new(); for part in parts { @@ -507,8 +517,8 @@ fn parse_field_list(input: &str) -> Result, HyperbytedbError> { Ok(fields) } -fn split_top_level_commas(input: &str) -> Vec<&str> { - let scan = scan_chars(input); +fn split_top_level_commas(input: &str) -> Result, HyperbytedbError> { + let scan = scan_chars(input)?; let mut parts = Vec::new(); let mut last = 0; for sc in &scan { @@ -518,14 +528,14 @@ fn split_top_level_commas(input: &str) -> Vec<&str> { } } parts.push(&input[last..]); - parts + Ok(parts) } fn parse_field_expr(input: &str) -> Result { let input = input.trim(); // Check for AS alias - let scan = scan_chars(input); + let scan = scan_chars(input)?; let (expr_str, alias) = if let Some((pos, end)) = find_keyword_position(input, &scan, "AS") { let expr_part = input[..pos].trim(); let alias_part = input[end..].trim().trim_matches('"'); @@ -585,7 +595,7 @@ pub fn parse_expr(input: &str) -> Result { } fn try_parse_logical_expr(input: &str) -> Result, HyperbytedbError> { - let scan = scan_chars(input); + let scan = scan_chars(input)?; // OR has the lowest precedence in InfluxQL, so split at OR first: the // operator split earliest ends up at the root of the tree and binds // loosest. Which OR occurrence is split at is semantically neutral. @@ -616,7 +626,7 @@ fn try_parse_comparison_expr(input: &str) -> Result, HyperbytedbErr (">", BinaryOp::Gt), ]; - let scan = scan_chars(input); + let scan = scan_chars(input)?; for (op_str, op) in &operators { if let Some(pos) = find_top_level_operator(input, &scan, op_str) { let left = parse_expr(&input[..pos])?; @@ -645,7 +655,7 @@ fn try_parse_arithmetic_expr(input: &str) -> Result, HyperbytedbErr ], ]; - let scan = scan_chars(input); + let scan = scan_chars(input)?; for level in levels { for sc in scan.iter().rev() { if sc.masked || sc.depth != 0 { @@ -733,6 +743,11 @@ fn parse_atom(input: &str) -> Result { let rest = input[1..].trim(); if rest.parse::().is_err() && rest.parse::().is_err() { let inner = parse_expr(rest)?; + if matches!(inner, Expr::DurationLiteral(_)) { + return Err(HyperbytedbError::QueryParse( + "duration must not be negative".to_string(), + )); + } return Ok(Expr::BinaryExpr(Box::new(BinaryExpr { left: Expr::IntegerLiteral(0), op: BinaryOp::Sub, @@ -784,7 +799,7 @@ fn parse_atom(input: &str) -> Result { let args = if args_str.trim().is_empty() { Vec::new() } else { - split_top_level_commas(args_str) + split_top_level_commas(args_str)? .iter() .map(|a| parse_expr(a)) .collect::, _>>()? @@ -796,7 +811,7 @@ fn parse_atom(input: &str) -> Result { } // Duration literal: number followed by unit - if let Some(dur) = try_parse_duration(input) { + if let Some(dur) = try_parse_duration(input)? { return Ok(Expr::DurationLiteral(dur)); } @@ -814,7 +829,7 @@ fn parse_atom(input: &str) -> Result { return Ok(Expr::Identifier(name)); } - let scan = scan_chars(input); + let scan = scan_chars(input)?; // Identifier with ::field or ::tag suffix if let Some(k) = (0..scan.len().saturating_sub(1)).find(|&k| { @@ -850,7 +865,7 @@ fn parse_atom(input: &str) -> Result { Ok(Expr::Identifier(input.to_string())) } -fn try_parse_duration(input: &str) -> Option { +fn try_parse_duration(input: &str) -> Result, HyperbytedbError> { let input = input.trim(); let units = [ ("ns", DurationUnit::Nanosecond), @@ -869,13 +884,18 @@ fn try_parse_duration(input: &str) -> Option { if let Some(num_str) = input.strip_suffix(suffix) && let Ok(value) = num_str.parse::() { - return Some(Duration { + if value < 0 { + return Err(HyperbytedbError::QueryParse( + "duration must not be negative".to_string(), + )); + } + return Ok(Some(Duration { value, unit: unit.clone(), - }); + })); } } - None + Ok(None) } fn parse_from_sources(input: &str) -> Result, HyperbytedbError> { @@ -892,7 +912,7 @@ fn parse_from_sources(input: &str) -> Result, Hyperbytedb } } - let parts = split_top_level_commas(input); + let parts = split_top_level_commas(input)?; let mut sources = Vec::new(); for part in parts { @@ -917,7 +937,7 @@ fn parse_measurement(input: &str) -> Result { // Fully qualified: "db"."rp"."measurement" or db.rp.measurement — split // on dots outside quotes so `FROM "app.requests"` stays one measurement. - let scan = scan_chars(input); + let scan = scan_chars(input)?; let mut parts: Vec<&str> = Vec::new(); let mut last = 0; for sc in &scan { @@ -996,7 +1016,7 @@ fn parse_group_by_clause(input: &str) -> Result<(GroupBy, Option), H let mut dims_str = input.to_string(); // Check for fill() at end - let scan = scan_chars(input); + let scan = scan_chars(input)?; if let Some(fill_pos) = rfind_top_level_ci(input, &scan, "FILL(") { let fill_end = input[fill_pos..].find(')').map(|p| fill_pos + p + 1); if let Some(fill_end) = fill_end { @@ -1006,7 +1026,7 @@ fn parse_group_by_clause(input: &str) -> Result<(GroupBy, Option), H } } - let parts = split_top_level_commas(&dims_str); + let parts = split_top_level_commas(&dims_str)?; let mut dimensions = Vec::new(); for part in parts { @@ -1020,13 +1040,13 @@ fn parse_group_by_clause(input: &str) -> Result<(GroupBy, Option), H && part.ends_with(')') { let args_str = &part[5..part.len() - 1]; - let args = split_top_level_commas(args_str); + let args = split_top_level_commas(args_str)?; - let interval = try_parse_duration(args[0].trim()).ok_or_else(|| { + let interval = try_parse_duration(args[0].trim())?.ok_or_else(|| { HyperbytedbError::QueryParse(format!("invalid duration in time(): {}", args[0])) })?; let offset = if args.len() > 1 { - Some(try_parse_duration(args[1].trim()).ok_or_else(|| { + Some(try_parse_duration(args[1].trim())?.ok_or_else(|| { HyperbytedbError::QueryParse(format!("invalid offset in time(): {}", args[1])) })?) } else { @@ -1048,18 +1068,18 @@ fn parse_group_by_clause(input: &str) -> Result<(GroupBy, Option), H /// Strip a trailing `fill(...)` from a clause string (e.g. WHERE or ORDER BY) /// that Grafana may send even without a GROUP BY clause. -fn strip_trailing_fill(input: &str) -> (String, Option) { - let scan = scan_chars(input); +fn strip_trailing_fill(input: &str) -> Result<(String, Option), HyperbytedbError> { + let scan = scan_chars(input)?; if let Some(pos) = rfind_top_level_ci(input, &scan, "FILL(") && let Some(close) = input[pos..].find(')') { let fill_inner = &input[pos + 5..pos + close]; let rest = input[..pos].trim().to_string(); if let Ok(f) = parse_fill_option(fill_inner) { - return (rest, Some(f)); + return Ok((rest, Some(f))); } } - (input.to_string(), None) + Ok((input.to_string(), None)) } fn parse_fill_option(input: &str) -> Result { @@ -1957,4 +1977,26 @@ mod tests { .any(|d| matches!(d, Dimension::Regex(r) if r == "host.*")) ); } + + #[test] + fn negative_durations_rejected_in_select() { + for q in [ + "SELECT * FROM cpu GROUP BY time(-5m)", + "SELECT * FROM cpu WHERE time > now() - -1h", + ] { + assert!( + parse_query(q).is_err(), + "negative duration must be rejected: {q}" + ); + } + } + + #[test] + fn unbalanced_paren_rejected() { + let err = parse_query("SELECT * FROM cpu) WHERE host = 'a'").unwrap_err(); + assert!( + err.to_string().contains("unbalanced"), + "expected unbalanced paren error, got: {err}" + ); + } } diff --git a/hyperbytedb/src/timeseriesql/to_clickhouse.rs b/hyperbytedb/src/timeseriesql/to_clickhouse.rs index acd2fd9..2f87442 100644 --- a/hyperbytedb/src/timeseriesql/to_clickhouse.rs +++ b/hyperbytedb/src/timeseriesql/to_clickhouse.rs @@ -1,3 +1,4 @@ +use crate::domain::chdb_naming::QuotedTableName; use crate::domain::column_mapping::ColumnMapping; use crate::domain::rollup::{RollupCombine, aggregate_source_field_name, mean_rollup_column_names}; use crate::error::HyperbytedbError; @@ -95,7 +96,7 @@ pub fn validate_select_into(stmt: &SelectStatement) -> Result<(), HyperbytedbErr #[derive(Debug, Clone, Copy)] pub struct SeriesJoin<'a> { /// Backtick-quoted `___series` table name. - pub table: &'a str, + pub table: &'a QuotedTableName, /// Force the inline tag-rejoin view even when the query body references no /// tag. Set when tombstone predicates (spliced into WHERE post-translation) /// reference tag columns that must be present in the FROM source. @@ -186,7 +187,7 @@ fn translate_inner( // Include GROUP BY tag columns in SELECT so they appear in the result // and can be used to split rows into separate InfluxDB series. for tag in gb.tag_dimensions() { - select_parts.push(select_tag_column_sql(tag, mapping)); + select_parts.push(select_tag_column_sql(tag, mapping)?); } } @@ -211,7 +212,7 @@ fn translate_inner( let projects_point_time = is_raw_select || (has_raw_transform && !has_group_by_time && !has_star); if projects_point_time { - select_parts.insert(0, quote_identifier("time")); + select_parts.insert(0, quote_phys_identifier("time")); } let field_strs: Vec = stmt @@ -226,7 +227,7 @@ fn translate_inner( mapping, ) }) - .collect::, _>>()?; + .collect::, HyperbytedbError>>()?; select_parts.extend(field_strs); write!(out, "{}", select_parts.join(", "))?; @@ -258,7 +259,7 @@ fn translate_inner( // Must match the SELECT expression: physical column name (handles the // `__tag__` collision prefix). Previously emitted the logical name, // which is wrong for collision-renamed tags. - gb_parts.push(group_by_tag_sql(tag, mapping)); + gb_parts.push(group_by_tag_sql(tag, mapping)?); } } @@ -300,7 +301,7 @@ fn translate_inner( // per-tag series is never filled — which surfaces as "no data" in Grafana. if do_fill && let Some(ref gb) = stmt.group_by { for tag in gb.tag_dimensions() { - write!(out, "{} ASC, ", group_by_tag_sql(tag, mapping))?; + write!(out, "{} ASC, ", group_by_tag_sql(tag, mapping)?)?; } } @@ -340,19 +341,21 @@ fn translate_inner( match effective_fill { // fill(previous): use INTERPOLATE to carry forward last known value FillOption::Previous if !field_aliases.is_empty() => { - let interp_cols: Vec = - field_aliases.iter().map(|a| quote_identifier(a)).collect(); + let interp_cols: Vec = field_aliases + .iter() + .map(|a| quote_identifier(a)) + .collect::, HyperbytedbError>>()?; write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; } // fill(linear): use INTERPOLATE with linear expressions FillOption::Linear if !field_aliases.is_empty() => { let interp_cols: Vec = field_aliases .iter() - .map(|a| { - let q = quote_identifier(a); - format!("{q} AS {q}") + .map(|a| -> Result { + let q = quote_identifier(a)?; + Ok(format!("{q} AS {q}")) }) - .collect(); + .collect::, HyperbytedbError>>()?; write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; } // fill(): WITH FILL-generated rows get column defaults @@ -362,8 +365,8 @@ fn translate_inner( FillOption::Value(v) if !field_aliases.is_empty() => { let interp_cols: Vec = field_aliases .iter() - .map(|a| format!("{} AS {}", quote_identifier(a), format_float(v))) - .collect(); + .map(|a| Ok(format!("{} AS {}", quote_identifier(a)?, format_float(v)))) + .collect::, HyperbytedbError>>()?; write!(out, " INTERPOLATE ({})", interp_cols.join(", "))?; } _ => {} @@ -385,8 +388,8 @@ fn translate_inner( // line so tombstone WHERE-splicing targets only the inner query. let mut order_parts: Vec = tag_dims .iter() - .map(|t| format!("{} ASC", quote_identifier(t))) - .collect(); + .map(|t| Ok(format!("{} ASC", quote_identifier(t)?))) + .collect::, HyperbytedbError>>()?; order_parts.push("__time DESC".to_string()); out = format!( "SELECT * FROM (\n{out}\n) ORDER BY {}", @@ -408,13 +411,13 @@ fn translate_inner( if !transform_aliases.is_empty() { let cond = transform_aliases .iter() - .map(|a| format!("{} IS NOT NULL", quote_identifier(a))) - .collect::>() + .map(|a| Ok(format!("{} IS NOT NULL", quote_identifier(a)?))) + .collect::, HyperbytedbError>>()? .join(" OR "); let dir = if time_desc { "DESC" } else { "ASC" }; out = format!( "SELECT * FROM (\n{out}\n) WHERE {cond} ORDER BY {} {dir}", - quote_identifier("time") + quote_phys_identifier("time") ); } } @@ -426,7 +429,7 @@ fn translate_inner( let by_cols = tag_dims .iter() .map(|t| quote_identifier(t)) - .collect::>() + .collect::, HyperbytedbError>>()? .join(", "); let limit = stmt.limit.unwrap_or(0); match stmt.offset { @@ -530,14 +533,14 @@ pub fn rename_time_bucket_alias(sql: &str) -> String { /// for the destination measurement schema. pub fn translate_select_into( stmt: &SelectStatement, - dest_table: &str, + dest_table: &QuotedTableName, source: &str, mapping: Option<&ColumnMapping>, ) -> Result { validate_select_into(stmt)?; let select_sql = translate_inner(stmt, source, mapping, None, None)?; let select_sql = rename_time_bucket_alias(&select_sql); - Ok(format!("INSERT INTO {}\n{}", dest_table, select_sql)) + Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) } fn translate_materialized_view_field( @@ -550,12 +553,12 @@ fn translate_materialized_view_field( { let source = aggregate_source_field_name(func)?; let col = mapping.physical_select_identifier(&source); - let col_q = quote_identifier(&col); + let col_q = quote_phys_identifier(&col); let (sum_col, count_col) = mean_rollup_column_names(&source); return Ok(format!( "sum({col_q}) AS {}, count({col_q}) AS {}", - quote_identifier(&sum_col), - quote_identifier(&count_col) + quote_phys_identifier(&sum_col), + quote_phys_identifier(&count_col) )); } translate_field(field, false, 0.0, group_by, Some(mapping)) @@ -582,8 +585,8 @@ fn mapping_with_mv_aggregate_fields(mapping: &ColumnMapping, fields: &[Field]) - /// assigns a destination `series_id` via [`crate::domain::series::series_id_ch_sql`]. pub fn translate_materialized_view_select( stmt: &SelectStatement, - source_fact: &str, - source_series: &str, + source_fact: &QuotedTableName, + source_series: &QuotedTableName, dest_measurement: &str, mapping: &ColumnMapping, ) -> Result { @@ -614,7 +617,7 @@ pub fn translate_materialized_view_select( let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( dest_measurement, &grouped_tags, - |tag| quote_identifier(&mapping.tag_column_name(tag)), + |tag| quote_phys_identifier(&mapping.physical_tag_column_name(tag)), "s", ); @@ -632,10 +635,10 @@ pub fn translate_materialized_view_select( { let source = aggregate_source_field_name(func)?; let col = mapping.physical_select_identifier(&source); - let col_q = quote_identifier(&col); + let col_q = quote_phys_identifier(&col); let (sum_col, count_col) = mean_rollup_column_names(&source); - let sum_expr = format!("sum({col_q}) AS {}", quote_identifier(&sum_col)); - let count_expr = format!("count({col_q}) AS {}", quote_identifier(&count_col)); + let sum_expr = format!("sum({col_q}) AS {}", quote_phys_identifier(&sum_col)); + let count_expr = format!("count({col_q}) AS {}", quote_phys_identifier(&count_col)); field_expr_by_name.insert(sum_col.clone(), sum_expr); field_expr_by_name.insert(count_col.clone(), count_expr); } else { @@ -662,7 +665,7 @@ pub fn translate_materialized_view_select( for tag in &grouped_tags { group_parts.push(format!( "s.{}", - quote_identifier(&mapping.tag_column_name(tag)) + quote_phys_identifier(&mapping.physical_tag_column_name(tag)) )); } @@ -697,7 +700,7 @@ pub fn translate_materialized_view_select( /// that reflects the destination schema for correct physical column naming. pub fn translate_materialized_view_series_select( stmt: &SelectStatement, - source_series: &str, + source_series: &QuotedTableName, dest_measurement: &str, mapping: &ColumnMapping, dest_field_names: Option<&std::collections::HashSet>, @@ -728,22 +731,22 @@ pub fn translate_materialized_view_series_select( Some(dfn) => { let fields: std::collections::HashSet<&str> = dfn.iter().map(|s| s.as_str()).collect(); - crate::domain::column_mapping::tag_column_name(tag, &fields) + crate::domain::chdb_naming::tag_column_name(tag, &fields) } - None => mapping.tag_column_name(tag), + None => mapping.physical_tag_column_name(tag), } }; let series_id_expr = crate::domain::series::series_id_ch_sql_for_tags( dest_measurement, &grouped_tags, - |tag| quote_identifier(&tag_phys_name(tag)), + |tag| quote_phys_identifier(&tag_phys_name(tag)), "s", ); let tag_cols: Vec = grouped_tags .iter() - .map(|tag| format!("s.{}", quote_identifier(&tag_phys_name(tag)))) + .map(|tag| format!("s.{}", quote_phys_identifier(&tag_phys_name(tag)))) .collect(); let mut select_parts = vec![format!("min({series_id_expr}) AS series_id")]; @@ -759,9 +762,9 @@ pub fn translate_materialized_view_series_select( /// `INSERT INTO SELECT ...` for one-time MV backfill of historical data. pub fn translate_materialized_view_backfill( stmt: &SelectStatement, - dest_table: &str, - source_fact: &str, - source_series: &str, + dest_table: &QuotedTableName, + source_fact: &QuotedTableName, + source_series: &QuotedTableName, dest_measurement: &str, mapping: &ColumnMapping, ) -> Result { @@ -783,14 +786,14 @@ fn materialized_view_dest_insert_columns( stmt: &SelectStatement, ) -> Result { let mut cols = vec![ - quote_identifier("time"), - quote_identifier("origin_node_id"), - quote_identifier("ingest_seq"), - quote_identifier("series_id"), + quote_phys_identifier("time"), + quote_phys_identifier("origin_node_id"), + quote_phys_identifier("ingest_seq"), + quote_phys_identifier("series_id"), ]; let mut field_names = materialized_view_dest_field_names(stmt)?; field_names.sort(); - cols.extend(field_names.into_iter().map(|n| quote_identifier(&n))); + cols.extend(field_names.into_iter().map(|n| quote_phys_identifier(&n))); Ok(cols.join(", ")) } @@ -820,8 +823,8 @@ fn materialized_view_dest_field_names( /// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` DDL for the fact MV. pub fn build_create_fact_materialized_view( - mv_name: &str, - dest_table: &str, + mv_name: &QuotedTableName, + dest_table: &QuotedTableName, select_sql: &str, ) -> String { format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_table} AS\n{select_sql}") @@ -829,8 +832,8 @@ pub fn build_create_fact_materialized_view( /// Full `CREATE MATERIALIZED VIEW ... TO ... AS SELECT ...` for the series MV. pub fn build_create_series_materialized_view( - mv_name: &str, - dest_series: &str, + mv_name: &QuotedTableName, + dest_series: &QuotedTableName, select_sql: &str, ) -> String { format!("CREATE MATERIALIZED VIEW {mv_name} TO {dest_series} AS\n{select_sql}") @@ -841,15 +844,15 @@ pub fn build_create_series_materialized_view( /// measurement's dimension table. pub fn translate_select_into_native( stmt: &SelectStatement, - dest_table: &str, - source_table: &str, + dest_table: &QuotedTableName, + source_table: &QuotedTableName, mapping: Option<&ColumnMapping>, series: Option>, ) -> Result { validate_select_into(stmt)?; - let select_sql = translate_inner(stmt, source_table, mapping, series, None)?; + let select_sql = translate_inner(stmt, source_table.as_str(), mapping, series, None)?; let select_sql = rename_time_bucket_alias(&select_sql); - Ok(format!("INSERT INTO {}\n{}", dest_table, select_sql)) + Ok(format!("INSERT INTO {dest_table}\n{select_sql}")) } /// Like translate, but uses a custom source expression instead of file() - used for subqueries. @@ -909,21 +912,21 @@ fn query_references_tag(stmt: &SelectStatement, m: &ColumnMapping) -> bool { /// `argMaxIf` can stitch a correct `available` from one row with a corrupt /// `used_percent` from another (e.g. async replication writing a second row /// for the same instant), which produces nonsense Grafana percentages. -pub fn build_coalesced_fact_view(fact_table: &str, mapping: &ColumnMapping) -> String { +pub fn build_coalesced_fact_view(fact_table: &QuotedTableName, mapping: &ColumnMapping) -> String { build_coalesced_fact_view_impl(fact_table, mapping, false) } /// Like [`build_coalesced_fact_view`], but preserves `ingest_seq` / `origin_node_id` for /// downstream aggregates (materialized view source dedup). pub fn build_coalesced_fact_view_with_row_meta( - fact_table: &str, + fact_table: &QuotedTableName, mapping: &ColumnMapping, ) -> String { build_coalesced_fact_view_impl(fact_table, mapping, true) } fn build_coalesced_fact_view_impl( - fact_table: &str, + fact_table: &QuotedTableName, mapping: &ColumnMapping, include_row_metadata: bool, ) -> String { @@ -932,7 +935,7 @@ fn build_coalesced_fact_view_impl( let field_aggs: Vec = field_cols .iter() .map(|f| { - let q = quote_identifier(f); + let q = quote_phys_identifier(f); let agg = match mapping.field_rollups.get(*f) { Some(RollupCombine::Sum) => format!("sum({q})"), Some(RollupCombine::Min) => format!("min({q})"), @@ -974,7 +977,9 @@ fn build_from_source( stmt: &SelectStatement, ) -> String { let fact = match mapping { - Some(m) => build_coalesced_fact_view(fact_table, m), + Some(m) => { + build_coalesced_fact_view(&QuotedTableName::new_quoted(fact_table.to_string()), m) + } None => fact_table.to_string(), }; let (Some(sj), Some(m)) = (series, mapping) else { @@ -983,7 +988,11 @@ fn build_from_source( if !sj.force && !query_references_tag(stmt, m) { return fact; } - let mut tag_cols: Vec = m.tag_keys.iter().map(|t| m.tag_column_name(t)).collect(); + let mut tag_cols: Vec = m + .tag_keys + .iter() + .map(|t| m.physical_tag_column_name(t)) + .collect(); if tag_cols.is_empty() { return fact; } @@ -998,7 +1007,7 @@ fn build_from_source( tag_cols.sort(); let projected = tag_cols .iter() - .map(|c| format!("s.{}", quote_identifier(c))) + .map(|c| format!("s.{}", quote_phys_identifier(c))) .collect::>() .join(", "); format!( @@ -1009,9 +1018,12 @@ fn build_from_source( /// GROUP BY expression for a tag: the physical column name, matching the SELECT /// side. Without a mapping, falls back to the logical name (unchanged behaviour). -fn group_by_tag_sql(tag: &str, mapping: Option<&ColumnMapping>) -> String { +fn group_by_tag_sql( + tag: &str, + mapping: Option<&ColumnMapping>, +) -> Result { match mapping { - Some(m) => quote_identifier(&m.tag_column_name(tag)), + Some(m) => Ok(quote_phys_identifier(&m.physical_tag_column_name(tag))), None => quote_identifier(tag), } } @@ -1044,15 +1056,22 @@ fn time_bucket_expr_on( } } -fn select_tag_column_sql(tag: &str, mapping: Option<&ColumnMapping>) -> String { +fn select_tag_column_sql( + tag: &str, + mapping: Option<&ColumnMapping>, +) -> Result { let Some(m) = mapping else { return quote_identifier(tag); }; - let phys = m.tag_column_name(tag); + let phys = m.physical_tag_column_name(tag); if phys == tag { quote_identifier(tag) } else { - format!("{} AS {}", quote_identifier(&phys), quote_identifier(tag)) + Ok(format!( + "{} AS {}", + quote_phys_identifier(&phys), + quote_identifier(tag)? + )) } } @@ -1069,7 +1088,7 @@ fn translate_field( .clone() .or_else(|| default_field_alias(&field.expr)); Ok(match alias { - Some(a) => format!("{} AS {}", sql, quote_identifier(&a)), + Some(a) => format!("{} AS {}", sql, quote_identifier(&a)?), None => sql, }) } @@ -1126,13 +1145,13 @@ fn translate_field_expr( let col = mapping .map(|m| m.physical_select_identifier(name)) .unwrap_or_else(|| name.clone()); - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } Expr::FieldRef { name, .. } => { let col = mapping .map(|m| m.physical_select_identifier(name)) .unwrap_or_else(|| name.clone()); - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } Expr::Call(func) => translate_aggregate_call(func, use_fill, fill_value, group_by, mapping), Expr::BinaryExpr(be) => translate_binary_expr(be, use_fill, fill_value, group_by, mapping), @@ -1198,8 +1217,8 @@ fn translate_aggregate_call( && let Expr::Identifier(name) | Expr::FieldRef { name, .. } = arg && let Some(mean_def) = m.mean_fields.get(name) { - let sum_q = quote_identifier(&mean_def.sum_col); - let count_q = quote_identifier(&mean_def.count_col); + let sum_q = quote_phys_identifier(&mean_def.sum_col); + let count_q = quote_phys_identifier(&mean_def.count_col); return Ok(wrap_fill(format!( "(sum({sum_q}) / nullIf(sum({count_q}), 0))" ))); @@ -1300,7 +1319,7 @@ fn translate_aggregate_call( "DERIVATIVE" | "NON_NEGATIVE_DERIVATIVE" => { let field_arg = get_single_arg(func, &name_upper)?; let f = translate_field_or_nested(field_arg, group_by, mapping)?; - let window = build_window_clause(group_by, mapping); + let window = build_window_clause(group_by, mapping)?; let unit_nanos: i64 = if func.args.len() >= 2 { match &func.args[1] { Expr::DurationLiteral(d) => d.to_nanos(), @@ -1326,7 +1345,7 @@ fn translate_aggregate_call( "DIFFERENCE" | "NON_NEGATIVE_DIFFERENCE" => { let arg = get_single_arg(func, &name_upper)?; let f = translate_field_or_nested(arg, group_by, mapping)?; - let window = build_window_clause(group_by, mapping); + let window = build_window_clause(group_by, mapping)?; let diff = format!("({f} - lagInFrame({f}, 1) {window})"); if name_upper == "NON_NEGATIVE_DIFFERENCE" { format!("if({diff} >= 0, {diff}, NULL)") @@ -1355,11 +1374,11 @@ fn translate_aggregate_call( .iter() .map(|t| { let phys = mapping - .map(|m| m.tag_column_name(t)) + .map(|m| m.physical_tag_column_name(t)) .unwrap_or_else(|| t.to_string()); - quote_identifier(&phys) + Ok(quote_phys_identifier(&phys)) }) - .collect::>() + .collect::, HyperbytedbError>>()? .join(", "); format!("PARTITION BY {p} ") }; @@ -1385,11 +1404,11 @@ fn translate_aggregate_call( .iter() .map(|t| { let phys = mapping - .map(|m| m.tag_column_name(t)) + .map(|m| m.physical_tag_column_name(t)) .unwrap_or_else(|| t.to_string()); - quote_identifier(&phys) + Ok(quote_phys_identifier(&phys)) }) - .collect::>() + .collect::, HyperbytedbError>>()? .join(", "); format!("PARTITION BY {p} ") }; @@ -1400,7 +1419,7 @@ fn translate_aggregate_call( "ELAPSED" => { let _field_arg = get_single_arg(func, "ELAPSED")?; let time_ref = window_time_ref(group_by); - let window = build_window_clause(group_by, mapping); + let window = build_window_clause(group_by, mapping)?; let unit_nanos: i64 = if func.args.len() >= 2 { match &func.args[1] { Expr::DurationLiteral(d) => d.to_nanos(), @@ -1437,7 +1456,7 @@ fn translate_aggregate_arg( let col = mapping .map(|m| m.physical_select_identifier(name)) .unwrap_or_else(|| name.clone()); - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } Expr::Star => Ok("*".to_string()), _ => Err(HyperbytedbError::QueryParse(format!( @@ -1476,24 +1495,29 @@ fn window_time_ref(group_by: Option<&GroupBy>) -> &'static str { /// Build the OVER (...) window clause for transform functions. /// Includes PARTITION BY for GROUP BY tag dimensions so that window /// functions (lagInFrame, etc.) operate within each series independently. -fn build_window_clause(group_by: Option<&GroupBy>, mapping: Option<&ColumnMapping>) -> String { +fn build_window_clause( + group_by: Option<&GroupBy>, + mapping: Option<&ColumnMapping>, +) -> Result { let time_ref = window_time_ref(group_by); let partition_tags: Vec<&str> = group_by.map(|gb| gb.tag_dimensions()).unwrap_or_default(); if partition_tags.is_empty() { - format!("OVER (ORDER BY {time_ref})") + Ok(format!("OVER (ORDER BY {time_ref})")) } else { let partition = partition_tags .iter() .map(|t| { let phys = mapping - .map(|m| m.tag_column_name(t)) + .map(|m| m.physical_tag_column_name(t)) .unwrap_or_else(|| t.to_string()); - quote_identifier(&phys) + Ok(quote_phys_identifier(&phys)) }) - .collect::>() + .collect::, HyperbytedbError>>()? .join(", "); - format!("OVER (PARTITION BY {partition} ORDER BY {time_ref})") + Ok(format!( + "OVER (PARTITION BY {partition} ORDER BY {time_ref})" + )) } } @@ -1518,19 +1542,14 @@ fn get_two_args<'a>( /// Translate a WHERE condition expression to ClickHouse SQL. /// Used by the DELETE statement handler to serialize tombstone predicates. -pub fn translate_condition(expr: &Expr, out: &mut String) -> Result<(), HyperbytedbError> { - translate_expr(expr, out, true, None) -} - -/// Like [`translate_condition`] but resolves tag identifiers to their physical -/// column names. Used to store tombstone predicates so the spliced WHERE clause -/// matches the tag columns exposed by the series-rejoin inline view. -pub fn translate_condition_with_mapping( +/// Tag identifiers are resolved to their physical column names so the spliced +/// WHERE clause matches the tag columns exposed by the series-rejoin inline view. +pub fn translate_condition( expr: &Expr, - mapping: Option<&ColumnMapping>, + mapping: &ColumnMapping, out: &mut String, ) -> Result<(), HyperbytedbError> { - translate_expr(expr, out, true, mapping) + translate_expr(expr, out, true, Some(mapping)) } fn tag_field_collision(m: &ColumnMapping, name: &str) -> bool { @@ -1547,7 +1566,11 @@ fn is_where_literal(e: &Expr) -> bool { ) } -fn where_identifier_physical_name(m: &ColumnMapping, name: &str, other: &Expr) -> String { +fn where_identifier_physical_name( + m: &ColumnMapping, + name: &str, + other: &Expr, +) -> Result { if !tag_field_collision(m, name) { return quote_identifier(name); } @@ -1555,8 +1578,10 @@ fn where_identifier_physical_name(m: &ColumnMapping, name: &str, other: &Expr) - Expr::IntegerLiteral(_) | Expr::FloatLiteral(_) | Expr::BooleanLiteral(_) => { quote_identifier(name) } - Expr::StringLiteral(_) | Expr::Regex(_) => quote_identifier(&m.tag_column_name(name)), - _ => quote_identifier(&m.tag_column_name(name)), + Expr::StringLiteral(_) | Expr::Regex(_) => { + Ok(quote_phys_identifier(&m.physical_tag_column_name(name))) + } + _ => Ok(quote_phys_identifier(&m.physical_tag_column_name(name))), } } @@ -1570,31 +1595,31 @@ fn regex_match_column_name( typ: Some(FieldType::Tag), } => { let col = mapping - .map(|m| m.tag_column_name(name)) + .map(|m| m.physical_tag_column_name(name)) .unwrap_or_else(|| name.clone()); - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } Expr::FieldRef { name, typ: Some(FieldType::Field), - } => Ok(quote_identifier(name)), + } => quote_identifier(name), Expr::FieldRef { name, typ: None } => { let col = mapping - .map(|m| m.tag_column_name(name)) + .map(|m| m.physical_tag_column_name(name)) .unwrap_or_else(|| name.clone()); - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } Expr::Identifier(n) => { let col = if let Some(m) = mapping { if tag_field_collision(m, n) { - m.tag_column_name(n) + m.physical_tag_column_name(n) } else { m.physical_select_identifier(n) } } else { n.clone() }; - Ok(quote_identifier(&col)) + Ok(quote_phys_identifier(&col)) } _ => Err(HyperbytedbError::QueryParse( "regex operator =~ / !~ requires identifier and regex".to_string(), @@ -1646,7 +1671,7 @@ fn try_translate_where_binary_expr( if !tag_field_collision(m, name) { return Ok(false); } - let col = where_identifier_physical_name(m, name, lit); + let col = where_identifier_physical_name(m, name, lit)?; if id_on_left { write!(out, "{}", col)?; write!(out, " {} ", binary_op_to_clickhouse(&be.op))?; @@ -1672,36 +1697,40 @@ fn translate_expr( } else if in_where { if let Some(m) = mapping { if tag_field_collision(m, name) { - write!(out, "{}", quote_identifier(&m.tag_column_name(name)))?; + write!( + out, + "{}", + quote_phys_identifier(&m.physical_tag_column_name(name)) + )?; } else { - write!(out, "{}", quote_identifier(name))?; + write!(out, "{}", quote_identifier(name)?)?; } } else { - write!(out, "{}", quote_identifier(name))?; + write!(out, "{}", quote_identifier(name)?)?; } } else { - write!(out, "{}", quote_identifier(name))?; + write!(out, "{}", quote_identifier(name)?)?; } } Expr::FieldRef { name, typ } => { let s = match typ { Some(FieldType::Tag) => { if let Some(m) = mapping { - quote_identifier(&m.tag_column_name(name)) + quote_phys_identifier(&m.physical_tag_column_name(name)) } else { - quote_identifier(name) + quote_identifier(name)? } } - Some(FieldType::Field) => quote_identifier(name), + Some(FieldType::Field) => quote_identifier(name)?, None => { if let Some(m) = mapping { if tag_field_collision(m, name) { - quote_identifier(&m.tag_column_name(name)) + quote_phys_identifier(&m.physical_tag_column_name(name)) } else { - quote_identifier(name) + quote_identifier(name)? } } else { - quote_identifier(name) + quote_identifier(name)? } } }; @@ -1883,7 +1912,20 @@ fn binary_op_to_clickhouse(op: &BinaryOp) -> &'static str { } } -fn quote_identifier(name: &str) -> String { +fn quote_identifier(name: &str) -> Result { + if name.chars().any(char::is_control) { + return Err(HyperbytedbError::QueryParse(format!( + "identifier contains control characters: {name:?}" + ))); + } + Ok(format!( + "\"{}\"", + name.replace('\\', "\\\\").replace('"', "\\\"") + )) +} + +/// Quote a physical column name from [`crate::domain::chdb_naming`] (already sanitized). +fn quote_phys_identifier(name: &str) -> String { format!("\"{}\"", name.replace('\\', "\\\\").replace('"', "\\\"")) } @@ -1971,8 +2013,8 @@ pub fn prepare_cq_select( /// `INSERT INTO SELECT ...` for a bounded CQ run against native tables. pub fn translate_bounded_cq_into( stmt: &SelectStatement, - dest_table: &str, - source_table: &str, + dest_table: &QuotedTableName, + source_table: &QuotedTableName, mapping: Option<&ColumnMapping>, series: Option>, start_nanos: i64, @@ -1982,7 +2024,7 @@ pub fn translate_bounded_cq_into( let prepared = prepare_cq_select(stmt, start_nanos, end_nanos, false); let select_sql = translate_inner( &prepared, - source_table, + source_table.as_str(), mapping, series, Some((Some(start_nanos), Some(end_nanos))), @@ -1994,15 +2036,24 @@ pub fn translate_bounded_cq_into( #[cfg(test)] mod tests { use super::*; + use crate::domain::chdb_naming::QuotedTableName; use crate::timeseriesql::parser; - const TEST_TABLE: &str = "`mydb_autogen_cpu`"; + fn test_table() -> QuotedTableName { + QuotedTableName::new_quoted("`mydb_autogen_cpu`".to_string()) + } - fn translate_test(stmt: &SelectStatement) -> String { - translate_native_table(stmt, TEST_TABLE, None, None, None).unwrap() + fn test_series_table() -> QuotedTableName { + QuotedTableName::new_quoted("`mydb_autogen_cpu_series`".to_string()) } - const SERIES_TABLE: &str = "`mydb_autogen_cpu_series`"; + fn qname(s: &str) -> QuotedTableName { + QuotedTableName::new_quoted(s.to_string()) + } + + fn translate_test(stmt: &SelectStatement) -> String { + translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() + } /// Mapping with `host` as a tag and `usage_idle` as a field (no collision). fn cpu_mapping() -> ColumnMapping { @@ -2013,14 +2064,15 @@ mod tests { } } - /// Translate with a series join available (force = false). fn translate_series(stmt: &SelectStatement, m: &ColumnMapping) -> String { + let table = test_table(); + let series = test_series_table(); translate_native_table( stmt, - TEST_TABLE, + table.as_str(), Some(m), Some(SeriesJoin { - table: SERIES_TABLE, + table: &series, force: false, tag_columns: &[], }), @@ -2037,6 +2089,25 @@ mod tests { } } + #[test] + fn group_by_tag_uses_physical_column_name() { + let mut map = ColumnMapping::default(); + map.tag_keys.insert("host-name".into()); + map.field_names.insert("v".into()); + let stmt = parse_select(r#"SELECT mean("v") FROM m GROUP BY time(1m), "host-name""#); + let sql = translate_series(&stmt, &map); + assert!( + sql.contains("\"host_name\""), + "tag with punctuation must map to sanitized physical column, got: {sql}" + ); + } + + #[test] + fn quote_identifier_rejects_control_characters() { + assert!(quote_identifier("host\ninject").is_err()); + assert!(quote_identifier("ok_name").is_ok()); + } + #[test] fn test_select_star() { let stmt = parse_select("SELECT * FROM cpu"); @@ -2196,9 +2267,14 @@ mod tests { ); let min = 1_781_541_739_132_000_000i64; let max = 1_781_552_539_132_000_000i64; - let sql = - translate_native_table(&stmt, TEST_TABLE, None, None, Some((Some(min), Some(max)))) - .unwrap(); + let sql = translate_native_table( + &stmt, + test_table().as_str(), + None, + None, + Some((Some(min), Some(max))), + ) + .unwrap(); assert!( sql.contains("WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541739132000000), INTERVAL 10 SECOND)"), "expected FROM bound aligned to bucket, got: {sql}" @@ -2225,9 +2301,14 @@ mod tests { ); let min = 1_781_541_730_000_000_000i64; let max = 1_781_541_790_000_000_000i64; - let sql = - translate_native_table(&stmt, TEST_TABLE, None, None, Some((Some(min), Some(max)))) - .unwrap(); + let sql = translate_native_table( + &stmt, + test_table().as_str(), + None, + None, + Some((Some(min), Some(max))), + ) + .unwrap(); assert!( sql.contains( "WITH FILL FROM toStartOfInterval(fromUnixTimestamp64Nano(1781541730000000000) - INTERVAL 30 SECOND, INTERVAL 1 MINUTE) + INTERVAL 30 SECOND" @@ -2352,7 +2433,8 @@ mod tests { fn test_select_into_does_not_default_fill() { // Writes must not insert synthetic NULL grid rows. let stmt = parse_select(r#"SELECT mean("value") INTO "dest" FROM "cpu" GROUP BY time(5m)"#); - let sql = translate_select_into(&stmt, "`dest`", "`src`", None).unwrap(); + let sql = + translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).unwrap(); assert!( !sql.contains("WITH FILL"), "SELECT INTO without fill() must not emit WITH FILL, got: {sql}" @@ -2744,8 +2826,13 @@ mod tests { fn test_translate_select_into() { let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu" WHERE "host" = 'server01' GROUP BY time(1h), "host""#; let stmt = parse_select(q); - let sql = translate_select_into(&stmt, "`mydb_autogen_cpu_1h`", "`mydb_autogen_cpu`", None) - .unwrap(); + let sql = translate_select_into( + &stmt, + &qname("`mydb_autogen_cpu_1h`"), + test_table().as_str(), + None, + ) + .unwrap(); assert!(sql.starts_with("INSERT INTO `mydb_autogen_cpu_1h`")); assert!(sql.contains("SELECT ")); assert!(sql.contains("time")); @@ -2759,7 +2846,9 @@ mod tests { fn test_select_into_requires_group_by_time() { let q = r#"SELECT mean("value") INTO "cpu_1h" FROM "cpu""#; let stmt = parse_select(q); - assert!(translate_select_into(&stmt, "`dest`", "`source`", None).is_err()); + assert!( + translate_select_into(&stmt, &qname("`dest`"), test_table().as_str(), None).is_err() + ); } #[test] @@ -2769,8 +2858,8 @@ mod tests { let map = cpu_mapping(); let sql = translate_materialized_view_select( &stmt, - "`mydb_autogen_cpu`", - "`mydb_autogen_cpu_series`", + &test_table(), + &test_series_table(), "cpu_5m", &map, ) @@ -2818,9 +2907,9 @@ mod tests { let map = cpu_mapping(); let sql = translate_materialized_view_backfill( &stmt, - "`dest`", - "`source`", - "`source_series`", + &qname("`dest`"), + &qname("`source`"), + &qname("`source_series`"), "server_stats_1m", &map, ) @@ -2844,7 +2933,7 @@ mod tests { let mut map = cpu_mapping(); map.field_rollups .insert("usage_idle".to_string(), RollupCombine::Sum); - let sql = build_coalesced_fact_view(TEST_TABLE, &map); + let sql = build_coalesced_fact_view(&test_table(), &map); assert!( sql.contains("sum(\"usage_idle\") AS \"usage_idle\""), "rollup fields should merge with sum(), got: {sql}" @@ -2858,7 +2947,7 @@ mod tests { #[test] fn raw_fact_view_still_uses_argmax_without_rollups() { let map = cpu_mapping(); - let sql = build_coalesced_fact_view(TEST_TABLE, &map); + let sql = build_coalesced_fact_view(&test_table(), &map); assert!( sql.contains("argMax(\"usage_idle\", `ingest_seq`)"), "raw measurements should keep argMax coalesce, got: {sql}" @@ -2883,12 +2972,14 @@ mod tests { .insert("count_value".to_string(), RollupCombine::Sum); let stmt = parse_select(r#"SELECT mean("value") FROM cpu GROUP BY time(5m), "host""#); + let table = test_table(); + let series = test_series_table(); let sql = translate_native_table( &stmt, - TEST_TABLE, + table.as_str(), Some(&map), Some(SeriesJoin { - table: SERIES_TABLE, + table: &series, force: false, tag_columns: &[], }), @@ -2907,12 +2998,14 @@ mod tests { let mut map = ColumnMapping::default(); map.tag_keys.insert("cpu".into()); map.field_names.insert("cpu".into()); + let table = test_table(); + let series = test_series_table(); let sql = translate_native_table( &stmt, - TEST_TABLE, + table.as_str(), Some(&map), Some(SeriesJoin { - table: SERIES_TABLE, + table: &series, force: false, tag_columns: &[], }), @@ -2968,12 +3061,14 @@ mod tests { ] { map.field_names.insert(f.into()); } + let table = test_table(); + let series = test_series_table(); let sql = translate_native_table( &stmt, - TEST_TABLE, + table.as_str(), Some(&map), Some(SeriesJoin { - table: SERIES_TABLE, + table: &series, force: false, tag_columns: &[], }), @@ -3055,12 +3150,14 @@ mod tests { // force=true (e.g. a tombstone references a tag) joins even a field-only body. let stmt = parse_select(r#"SELECT mean("usage_idle") FROM cpu WHERE time > 0"#); let m = cpu_mapping(); + let table = test_table(); + let series = test_series_table(); let sql = translate_native_table( &stmt, - TEST_TABLE, + table.as_str(), Some(&m), Some(SeriesJoin { - table: SERIES_TABLE, + table: &series, force: true, tag_columns: &[], }), @@ -3091,7 +3188,7 @@ mod tests { let sql = translate_materialized_view_series_select( &stmt, - "`source_series`", + &qname("`source_series`"), "dest", &src_mapping, Some(&dest_field_names), @@ -3114,7 +3211,7 @@ mod tests { let sql = translate_materialized_view_series_select( &stmt, - "`source_series`", + &qname("`source_series`"), "dest", &src_mapping, None, @@ -3352,7 +3449,8 @@ mod tests { fill: None, timezone: None, }; - let inner_sql = translate_native_table(&inner, TEST_TABLE, None, None, None).unwrap(); + let inner_sql = + translate_native_table(&inner, test_table().as_str(), None, None, None).unwrap(); let inner_sql = rename_time_bucket_alias(&inner_sql); assert!( inner_sql.contains("AS time"), @@ -3403,7 +3501,7 @@ mod tests { stmt.timezone = Some("America/New_York".to_string()); let sql = translate_native_table( &stmt, - TEST_TABLE, + test_table().as_str(), None, None, Some((Some(1_000_000_000), Some(3_000_000_000))), @@ -3437,6 +3535,6 @@ mod tests { } fn translate_test_tz(stmt: &SelectStatement) -> String { - translate_native_table(stmt, TEST_TABLE, None, None, None).unwrap() + translate_native_table(stmt, test_table().as_str(), None, None, None).unwrap() } } diff --git a/hyperbytedb/tests/compat/cq_tests.rs b/hyperbytedb/tests/compat/cq_tests.rs index af562b1..a148d9d 100644 --- a/hyperbytedb/tests/compat/cq_tests.rs +++ b/hyperbytedb/tests/compat/cq_tests.rs @@ -322,7 +322,7 @@ async fn advanced_every_cq_recomputes_current_hour_bucket() { ); let run_at = Utc.with_ymd_and_hms(2016, 8, 28, 8, 30, 0).unwrap(); - let window = coverage_window(run_at, &cq); + let window = coverage_window(run_at, &cq).unwrap(); assert_eq!( window.start, Utc.with_ymd_and_hms(2016, 8, 28, 8, 0, 0).unwrap() diff --git a/hyperbytedb/tests/compat/http_tests.rs b/hyperbytedb/tests/compat/http_tests.rs index 7f6a71b..f55c5b0 100644 --- a/hyperbytedb/tests/compat/http_tests.rs +++ b/hyperbytedb/tests/compat/http_tests.rs @@ -120,6 +120,7 @@ impl HttpTestContext { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, diff --git a/hyperbytedb/tests/compat/main.rs b/hyperbytedb/tests/compat/main.rs index c662280..db60fcd 100644 --- a/hyperbytedb/tests/compat/main.rs +++ b/hyperbytedb/tests/compat/main.rs @@ -28,7 +28,7 @@ use hyperbytedb::application::ingestion_service::IngestionServiceImpl; use hyperbytedb::application::query_service::QueryServiceImpl; use hyperbytedb::domain::point::Point; use hyperbytedb::domain::query_result::QueryResponse; -use hyperbytedb::error::HyperbytedbError; +use hyperbytedb::error::{ChainedError, HyperbytedbError}; use hyperbytedb::ports::ingestion::{IngestionPort, WritePayloadFormat}; use hyperbytedb::ports::metadata::MetadataPort; use hyperbytedb::ports::points_sink::{PointsSinkPort, WriteAck}; @@ -85,7 +85,8 @@ impl TestContext { /// Create a TestContext that uses chDB for queries. /// Fails if chDB is not available. pub fn new() -> Result { - let tmpdir = tempfile::tempdir().map_err(|e| HyperbytedbError::Internal(e.to_string()))?; + let tmpdir = tempfile::tempdir() + .map_err(|e| HyperbytedbError::Internal(ChainedError::from_error(e)))?; let root = tmpdir.path(); let wal_path = root.join("wal"); @@ -134,7 +135,8 @@ impl TestContext { /// Create a TestContext that uses a mock QueryPort (no chDB required). /// Use for metadata, DDL, ingestion, and error tests. pub fn new_no_chdb() -> Result { - let tmpdir = tempfile::tempdir().map_err(|e| HyperbytedbError::Internal(e.to_string()))?; + let tmpdir = tempfile::tempdir() + .map_err(|e| HyperbytedbError::Internal(ChainedError::from_error(e)))?; let root = tmpdir.path(); let wal_path = root.join("wal"); diff --git a/hyperbytedb/tests/integration.rs b/hyperbytedb/tests/integration.rs index 07c3f5c..a55734f 100644 --- a/hyperbytedb/tests/integration.rs +++ b/hyperbytedb/tests/integration.rs @@ -102,6 +102,7 @@ fn setup(dir: &tempfile::TempDir) -> (Arc, Arc) { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, @@ -186,6 +187,7 @@ async fn test_auth_blocks_unauthenticated() { drain_service: None, raft: None, auth_enabled: true, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, @@ -224,13 +226,23 @@ async fn test_auth_blocks_unauthenticated() { .unwrap(); assert_eq!( resp.status(), - StatusCode::OK, - "Authenticated query should succeed" + StatusCode::UNAUTHORIZED, + "Query-param credentials should be rejected when allow_query_param_credentials is false" ); let resp = client .get(format!("{url}/query")) - .query(&[("q", "SHOW DATABASES"), ("u", "admin"), ("p", "wrong")]) + .query(&[("q", "SHOW DATABASES")]) + .basic_auth("admin", Some("secret123")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK, "Basic auth should succeed"); + + let resp = client + .get(format!("{url}/query")) + .query(&[("q", "SHOW DATABASES")]) + .basic_auth("admin", Some("wrong")) .send() .await .unwrap(); @@ -289,6 +301,7 @@ async fn test_cardinality_limit() { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, @@ -437,6 +450,7 @@ async fn test_metrics_endpoint() { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: Some(prometheus_handle), statement_summary: None, statement_summary_require_auth: true, @@ -773,6 +787,7 @@ async fn test_rate_limiter_refills_and_denies() { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: Some(prometheus_handle), statement_summary: None, statement_summary_require_auth: true, @@ -929,6 +944,7 @@ async fn test_cross_database_on_clause_requires_authorization() { drain_service: None, raft: None, auth_enabled: true, + auth_allow_query_param_credentials: true, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, diff --git a/hyperbytedb/tests/raft_integration.rs b/hyperbytedb/tests/raft_integration.rs index 6002a87..12e8a93 100644 --- a/hyperbytedb/tests/raft_integration.rs +++ b/hyperbytedb/tests/raft_integration.rs @@ -138,6 +138,7 @@ async fn start_cluster_node_with_listener( drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, @@ -339,6 +340,7 @@ async fn test_cluster_endpoints_without_peers() { drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true, diff --git a/hyperbytedb/tests/security_auth.rs b/hyperbytedb/tests/security_auth.rs index 78c9bcc..0f56530 100644 --- a/hyperbytedb/tests/security_auth.rs +++ b/hyperbytedb/tests/security_auth.rs @@ -146,6 +146,7 @@ async fn start_auth_cluster_node(dir: &std::path::Path) -> AuthClusterNode { drain_service: None, raft: None, auth_enabled: true, + auth_allow_query_param_credentials: true, prometheus_handle: None, statement_summary: Some(Arc::new(StatementSummary::new(100))), statement_summary_require_auth: true, diff --git a/hyperbytedb/tests/sync_quorum_integration.rs b/hyperbytedb/tests/sync_quorum_integration.rs index 1c5810e..29c7d74 100644 --- a/hyperbytedb/tests/sync_quorum_integration.rs +++ b/hyperbytedb/tests/sync_quorum_integration.rs @@ -180,6 +180,7 @@ async fn start_node_on( drain_service: None, raft: None, auth_enabled: false, + auth_allow_query_param_credentials: false, prometheus_handle: None, statement_summary: None, statement_summary_require_auth: true,